I have a Java application that runs an embedded infinispan instance. A Jgroups configuration in the app allows infinispan to join other instances of my app running horizontally. So in a cluster of 3 instances there are 3 embedded infinispan instances running at 3 different posts 7800, 7801, 7802.
I use the 'Clumsy' tool to drop traffic to/from my third node's 7802 port and then as expected my other two nodes will have their nodeRemoved() listener methods fired which is great. (Note that node 3 is still running, its just that I have switched off its local/internet traffic for its infinispan port to simulate isolation)
Now, if I re-enable traffic on Node 3's 7802 port, I notice that infinispan there never seems to properly rejoin or trigger nodeAdded() in the other nodes.
I am wondering what I am missing
infinispan.xml and jgroups stack , which I have swapped out for now
<infinispan xmlns="urn:infinispan:config:13.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="urn:infinispan:config:13.0
https://infinispan.org/schemas/infinispan-config-13.0.xsd">
<jgroups>
<!-- Creates a custom JGroups stack named "my-stack". -->
<!-- Inherits properties from the default TCP stack. -->
<stack name="my-stack">
<TCP
bind_addr="${jgroups.bind_addr:site_local}"
bind_port="${jgroups.bind_port:7800}"
recv_buf_size="${tcp.recv_buf_size:130k}"
send_buf_size="${tcp.send_buf_size:130k}"
max_bundle_size="64K"
sock_conn_timeout="300"
thread_pool.min_threads="0"
thread_pool.max_threads="20"
thread_pool.keep_alive_time="30000"/>
<RED/>
<TCPPING async_discovery="true"
initial_hosts="172.26.240.1[7800],172.26.240.1[7801],172.26.240.1[7802]"
port_range="2"/>
<MERGE3 min_interval="10000"
max_interval="30000"/>
<FD_SOCK/>
<FD_ALL timeout="9000" interval="3000"/>
<VERIFY_SUSPECT timeout="1500"/>
<BARRIER/>
<pbcast.NAKACK2 use_mcast_xmit="false"
discard_delivered_msgs="true"/>
<UNICAST3/>
<pbcast.STABLE desired_avg_gossip="50000"
max_bytes="4M"/>
<pbcast.GMS print_local_addr="true" join_timeout="2000"/>
<UFC max_credits="2M"
min_threshold="0.4"/>
<MFC max_credits="2M"
min_threshold="0.4"/>
<FRAG2 frag_size="60K"/>
<!--RSVP resend_interval="2000" timeout="10000"/-->
<COUNTER bypass_bundling="true" timeout="5000"/>
<pbcast.STATE_TRANSFER/>
</stack>
</jgroups>
<cache-container>
<metrics gauges="false" histograms="false"/>
<transport stack="tcp" cluster="${cluster.name}" initial-cluster-size="1"/>
<!-- <transport stack="my-stack" cluster="${cluster.name}" initial-cluster-size="1"/>-->
<serialization marshaller="org.infinispan.commons.marshall.JavaSerializationMarshaller">
<allow-list>
<regex>.*</regex>
</allow-list>
</serialization>
<distributed-cache name="my-sessions">
<encoding>
<key media-type="application/x-java-serialized-object"/>
<value media-type="application/x-java-serialized-object"/>
</encoding>
</distributed-cache>
<!-- A map which is used for cluster-wide properties. -->
<distributed-cache name="cluster-properties">
<encoding>
<key media-type="application/x-java-serialized-object"/>
<value media-type="application/x-java-serialized-object"/>
</encoding>
</distributed-cache>
<!-- A map which is used for cache synchronization. -->
<replicated-cache name="pl-commit-messages">
<encoding>
<key media-type="application/x-java-serialized-object"/>
<value media-type="application/x-java-serialized-object"/>
</encoding>
</replicated-cache>
</cache-container>
</infinispan>