tristantarrant commented on issue #3399: URL: https://github.com/apache/logging-log4j2/issues/3399#issuecomment-2598502697
We've been seeing these in the Infinispan testsuite and we don't enable vthreads by default. ``` 2025-01-17 05:28:09 Full thread dump OpenJDK 64-Bit Server VM (21.0.4+7-LTS mixed mode, sharing): "testng-ConflictManagerTest" #47 [950191] prio=5 os_prio=0 cpu=14892.05ms elapsed=436.52s tid=0x00007f02818f49d0 nid=950191 waiting on condition [0x00007f02125f1000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.$$BlockHound$$_park(java.base@21.0.4/Native Method) - parking to wait for <0x00000000c06328c8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at jdk.internal.misc.Unsafe.park(java.base@21.0.4/Unsafe.java) at java.util.concurrent.locks.LockSupport.park(java.base@21.0.4/LockSupport.java:221) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@21.0.4/AbstractQueuedSynchronizer.java:754) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@21.0.4/AbstractQueuedSynchronizer.java:1079) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@21.0.4/ReentrantReadWriteLock.java:738) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.getLogger(InternalLoggerRegistry.java:71) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.computeIfAbsent(InternalLoggerRegistry.java:145) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:566) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:70) at org.apache.logging.log4j.spi.LoggerContext.getLogger(LoggerContext.java:59) at org.apache.logging.log4j.LogManager.getLogger(LogManager.java:571) at org.apache.logging.log4j.LogManager.getFormatterLogger(LogManager.java:454) at org.jgroups.logging.Log4J2LogImpl.<init>(Log4J2LogImpl.java:26) at org.jgroups.logging.LogFactory.getLog(LogFactory.java:69) at org.jgroups.stack.Protocol.<init>(Protocol.java:68) at org.jgroups.protocols.pbcast.NAKACK2.<init>(NAKACK2.java:42) at java.lang.invoke.LambdaForm$DMH/0x00007f0218ce3400.newInvokeSpecial(java.base@21.0.4/LambdaForm$DMH) at java.lang.invoke.Invokers$Holder.invokeExact_MT(java.base@21.0.4/Invokers$Holder) at jdk.internal.reflect.DirectConstructorHandleAccessor.invokeImpl(java.base@21.0.4/DirectConstructorHandleAccessor.java:86) at jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(java.base@21.0.4/DirectConstructorHandleAccessor.java:62) at java.lang.reflect.Constructor.newInstanceWithCaller(java.base@21.0.4/Constructor.java:502) at java.lang.reflect.Constructor.newInstance(java.base@21.0.4/Constructor.java:486) at org.jgroups.stack.Configurator.createLayer(Configurator.java:190) at org.jgroups.stack.Configurator.createProtocols(Configurator.java:170) at org.jgroups.stack.Configurator.createProtocolsAndInitializeAttrs(Configurator.java:104) at org.jgroups.stack.Configurator.setupProtocolStack(Configurator.java:65) at org.jgroups.stack.Configurator.setupProtocolStack(Configurator.java:55) at org.jgroups.stack.ProtocolStack.setup(ProtocolStack.java:439) at org.jgroups.JChannel.init(JChannel.java:916) at org.jgroups.JChannel.<init>(JChannel.java:128) at org.jgroups.JChannel.<init>(JChannel.java:118) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.buildChannel(JGroupsTransport.java:708) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.initChannel(JGroupsTransport.java:467) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.start(JGroupsTransport.java:451) at org.infinispan.remoting.transport.jgroups.CorePackageImpl$2.start(CorePackageImpl.java:64) at org.infinispan.remoting.transport.jgroups.CorePackageImpl$2.start(CorePackageImpl.java:49) at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:616) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:607) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.GlobalComponentRegistry.preStart(GlobalComponentRegistry.java:307) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:241) at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:783) at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:751) at org.infinispan.test.MultipleCacheManagersTest.addClusterEnabledCacheManager(MultipleCacheManagersTest.java:278) at org.infinispan.test.MultipleCacheManagersTest.createClusteredCaches(MultipleCacheManagersTest.java:444) at org.infinispan.test.MultipleCacheManagersTest.createClusteredCaches(MultipleCacheManagersTest.java:415) at org.infinispan.partitionhandling.BasePartitionHandlingTest.createCacheManagers(BasePartitionHandlingTest.java:79) at org.infinispan.conflict.impl.ConflictManagerTest.createCacheManagers(ConflictManagerTest.java:71) at org.infinispan.test.MultipleCacheManagersTest.callCreateCacheManagers(MultipleCacheManagersTest.java:124) at org.infinispan.test.MultipleCacheManagersTest.createBeforeMethod(MultipleCacheManagersTest.java:134) at java.lang.invoke.LambdaForm$DMH/0x00007f0218464c00.invokeVirtual(java.base@21.0.4/LambdaForm$DMH) at java.lang.invoke.LambdaForm$MH/0x00007f0218c78400.invoke(java.base@21.0.4/LambdaForm$MH) at java.lang.invoke.Invokers$Holder.invokeExact_MT(java.base@21.0.4/Invokers$Holder) at jdk.internal.reflect.DirectMethodHandleAccessor.invokeImpl(java.base@21.0.4/DirectMethodHandleAccessor.java:153) at jdk.internal.reflect.DirectMethodHandleAccessor.invoke(java.base@21.0.4/DirectMethodHandleAccessor.java:103) at java.lang.reflect.Method.invoke(java.base@21.0.4/Method.java:580) at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:124) at org.testng.internal.MethodInvocationHelper.invokeMethodConsideringTimeout(MethodInvocationHelper.java:59) at org.testng.internal.Invoker.invokeConfigurationMethod(Invoker.java:458) at org.testng.internal.Invoker.invokeConfigurations(Invoker.java:222) at org.testng.internal.Invoker.invokeMethod(Invoker.java:523) at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:719) at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:989) at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:125) at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:109) at org.testng.TestRunner.privateRun(TestRunner.java:648) at org.testng.TestRunner.run(TestRunner.java:505) at org.testng.SuiteRunner.runTest(SuiteRunner.java:455) at org.testng.SuiteRunner.access$000(SuiteRunner.java:40) at org.testng.SuiteRunner$SuiteWorker.run(SuiteRunner.java:489) at org.testng.internal.thread.ThreadUtil$1.call(ThreadUtil.java:52) at java.util.concurrent.FutureTask.run(java.base@21.0.4/FutureTask.java:317) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@21.0.4/ThreadPoolExecutor.java:1144) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@21.0.4/ThreadPoolExecutor.java:642) at java.lang.Thread.runWith(java.base@21.0.4/Thread.java:1596) at java.lang.Thread.run(java.base@21.0.4/Thread.java:1583) Locked ownable synchronizers: - <0x00000000c1822d90> (a java.util.concurrent.ThreadPoolExecutor$Worker) "jgroups-7,IracCustomConflictTest-NodeA" #23652 [973529] prio=5 os_prio=0 cpu=7.10ms elapsed=353.61s tid=0x00007f0281676820 nid=973529 waiting on condition [0x00007f018f9e7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.$$BlockHound$$_park(java.base@21.0.4/Native Method) - parking to wait for <0x00000000c06328c8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at jdk.internal.misc.Unsafe.park(java.base@21.0.4/Unsafe.java) at java.util.concurrent.locks.LockSupport.park(java.base@21.0.4/LockSupport.java:221) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@21.0.4/AbstractQueuedSynchronizer.java:754) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@21.0.4/AbstractQueuedSynchronizer.java:1079) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@21.0.4/ReentrantReadWriteLock.java:738) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.getLogger(InternalLoggerRegistry.java:71) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.computeIfAbsent(InternalLoggerRegistry.java:145) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:566) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:539) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:70) at org.apache.logging.log4j.LogManager.getLogger(LogManager.java:622) at org.jboss.logging.Log4j2Logger.<init>(Log4j2Logger.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:32) at org.jboss.logging.Logger.getLogger(Logger.java:2467) at org.infinispan.util.logging.LogFactory.getLogger(LogFactory.java:28) at org.infinispan.util.logging.events.impl.BasicEventLogger.log(BasicEventLogger.java:57) at org.infinispan.util.logging.events.impl.DecoratedEventLogger.log(DecoratedEventLogger.java:41) at org.infinispan.util.logging.events.EventLogger.info(EventLogger.java:41) at org.infinispan.topology.ClusterCacheStatus.startQueuedRebalance(ClusterCacheStatus.java:1007) - locked <0x00000000dca10cc0> (a org.infinispan.topology.ClusterCacheStatus) at org.infinispan.topology.ClusterCacheStatus.queueRebalance(ClusterCacheStatus.java:152) - locked <0x00000000dca10cc0> (a org.infinispan.topology.ClusterCacheStatus) at org.infinispan.partitionhandling.impl.PreferAvailabilityStrategy.onJoin(PreferAvailabilityStrategy.java:40) at org.infinispan.topology.ClusterCacheStatus.doJoin(ClusterCacheStatus.java:747) - locked <0x00000000dca10cc0> (a org.infinispan.topology.ClusterCacheStatus) at org.infinispan.topology.ClusterTopologyManagerImpl.lambda$handleJoin$4(ClusterTopologyManagerImpl.java:265) at org.infinispan.topology.ClusterTopologyManagerImpl$$Lambda/0x00007f02189fe3e8.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniApplyNow(java.base@21.0.4/CompletableFuture.java:684) at java.util.concurrent.CompletableFuture.uniApplyStage(java.base@21.0.4/CompletableFuture.java:662) at java.util.concurrent.CompletableFuture.thenApply(java.base@21.0.4/CompletableFuture.java:2200) at java.util.concurrent.CompletableFuture.thenApply(java.base@21.0.4/CompletableFuture.java:144) at org.infinispan.topology.ClusterTopologyManagerImpl.lambda$handleJoin$5(ClusterTopologyManagerImpl.java:265) at org.infinispan.topology.ClusterTopologyManagerImpl$$Lambda/0x00007f02189f5930.apply(Unknown Source) at java.util.concurrent.CompletableFuture.uniComposeStage(java.base@21.0.4/CompletableFuture.java:1187) at java.util.concurrent.CompletableFuture.thenCompose(java.base@21.0.4/CompletableFuture.java:2341) at java.util.concurrent.CompletableFuture.thenCompose(java.base@21.0.4/CompletableFuture.java:144) at org.infinispan.topology.ClusterTopologyManagerImpl.handleJoin(ClusterTopologyManagerImpl.java:257) at org.infinispan.commands.topology.CacheJoinCommand.invokeAsync(CacheJoinCommand.java:46) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler$ReplicableCommandRunner.run(GlobalInboundInvocationHandler.java:156) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleReplicableCommand(GlobalInboundInvocationHandler.java:133) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleFromCluster(GlobalInboundInvocationHandler.java:79) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1527) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1454) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1649) at org.jgroups.JChannel.up(JChannel.java:748) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:941) at org.jgroups.protocols.relay.RELAY2.up(RELAY2.java:153) at org.jgroups.protocols.FRAG2.up(FRAG2.java:139) at org.jgroups.protocols.FlowControl.up(FlowControl.java:253) at org.jgroups.protocols.FlowControl.up(FlowControl.java:261) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:853) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:235) at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1144) at org.jgroups.protocols.UNICAST3.addMessage(UNICAST3.java:880) at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:862) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:474) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:669) at org.jgroups.protocols.VERIFY_SUSPECT2.up(VERIFY_SUSPECT2.java:105) at org.jgroups.protocols.FailureDetection.up(FailureDetection.java:180) at org.jgroups.protocols.Discovery.up(Discovery.java:296) at org.jgroups.stack.Protocol.up(Protocol.java:360) at org.jgroups.protocols.TP.passMessageUp(Unknown Source) at org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:95) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@21.0.4/ThreadPoolExecutor.java:1144) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@21.0.4/ThreadPoolExecutor.java:642) at java.lang.Thread.runWith(java.base@21.0.4/Thread.java:1596) at java.lang.Thread.run(java.base@21.0.4/Thread.java:1583) Locked ownable synchronizers: - <0x00000000dca10fe0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "jgroups-7,RetryMechanismTest-NodeK" #36714 [986397] prio=5 os_prio=0 cpu=1.80ms elapsed=311.55s tid=0x00007f01f405b510 nid=986397 waiting on condition [0x00007f018ab99000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.$$BlockHound$$_park(java.base@21.0.4/Native Method) - parking to wait for <0x00000000c06328c8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at jdk.internal.misc.Unsafe.park(java.base@21.0.4/Unsafe.java) at java.util.concurrent.locks.LockSupport.park(java.base@21.0.4/LockSupport.java:221) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@21.0.4/AbstractQueuedSynchronizer.java:754) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@21.0.4/AbstractQueuedSynchronizer.java:1079) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@21.0.4/ReentrantReadWriteLock.java:738) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.getLogger(InternalLoggerRegistry.java:71) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.computeIfAbsent(InternalLoggerRegistry.java:145) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:566) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:70) at org.apache.logging.log4j.spi.LoggerContext.getLogger(LoggerContext.java:59) at org.apache.logging.log4j.LogManager.getLogger(LogManager.java:571) at org.apache.logging.log4j.LogManager.getFormatterLogger(LogManager.java:454) at org.jgroups.logging.Log4J2LogImpl.<init>(Log4J2LogImpl.java:26) at org.jgroups.logging.LogFactory.getLog(LogFactory.java:69) at org.jgroups.stack.Protocol.<init>(Protocol.java:68) at org.jgroups.protocols.RED.<init>(RED.java:26) at java.lang.invoke.LambdaForm$DMH/0x00007f0218ce2c00.newInvokeSpecial(java.base@21.0.4/LambdaForm$DMH) at java.lang.invoke.Invokers$Holder.invokeExact_MT(java.base@21.0.4/Invokers$Holder) at jdk.internal.reflect.DirectConstructorHandleAccessor.invokeImpl(java.base@21.0.4/DirectConstructorHandleAccessor.java:86) at jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(java.base@21.0.4/DirectConstructorHandleAccessor.java:62) at java.lang.reflect.Constructor.newInstanceWithCaller(java.base@21.0.4/Constructor.java:502) at java.lang.reflect.Constructor.newInstance(java.base@21.0.4/Constructor.java:486) at org.jgroups.stack.Configurator.createLayer(Configurator.java:190) at org.jgroups.stack.Configurator.createProtocols(Configurator.java:170) at org.jgroups.stack.Configurator.createProtocolsAndInitializeAttrs(Configurator.java:104) at org.jgroups.stack.Configurator.setupProtocolStack(Configurator.java:65) at org.jgroups.stack.Configurator.setupProtocolStack(Configurator.java:55) at org.jgroups.stack.ProtocolStack.setup(ProtocolStack.java:439) at org.jgroups.JChannel.init(JChannel.java:916) at org.jgroups.JChannel.<init>(JChannel.java:128) at org.jgroups.JChannel.<init>(JChannel.java:109) at org.jgroups.protocols.relay.config.RelayConfig$PropertiesBridgeConfig.createChannel(RelayConfig.java:178) at org.jgroups.protocols.relay.Relayer2.start(Relayer2.java:45) at org.jgroups.protocols.relay.RELAY2.startRelayer(RELAY2.java:476) at org.jgroups.protocols.relay.RELAY2.lambda$handleView$2(RELAY2.java:239) at org.jgroups.protocols.relay.RELAY2$$Lambda/0x00007f0218d2b0b8.run(Unknown Source) at org.jgroups.util.TimeScheduler3$Task.run(TimeScheduler3.java:332) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@21.0.4/ThreadPoolExecutor.java:1144) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@21.0.4/ThreadPoolExecutor.java:642) at java.lang.Thread.runWith(java.base@21.0.4/Thread.java:1596) at java.lang.Thread.run(java.base@21.0.4/Thread.java:1583) Locked ownable synchronizers: - <0x00000000c3d088b0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "jgroups-8,FunctionalTxInMemoryTest-NodeA" #48780 [998446] prio=5 os_prio=0 cpu=2.07ms elapsed=277.36s tid=0x00007f01b0405020 nid=998446 waiting on condition [0x00007f0183826000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.$$BlockHound$$_park(java.base@21.0.4/Native Method) - parking to wait for <0x00000000c06328c8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at jdk.internal.misc.Unsafe.park(java.base@21.0.4/Unsafe.java) at java.util.concurrent.locks.LockSupport.park(java.base@21.0.4/LockSupport.java:221) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@21.0.4/AbstractQueuedSynchronizer.java:754) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@21.0.4/AbstractQueuedSynchronizer.java:1079) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@21.0.4/ReentrantReadWriteLock.java:738) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.getLogger(InternalLoggerRegistry.java:71) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.computeIfAbsent(InternalLoggerRegistry.java:145) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:566) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:539) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:70) at org.apache.logging.log4j.LogManager.getLogger(LogManager.java:622) at org.jboss.logging.Log4j2Logger.<init>(Log4j2Logger.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:32) at org.jboss.logging.Logger.getLogger(Logger.java:2467) at org.infinispan.util.logging.LogFactory.getLogger(LogFactory.java:28) at org.infinispan.util.logging.events.impl.BasicEventLogger.log(BasicEventLogger.java:57) at org.infinispan.util.logging.events.impl.DecoratedEventLogger.log(DecoratedEventLogger.java:41) at org.infinispan.util.logging.events.EventLogger.info(EventLogger.java:41) at org.infinispan.topology.ClusterCacheStatus.endReadNewPhase(ClusterCacheStatus.java:483) at org.infinispan.topology.ClusterCacheStatus$$Lambda/0x00007f0218a7f898.run(Unknown Source) at org.infinispan.topology.RebalanceConfirmationCollector.confirmPhase(RebalanceConfirmationCollector.java:54) - locked <0x00000000c2646e58> (a org.infinispan.topology.RebalanceConfirmationCollector) at org.infinispan.topology.ClusterCacheStatus.confirmRebalancePhase(ClusterCacheStatus.java:362) - locked <0x00000000c2646d20> (a org.infinispan.topology.ClusterCacheStatus) at org.infinispan.topology.ClusterTopologyManagerImpl.handleRebalancePhaseConfirm(ClusterTopologyManagerImpl.java:340) at org.infinispan.commands.topology.RebalancePhaseConfirmCommand.invokeAsync(RebalancePhaseConfirmCommand.java:43) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler$ReplicableCommandRunner.run(GlobalInboundInvocationHandler.java:156) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleReplicableCommand(GlobalInboundInvocationHandler.java:133) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleFromCluster(GlobalInboundInvocationHandler.java:79) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1527) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1454) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1649) at org.jgroups.JChannel.up(JChannel.java:748) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:941) at org.jgroups.protocols.FRAG2.up(FRAG2.java:139) at org.jgroups.protocols.FlowControl.up(FlowControl.java:253) at org.jgroups.protocols.FlowControl.up(FlowControl.java:261) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:853) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:235) at org.jgroups.protocols.UNICAST3.deliverMessage(UNICAST3.java:1144) at org.jgroups.protocols.UNICAST3.addMessage(UNICAST3.java:880) at org.jgroups.protocols.UNICAST3.handleDataReceived(UNICAST3.java:862) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:474) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:669) at org.jgroups.protocols.Discovery.up(Discovery.java:296) at org.jgroups.stack.Protocol.up(Protocol.java:360) at org.jgroups.protocols.TP.passMessageUp(Unknown Source) at org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:95) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@21.0.4/ThreadPoolExecutor.java:1144) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@21.0.4/ThreadPoolExecutor.java:642) at java.lang.Thread.runWith(java.base@21.0.4/Thread.java:1596) at java.lang.Thread.run(java.base@21.0.4/Thread.java:1583) Locked ownable synchronizers: - <0x00000000c3d08cf0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "jgroups-7,StateTransferLinkFailuresTest-NodeK" #49037 [998702] prio=5 os_prio=0 cpu=2.61ms elapsed=276.80s tid=0x00007f01c4098380 nid=998702 waiting on condition [0x00007f0186452000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.$$BlockHound$$_park(java.base@21.0.4/Native Method) - parking to wait for <0x00000000c06328c8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at jdk.internal.misc.Unsafe.park(java.base@21.0.4/Unsafe.java) at java.util.concurrent.locks.LockSupport.park(java.base@21.0.4/LockSupport.java:221) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@21.0.4/AbstractQueuedSynchronizer.java:754) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@21.0.4/AbstractQueuedSynchronizer.java:1079) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@21.0.4/ReentrantReadWriteLock.java:738) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.getLogger(InternalLoggerRegistry.java:71) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.computeIfAbsent(InternalLoggerRegistry.java:145) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:566) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:539) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:70) at org.apache.logging.log4j.LogManager.getLogger(LogManager.java:622) at org.jboss.logging.Log4j2Logger.<init>(Log4j2Logger.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:32) at org.jboss.logging.Logger.getLogger(Logger.java:2467) at org.infinispan.util.logging.LogFactory.getLogger(LogFactory.java:28) at org.infinispan.util.logging.events.impl.BasicEventLogger.log(BasicEventLogger.java:57) at org.infinispan.util.logging.events.impl.DecoratedEventLogger.log(DecoratedEventLogger.java:41) at org.infinispan.util.logging.events.EventLogger.info(EventLogger.java:41) at org.infinispan.topology.LocalTopologyManagerImpl.handleRebalance(LocalTopologyManagerImpl.java:663) at org.infinispan.commands.topology.RebalanceStartCommand.invokeAsync(RebalanceStartCommand.java:59) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler$ReplicableCommandRunner.run(GlobalInboundInvocationHandler.java:156) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleReplicableCommand(GlobalInboundInvocationHandler.java:133) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleFromCluster(GlobalInboundInvocationHandler.java:79) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1527) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1454) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.lambda$up$0(JGroupsTransport.java:1663) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks$$Lambda/0x00007f0218a78688.accept(Unknown Source) at java.lang.Iterable.forEach(java.base@21.0.4/Iterable.java:75) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1655) at org.jgroups.JChannel.up(JChannel.java:764) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:944) at org.jgroups.protocols.relay.RELAY2.up(RELAY2.java:209) at org.jgroups.protocols.FRAG2.up(FRAG2.java:161) at org.jgroups.protocols.FlowControl.up(FlowControl.java:319) at org.jgroups.protocols.FlowControl.up(FlowControl.java:319) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:867) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:255) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:519) at org.jgroups.protocols.pbcast.NAKACK2.deliverBatch(NAKACK2.java:1028) at org.jgroups.protocols.pbcast.NAKACK2.handleMessageBatch(NAKACK2.java:925) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:758) at org.jgroups.protocols.VERIFY_SUSPECT2.up(VERIFY_SUSPECT2.java:119) at org.jgroups.protocols.FailureDetection.up(FailureDetection.java:193) at org.jgroups.protocols.Discovery.up(Discovery.java:316) at org.jgroups.protocols.RED.up(RED.java:123) at org.jgroups.protocols.TP.passBatchUp(Unknown Source) at org.jgroups.util.SubmitToThreadPool$BatchHandler.passBatchUp(SubmitToThreadPool.java:137) at org.jgroups.util.SubmitToThreadPool$BatchHandler.run(SubmitToThreadPool.java:133) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@21.0.4/ThreadPoolExecutor.java:1144) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@21.0.4/ThreadPoolExecutor.java:642) at java.lang.Thread.runWith(java.base@21.0.4/Thread.java:1596) at java.lang.Thread.run(java.base@21.0.4/Thread.java:1583) Locked ownable synchronizers: - <0x00000000db8d7c28> (a java.util.concurrent.ThreadPoolExecutor$Worker) "jgroups-8,GroupsChFunctionalTest-NodeC" #68221 [1017748] prio=5 os_prio=0 cpu=3.46ms elapsed=162.26s tid=0x00007f01f806de40 nid=1017748 waiting on condition [0x00007f019d4d2000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.$$BlockHound$$_park(java.base@21.0.4/Native Method) - parking to wait for <0x00000000c06328c8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at jdk.internal.misc.Unsafe.park(java.base@21.0.4/Unsafe.java) at java.util.concurrent.locks.LockSupport.park(java.base@21.0.4/LockSupport.java:221) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@21.0.4/AbstractQueuedSynchronizer.java:754) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@21.0.4/AbstractQueuedSynchronizer.java:1079) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@21.0.4/ReentrantReadWriteLock.java:738) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.getLogger(InternalLoggerRegistry.java:71) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.computeIfAbsent(InternalLoggerRegistry.java:145) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:566) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:539) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:70) at org.apache.logging.log4j.LogManager.getLogger(LogManager.java:622) at org.jboss.logging.Log4j2Logger.<init>(Log4j2Logger.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:32) at org.jboss.logging.Logger.getLogger(Logger.java:2467) at org.infinispan.util.logging.LogFactory.getLogger(LogFactory.java:28) at org.infinispan.util.logging.events.impl.BasicEventLogger.log(BasicEventLogger.java:57) at org.infinispan.util.logging.events.impl.DecoratedEventLogger.log(DecoratedEventLogger.java:41) at org.infinispan.util.logging.events.EventLogger.info(EventLogger.java:41) at org.infinispan.topology.LocalTopologyManagerImpl.handleRebalance(LocalTopologyManagerImpl.java:663) at org.infinispan.commands.topology.RebalanceStartCommand.invokeAsync(RebalanceStartCommand.java:59) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler$ReplicableCommandRunner.run(GlobalInboundInvocationHandler.java:156) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleReplicableCommand(GlobalInboundInvocationHandler.java:133) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleFromCluster(GlobalInboundInvocationHandler.java:79) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1527) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1454) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.lambda$up$0(JGroupsTransport.java:1663) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks$$Lambda/0x00007f0218a78688.accept(Unknown Source) at java.lang.Iterable.forEach(java.base@21.0.4/Iterable.java:75) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1655) at org.jgroups.JChannel.up(JChannel.java:764) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:944) at org.jgroups.protocols.FRAG2.up(FRAG2.java:161) at org.jgroups.protocols.FlowControl.up(FlowControl.java:319) at org.jgroups.protocols.FlowControl.up(FlowControl.java:319) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:867) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:255) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:519) at org.jgroups.protocols.pbcast.NAKACK2.deliverBatch(NAKACK2.java:1028) at org.jgroups.protocols.pbcast.NAKACK2.handleMessageBatch(NAKACK2.java:925) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:758) at org.jgroups.protocols.Discovery.up(Discovery.java:316) at org.jgroups.protocols.RED.up(RED.java:123) at org.jgroups.protocols.TP.passBatchUp(Unknown Source) at org.jgroups.util.SubmitToThreadPool$BatchHandler.passBatchUp(SubmitToThreadPool.java:137) at org.jgroups.util.SubmitToThreadPool$BatchHandler.run(SubmitToThreadPool.java:133) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@21.0.4/ThreadPoolExecutor.java:1144) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@21.0.4/ThreadPoolExecutor.java:642) at java.lang.Thread.runWith(java.base@21.0.4/Thread.java:1596) at java.lang.Thread.run(java.base@21.0.4/Thread.java:1583) Locked ownable synchronizers: - <0x00000000d3000648> (a java.util.concurrent.ThreadPoolExecutor$Worker) "jgroups-6,DistSyncOnePhaseWriteSkewTxStateTransferTest-NodeBS" #78708 [1028101] prio=5 os_prio=0 cpu=0.50ms elapsed=109.00s tid=0x00007f01ac40f850 nid=1028101 waiting on condition [0x00007f02117e4000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.$$BlockHound$$_park(java.base@21.0.4/Native Method) - parking to wait for <0x00000000c06328c8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at jdk.internal.misc.Unsafe.park(java.base@21.0.4/Unsafe.java) at java.util.concurrent.locks.LockSupport.park(java.base@21.0.4/LockSupport.java:221) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@21.0.4/AbstractQueuedSynchronizer.java:754) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@21.0.4/AbstractQueuedSynchronizer.java:1079) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@21.0.4/ReentrantReadWriteLock.java:738) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.getLogger(InternalLoggerRegistry.java:71) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.computeIfAbsent(InternalLoggerRegistry.java:145) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:566) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:70) at org.apache.logging.log4j.spi.LoggerContext.getLogger(LoggerContext.java:59) at org.apache.logging.log4j.LogManager.getLogger(LogManager.java:571) at org.apache.logging.log4j.LogManager.getFormatterLogger(LogManager.java:454) at org.jgroups.logging.Log4J2LogImpl.<init>(Log4J2LogImpl.java:26) at org.jgroups.logging.LogFactory.getLog(LogFactory.java:69) at org.jgroups.stack.Protocol.<init>(Protocol.java:68) at org.jgroups.protocols.UNICAST3.<init>(UNICAST3.java:39) at java.lang.invoke.LambdaForm$DMH/0x00007f0218cf2000.newInvokeSpecial(java.base@21.0.4/LambdaForm$DMH) at java.lang.invoke.Invokers$Holder.invokeExact_MT(java.base@21.0.4/Invokers$Holder) at jdk.internal.reflect.DirectConstructorHandleAccessor.invokeImpl(java.base@21.0.4/DirectConstructorHandleAccessor.java:86) at jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(java.base@21.0.4/DirectConstructorHandleAccessor.java:62) at java.lang.reflect.Constructor.newInstanceWithCaller(java.base@21.0.4/Constructor.java:502) at java.lang.reflect.Constructor.newInstance(java.base@21.0.4/Constructor.java:486) at org.jgroups.stack.Configurator.createLayer(Configurator.java:190) at org.jgroups.stack.Configurator.createProtocols(Configurator.java:170) at org.jgroups.stack.Configurator.createProtocolsAndInitializeAttrs(Configurator.java:104) at org.jgroups.stack.Configurator.setupProtocolStack(Configurator.java:65) at org.jgroups.stack.Configurator.setupProtocolStack(Configurator.java:55) at org.jgroups.stack.ProtocolStack.setup(ProtocolStack.java:439) at org.jgroups.JChannel.init(JChannel.java:916) at org.jgroups.JChannel.<init>(JChannel.java:128) at org.jgroups.JChannel.<init>(JChannel.java:109) at org.jgroups.protocols.relay.config.RelayConfig$PropertiesBridgeConfig.createChannel(RelayConfig.java:178) at org.jgroups.protocols.relay.Relayer2.start(Relayer2.java:45) at org.jgroups.protocols.relay.RELAY2.startRelayer(RELAY2.java:476) at org.jgroups.protocols.relay.RELAY2.lambda$handleView$2(RELAY2.java:239) at org.jgroups.protocols.relay.RELAY2$$Lambda/0x00007f0218d2b0b8.run(Unknown Source) at org.jgroups.util.TimeScheduler3$Task.run(TimeScheduler3.java:332) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@21.0.4/ThreadPoolExecutor.java:1144) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@21.0.4/ThreadPoolExecutor.java:642) at java.lang.Thread.runWith(java.base@21.0.4/Thread.java:1596) at java.lang.Thread.run(java.base@21.0.4/Thread.java:1583) Locked ownable synchronizers: - <0x00000000c6224bf0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "jgroups-6,IracLocalStateTransferTest-NodeB" #93184 [1042334] prio=5 os_prio=0 cpu=2.77ms elapsed=44.39s tid=0x00007f01cc2e6ef0 nid=1042334 waiting on condition [0x00007f0195478000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.$$BlockHound$$_park(java.base@21.0.4/Native Method) - parking to wait for <0x00000000c06328c8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at jdk.internal.misc.Unsafe.park(java.base@21.0.4/Unsafe.java) at java.util.concurrent.locks.LockSupport.park(java.base@21.0.4/LockSupport.java:221) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@21.0.4/AbstractQueuedSynchronizer.java:754) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@21.0.4/AbstractQueuedSynchronizer.java:1079) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@21.0.4/ReentrantReadWriteLock.java:738) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.getLogger(InternalLoggerRegistry.java:71) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.computeIfAbsent(InternalLoggerRegistry.java:145) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:566) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:539) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:70) at org.apache.logging.log4j.LogManager.getLogger(LogManager.java:622) at org.jboss.logging.Log4j2Logger.<init>(Log4j2Logger.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:32) at org.jboss.logging.Logger.getLogger(Logger.java:2467) at org.infinispan.util.logging.LogFactory.getLogger(LogFactory.java:28) at org.infinispan.util.logging.events.impl.BasicEventLogger.log(BasicEventLogger.java:57) at org.infinispan.util.logging.events.impl.DecoratedEventLogger.log(DecoratedEventLogger.java:41) at org.infinispan.util.logging.events.EventLogger.info(EventLogger.java:41) at org.infinispan.topology.LocalTopologyManagerImpl.handleRebalance(LocalTopologyManagerImpl.java:663) at org.infinispan.commands.topology.RebalanceStartCommand.invokeAsync(RebalanceStartCommand.java:59) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler$ReplicableCommandRunner.run(GlobalInboundInvocationHandler.java:156) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleReplicableCommand(GlobalInboundInvocationHandler.java:133) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleFromCluster(GlobalInboundInvocationHandler.java:79) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1527) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1454) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.lambda$up$0(JGroupsTransport.java:1663) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks$$Lambda/0x00007f0218a78688.accept(Unknown Source) at java.lang.Iterable.forEach(java.base@21.0.4/Iterable.java:75) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1655) at org.jgroups.JChannel.up(JChannel.java:764) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:944) at org.jgroups.protocols.relay.RELAY2.up(RELAY2.java:209) at org.jgroups.protocols.FRAG2.up(FRAG2.java:161) at org.jgroups.protocols.FlowControl.up(FlowControl.java:319) at org.jgroups.protocols.FlowControl.up(FlowControl.java:319) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:867) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:255) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:519) at org.jgroups.protocols.pbcast.NAKACK2.deliverBatch(NAKACK2.java:1028) at org.jgroups.protocols.pbcast.NAKACK2.handleMessageBatch(NAKACK2.java:925) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:758) at org.jgroups.protocols.VERIFY_SUSPECT2.up(VERIFY_SUSPECT2.java:119) at org.jgroups.protocols.FailureDetection.up(FailureDetection.java:193) at org.jgroups.protocols.Discovery.up(Discovery.java:316) at org.jgroups.protocols.RED.up(RED.java:123) at org.jgroups.protocols.TP.passBatchUp(TP.java:1269) at org.jgroups.util.SubmitToThreadPool$BatchHandler.passBatchUp(SubmitToThreadPool.java:137) at org.jgroups.util.SubmitToThreadPool$BatchHandler.run(SubmitToThreadPool.java:133) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@21.0.4/ThreadPoolExecutor.java:1144) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@21.0.4/ThreadPoolExecutor.java:642) at java.lang.Thread.runWith(java.base@21.0.4/Thread.java:1596) at java.lang.Thread.run(java.base@21.0.4/Thread.java:1583) Locked ownable synchronizers: - <0x00000000c1316b68> (a java.util.concurrent.ThreadPoolExecutor$Worker) "jgroups-6,IracLocalStateTransferTest-NodeC" #94029 [1043151] prio=5 os_prio=0 cpu=1.56ms elapsed=39.41s tid=0x00007f020c7c85d0 nid=1043151 waiting on condition [0x00007f0211be9000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.$$BlockHound$$_park(java.base@21.0.4/Native Method) - parking to wait for <0x00000000c06328c8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at jdk.internal.misc.Unsafe.park(java.base@21.0.4/Unsafe.java) at java.util.concurrent.locks.LockSupport.park(java.base@21.0.4/LockSupport.java:221) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@21.0.4/AbstractQueuedSynchronizer.java:754) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@21.0.4/AbstractQueuedSynchronizer.java:990) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(java.base@21.0.4/ReentrantReadWriteLock.java:959) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.computeIfAbsent(InternalLoggerRegistry.java:151) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:566) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:539) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:70) at org.apache.logging.log4j.LogManager.getLogger(LogManager.java:622) at org.jboss.logging.Log4j2Logger.<init>(Log4j2Logger.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:32) at org.jboss.logging.Logger.getLogger(Logger.java:2467) at org.infinispan.util.logging.LogFactory.getLogger(LogFactory.java:28) at org.infinispan.util.logging.events.impl.BasicEventLogger.log(BasicEventLogger.java:57) at org.infinispan.util.logging.events.impl.DecoratedEventLogger.log(DecoratedEventLogger.java:41) at org.infinispan.util.logging.events.EventLogger.info(EventLogger.java:41) at org.infinispan.topology.LocalTopologyManagerImpl.handleRebalance(LocalTopologyManagerImpl.java:663) at org.infinispan.commands.topology.RebalanceStartCommand.invokeAsync(RebalanceStartCommand.java:59) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler$ReplicableCommandRunner.run(GlobalInboundInvocationHandler.java:156) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleReplicableCommand(GlobalInboundInvocationHandler.java:133) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleFromCluster(GlobalInboundInvocationHandler.java:79) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1527) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1454) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1649) at org.jgroups.JChannel.up(JChannel.java:748) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:941) at org.jgroups.protocols.relay.RELAY2.up(RELAY2.java:153) at org.jgroups.protocols.FRAG2.up(FRAG2.java:139) at org.jgroups.protocols.FlowControl.up(FlowControl.java:261) at org.jgroups.protocols.FlowControl.up(FlowControl.java:253) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:853) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:235) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:462) at org.jgroups.protocols.pbcast.NAKACK2.deliver(NAKACK2.java:1006) at org.jgroups.protocols.pbcast.NAKACK2.handleMessage(NAKACK2.java:894) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:679) at org.jgroups.protocols.VERIFY_SUSPECT2.up(VERIFY_SUSPECT2.java:105) at org.jgroups.protocols.FailureDetection.up(FailureDetection.java:180) at org.jgroups.protocols.Discovery.up(Discovery.java:296) at org.jgroups.stack.Protocol.up(Protocol.java:360) at org.jgroups.protocols.TP.passMessageUp(TP.java:1232) at org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:95) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@21.0.4/ThreadPoolExecutor.java:1144) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@21.0.4/ThreadPoolExecutor.java:642) at java.lang.Thread.runWith(java.base@21.0.4/Thread.java:1596) at java.lang.Thread.run(java.base@21.0.4/Thread.java:1583) Locked ownable synchronizers: - <0x00000000c62003f0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "jgroups-9,IracLocalStateTransferTest-NodeH" #95762 [1044837] prio=5 os_prio=0 cpu=0.38ms elapsed=31.12s tid=0x00007f026007c560 nid=1044837 waiting on condition [0x00007f019cac9000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.$$BlockHound$$_park(java.base@21.0.4/Native Method) - parking to wait for <0x00000000c06328c8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at jdk.internal.misc.Unsafe.park(java.base@21.0.4/Unsafe.java) at java.util.concurrent.locks.LockSupport.park(java.base@21.0.4/LockSupport.java:221) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@21.0.4/AbstractQueuedSynchronizer.java:754) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@21.0.4/AbstractQueuedSynchronizer.java:1079) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@21.0.4/ReentrantReadWriteLock.java:738) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.getLogger(InternalLoggerRegistry.java:71) at org.apache.logging.log4j.core.util.internal.InternalLoggerRegistry.computeIfAbsent(InternalLoggerRegistry.java:145) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:566) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:539) at org.apache.logging.log4j.core.LoggerContext.getLogger(LoggerContext.java:70) at org.apache.logging.log4j.LogManager.getLogger(LogManager.java:622) at org.jboss.logging.Log4j2Logger.<init>(Log4j2Logger.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:36) at org.jboss.logging.Log4j2LoggerProvider.getLogger(Log4j2LoggerProvider.java:32) at org.jboss.logging.Logger.getLogger(Logger.java:2467) at org.infinispan.util.logging.LogFactory.getLogger(LogFactory.java:28) at org.infinispan.util.logging.events.impl.BasicEventLogger.log(BasicEventLogger.java:57) at org.infinispan.util.logging.events.impl.DecoratedEventLogger.log(DecoratedEventLogger.java:41) at org.infinispan.util.logging.events.EventLogger.info(EventLogger.java:41) at org.infinispan.topology.LocalTopologyManagerImpl.handleRebalance(LocalTopologyManagerImpl.java:663) at org.infinispan.commands.topology.RebalanceStartCommand.invokeAsync(RebalanceStartCommand.java:59) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler$ReplicableCommandRunner.run(GlobalInboundInvocationHandler.java:156) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleReplicableCommand(GlobalInboundInvocationHandler.java:133) at org.infinispan.remoting.inboundhandler.GlobalInboundInvocationHandler.handleFromCluster(GlobalInboundInvocationHandler.java:79) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processRequest(JGroupsTransport.java:1527) at org.infinispan.remoting.transport.jgroups.JGroupsTransport.processMessage(JGroupsTransport.java:1454) at org.infinispan.remoting.transport.jgroups.JGroupsTransport$ChannelCallbacks.up(JGroupsTransport.java:1649) at org.jgroups.JChannel.up(JChannel.java:748) at org.jgroups.stack.ProtocolStack.up(ProtocolStack.java:941) at org.jgroups.protocols.relay.RELAY2.up(RELAY2.java:153) at org.jgroups.protocols.FRAG2.up(FRAG2.java:139) at org.jgroups.protocols.FlowControl.up(FlowControl.java:261) at org.jgroups.protocols.FlowControl.up(FlowControl.java:253) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:853) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:235) at org.jgroups.protocols.UNICAST3.up(UNICAST3.java:462) at org.jgroups.protocols.pbcast.NAKACK2.deliver(NAKACK2.java:1006) at org.jgroups.protocols.pbcast.NAKACK2.handleMessage(NAKACK2.java:894) at org.jgroups.protocols.pbcast.NAKACK2.up(NAKACK2.java:679) at org.jgroups.protocols.VERIFY_SUSPECT2.up(VERIFY_SUSPECT2.java:105) at org.jgroups.protocols.FailureDetection.up(FailureDetection.java:180) at org.jgroups.protocols.Discovery.up(Discovery.java:296) at org.jgroups.stack.Protocol.up(Protocol.java:360) at org.jgroups.protocols.TP.passMessageUp(TP.java:1232) at org.jgroups.util.SubmitToThreadPool$SingleMessageHandler.run(SubmitToThreadPool.java:95) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@21.0.4/ThreadPoolExecutor.java:1144) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@21.0.4/ThreadPoolExecutor.java:642) at java.lang.Thread.runWith(java.base@21.0.4/Thread.java:1596) at java.lang.Thread.run(java.base@21.0.4/Thread.java:1583) Locked ownable synchronizers: - <0x00000000dccb3a50> (a java.util.concurrent.ThreadPoolExecutor$Worker) ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@logging.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org