[ https://issues.apache.org/jira/browse/GEODE-10332?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Kirk Lund updated GEODE-10332: ------------------------------ Description: NOTE: ClientAuthenticationDUnitTest is a dunit test that runs during geode-core:upgradeTest which is running tests in parallel. To be well-behaved all ports must come from AvailablePortHelper and be well below the range of ephemeral ports. This failure was seen during repeated precheckin runs for PR #7571. Initial analysis of output suggests that some of the following test problems might be contributing: # Some ports are using AvailablePortHelper while others seem to be using default ports. Any use of default ports will eventually cause failures of one or more tests when they run in parallel. # The membership port range is too high. It's up in the ephemeral port range which means it's eventually going to collide with ephemeral ports being used by other processes. # The assertions in ClientAuthenticationTestCase don't provide enough info so debugging this test will require a lot of digging through log files and studying test code in multiple class files. {noformat} org.apache.geode.test.dunit.RMIException: While invoking org.apache.geode.security.ClientAuthenticationTestCase$$Lambda$490/0x000000010041e040.call in VM 1, Host Host heavy-lifter-afcd3383-60cd-5694-909f-3bc9af935683.c.apachegeode-ci.internal with 4 VMs, Geode 10240.0.0, Java 11 at org.apache.geode.test.dunit.VM.executeMethodOnObject(VM.java:688) at org.apache.geode.test.dunit.VM.invoke(VM.java:525) at org.apache.geode.security.ClientAuthenticationTestCase.doTestCredentialsForNotifications(ClientAuthenticationTestCase.java:595) at org.apache.geode.security.ClientAuthenticationDUnitTest.testCredentialsForNotifications(ClientAuthenticationDUnitTest.java:104) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:566) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runner.JUnitCore.run(JUnitCore.java:137) at org.junit.runner.JUnitCore.run(JUnitCore.java:115) at org.junit.vintage.engine.execution.RunnerExecutor.execute(RunnerExecutor.java:42) at org.junit.vintage.engine.VintageTestEngine.executeAllChildren(VintageTestEngine.java:80) at org.junit.vintage.engine.VintageTestEngine.execute(VintageTestEngine.java:72) at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:108) at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88) at org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54) at org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67) at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52) at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:96) at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:75) at org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.processAllTestClasses(JUnitPlatformTestClassProcessor.java:99) at org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.access$000(JUnitPlatformTestClassProcessor.java:79) at org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor.stop(JUnitPlatformTestClassProcessor.java:75) at org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.stop(SuiteTestClassProcessor.java:61) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:566) at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36) at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) at org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33) at org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94) at com.sun.proxy.$Proxy2.stop(Unknown Source) at org.gradle.api.internal.tasks.testing.worker.TestWorker.stop(TestWorker.java:133) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:566) at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36) at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) at org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:182) at org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:164) at org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:414) at org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64) at org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:48) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at org.gradle.internal.concurrent.ThreadFactoryImpl$ManagedThreadRunnable.run(ThreadFactoryImpl.java:56) at java.lang.Thread.run(Thread.java:829) Caused by: java.lang.AssertionError: Got unexpected exception when starting peer at org.apache.geode.test.dunit.Assert.fail(Assert.java:66) at org.apache.geode.security.SecurityTestUtils.createCacheServer(SecurityTestUtils.java:220) at org.apache.geode.security.SecurityTestUtils.createCacheServer(SecurityTestUtils.java:185) at org.apache.geode.security.ClientAuthenticationTestUtils.createCacheServer(ClientAuthenticationTestUtils.java:67) at org.apache.geode.security.ClientAuthenticationTestUtils.createCacheServer(ClientAuthenticationTestUtils.java:43) at org.apache.geode.security.ClientAuthenticationTestCase.lambda$doTestCredentialsForNotifications$4695cb09$1(ClientAuthenticationTestCase.java:596) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:566) at org.apache.geode.test.dunit.internal.MethodInvoker.executeObject(MethodInvoker.java:123) at org.apache.geode.test.dunit.internal.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:78) at jdk.internal.reflect.GeneratedMethodAccessor293.invoke(Unknown Source) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:566) at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:359) at sun.rmi.transport.Transport$1.run(Transport.java:200) at sun.rmi.transport.Transport$1.run(Transport.java:197) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.Transport.serviceCall(Transport.java:196) at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:562) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:796) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:677) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:676) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ... 1 more Caused by: org.apache.geode.GemFireConfigException: Problem configuring membership services at org.apache.geode.distributed.internal.DistributionImpl.start(DistributionImpl.java:184) at org.apache.geode.distributed.internal.DistributionImpl.createDistribution(DistributionImpl.java:222) at org.apache.geode.distributed.internal.ClusterDistributionManager.<init>(ClusterDistributionManager.java:488) at org.apache.geode.distributed.internal.ClusterDistributionManager.<init>(ClusterDistributionManager.java:521) at org.apache.geode.distributed.internal.ClusterDistributionManager.create(ClusterDistributionManager.java:350) at org.apache.geode.distributed.internal.InternalDistributedSystem$DefaultClusterDistributionManagerConstructor.create(InternalDistributedSystem.java:2993) at org.apache.geode.distributed.internal.InternalDistributedSystem.initialize(InternalDistributedSystem.java:758) at org.apache.geode.distributed.internal.InternalDistributedSystem$Builder.build(InternalDistributedSystem.java:3048) at org.apache.geode.distributed.internal.InternalDistributedSystem.connectInternal(InternalDistributedSystem.java:283) at org.apache.geode.distributed.internal.InternalDistributedSystem.connectInternal(InternalDistributedSystem.java:209) at org.apache.geode.distributed.DistributedSystem.connect(DistributedSystem.java:161) at org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase.getSystem(JUnit4DistributedTestCase.java:182) at org.apache.geode.security.SecurityTestUtils.createSystem(SecurityTestUtils.java:1760) at org.apache.geode.security.SecurityTestUtils.createCacheServer(SecurityTestUtils.java:202) ... 26 more Caused by: org.apache.geode.distributed.internal.membership.api.MembershipConfigurationException: Unable to join the distributed system. Could not contact any of the locators: [localhost:27917] at org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.join(GMSJoinLeave.java:415) at org.apache.geode.distributed.internal.membership.gms.GMSMembership.join(GMSMembership.java:536) at org.apache.geode.distributed.internal.membership.gms.GMSMembership.access$1200(GMSMembership.java:75) at org.apache.geode.distributed.internal.membership.gms.GMSMembership$ManagerImpl.joinDistributedSystem(GMSMembership.java:1754) at org.apache.geode.distributed.internal.membership.gms.Services.start(Services.java:247) at org.apache.geode.distributed.internal.membership.gms.GMSMembership.start(GMSMembership.java:1644) at org.apache.geode.distributed.internal.DistributionImpl.start(DistributionImpl.java:171) ... 39 more Caused by: java.net.SocketTimeoutException: Read timed out at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:115) at java.net.SocketInputStream.read(SocketInputStream.java:168) at java.net.SocketInputStream.read(SocketInputStream.java:140) at java.io.BufferedInputStream.fill(BufferedInputStream.java:252) at java.io.BufferedInputStream.read(BufferedInputStream.java:271) at java.io.FilterInputStream.read(FilterInputStream.java:83) at java.io.FilterInputStream.read(FilterInputStream.java:83) at org.apache.geode.distributed.internal.tcpserver.TcpClient.assertNotSslAlert(TcpClient.java:298) at org.apache.geode.distributed.internal.tcpserver.TcpClient.getServerVersion(TcpClient.java:274) at org.apache.geode.distributed.internal.tcpserver.TcpClient.getServerVersion(TcpClient.java:265) at org.apache.geode.distributed.internal.tcpserver.TcpClient.lambda$getServerVersion$0(TcpClient.java:252) at org.apache.geode.internal.lang.utils.function.Checked.lambda$rethrowFunction$0(Checked.java:42) at java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) at org.apache.geode.internal.lang.utils.JavaWorkarounds.computeIfAbsent(JavaWorkarounds.java:34) at org.apache.geode.distributed.internal.tcpserver.TcpClient.getServerVersion(TcpClient.java:247) at org.apache.geode.distributed.internal.tcpserver.TcpClient.requestToServer(TcpClient.java:144) at org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.findCoordinator(GMSJoinLeave.java:1182) at org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.join(GMSJoinLeave.java:343) ... 45 more {noformat} was: geode-core:upgradeTest This failure was seen during repeated precheckin runs for PR #7571. Initial analysis of output suggests that some of the following test problems might be contributing: # Some ports are using AvailablePortHelper while others seem to be using default ports. Any use of default ports will eventually cause failures of one or more tests when they run in parallel. # The membership port range is too high. It's up in the ephemeral port range which means it's eventually going to collide with ephemeral ports being used by other processes. # The assertions in ClientAuthenticationTestCase don't provide enough info so debugging of this test will require a lot of digging through log files and studying test code in multiple class files. {noformat} org.apache.geode.test.dunit.RMIException: While invoking org.apache.geode.security.ClientAuthenticationTestCase$$Lambda$490/0x000000010041e040.call in VM 1, Host Host heavy-lifter-afcd3383-60cd-5694-909f-3bc9af935683.c.apachegeode-ci.internal with 4 VMs, Geode 10240.0.0, Java 11 at org.apache.geode.test.dunit.VM.executeMethodOnObject(VM.java:688) at org.apache.geode.test.dunit.VM.invoke(VM.java:525) at org.apache.geode.security.ClientAuthenticationTestCase.doTestCredentialsForNotifications(ClientAuthenticationTestCase.java:595) at org.apache.geode.security.ClientAuthenticationDUnitTest.testCredentialsForNotifications(ClientAuthenticationDUnitTest.java:104) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:566) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runner.JUnitCore.run(JUnitCore.java:137) at org.junit.runner.JUnitCore.run(JUnitCore.java:115) at org.junit.vintage.engine.execution.RunnerExecutor.execute(RunnerExecutor.java:42) at org.junit.vintage.engine.VintageTestEngine.executeAllChildren(VintageTestEngine.java:80) at org.junit.vintage.engine.VintageTestEngine.execute(VintageTestEngine.java:72) at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:108) at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88) at org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54) at org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67) at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52) at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:96) at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:75) at org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.processAllTestClasses(JUnitPlatformTestClassProcessor.java:99) at org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.access$000(JUnitPlatformTestClassProcessor.java:79) at org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor.stop(JUnitPlatformTestClassProcessor.java:75) at org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.stop(SuiteTestClassProcessor.java:61) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:566) at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36) at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) at org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33) at org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94) at com.sun.proxy.$Proxy2.stop(Unknown Source) at org.gradle.api.internal.tasks.testing.worker.TestWorker.stop(TestWorker.java:133) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:566) at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36) at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) at org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:182) at org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:164) at org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:414) at org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64) at org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:48) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at org.gradle.internal.concurrent.ThreadFactoryImpl$ManagedThreadRunnable.run(ThreadFactoryImpl.java:56) at java.lang.Thread.run(Thread.java:829) Caused by: java.lang.AssertionError: Got unexpected exception when starting peer at org.apache.geode.test.dunit.Assert.fail(Assert.java:66) at org.apache.geode.security.SecurityTestUtils.createCacheServer(SecurityTestUtils.java:220) at org.apache.geode.security.SecurityTestUtils.createCacheServer(SecurityTestUtils.java:185) at org.apache.geode.security.ClientAuthenticationTestUtils.createCacheServer(ClientAuthenticationTestUtils.java:67) at org.apache.geode.security.ClientAuthenticationTestUtils.createCacheServer(ClientAuthenticationTestUtils.java:43) at org.apache.geode.security.ClientAuthenticationTestCase.lambda$doTestCredentialsForNotifications$4695cb09$1(ClientAuthenticationTestCase.java:596) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:566) at org.apache.geode.test.dunit.internal.MethodInvoker.executeObject(MethodInvoker.java:123) at org.apache.geode.test.dunit.internal.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:78) at jdk.internal.reflect.GeneratedMethodAccessor293.invoke(Unknown Source) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:566) at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:359) at sun.rmi.transport.Transport$1.run(Transport.java:200) at sun.rmi.transport.Transport$1.run(Transport.java:197) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.Transport.serviceCall(Transport.java:196) at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:562) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:796) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:677) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:676) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ... 1 more Caused by: org.apache.geode.GemFireConfigException: Problem configuring membership services at org.apache.geode.distributed.internal.DistributionImpl.start(DistributionImpl.java:184) at org.apache.geode.distributed.internal.DistributionImpl.createDistribution(DistributionImpl.java:222) at org.apache.geode.distributed.internal.ClusterDistributionManager.<init>(ClusterDistributionManager.java:488) at org.apache.geode.distributed.internal.ClusterDistributionManager.<init>(ClusterDistributionManager.java:521) at org.apache.geode.distributed.internal.ClusterDistributionManager.create(ClusterDistributionManager.java:350) at org.apache.geode.distributed.internal.InternalDistributedSystem$DefaultClusterDistributionManagerConstructor.create(InternalDistributedSystem.java:2993) at org.apache.geode.distributed.internal.InternalDistributedSystem.initialize(InternalDistributedSystem.java:758) at org.apache.geode.distributed.internal.InternalDistributedSystem$Builder.build(InternalDistributedSystem.java:3048) at org.apache.geode.distributed.internal.InternalDistributedSystem.connectInternal(InternalDistributedSystem.java:283) at org.apache.geode.distributed.internal.InternalDistributedSystem.connectInternal(InternalDistributedSystem.java:209) at org.apache.geode.distributed.DistributedSystem.connect(DistributedSystem.java:161) at org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase.getSystem(JUnit4DistributedTestCase.java:182) at org.apache.geode.security.SecurityTestUtils.createSystem(SecurityTestUtils.java:1760) at org.apache.geode.security.SecurityTestUtils.createCacheServer(SecurityTestUtils.java:202) ... 26 more Caused by: org.apache.geode.distributed.internal.membership.api.MembershipConfigurationException: Unable to join the distributed system. Could not contact any of the locators: [localhost:27917] at org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.join(GMSJoinLeave.java:415) at org.apache.geode.distributed.internal.membership.gms.GMSMembership.join(GMSMembership.java:536) at org.apache.geode.distributed.internal.membership.gms.GMSMembership.access$1200(GMSMembership.java:75) at org.apache.geode.distributed.internal.membership.gms.GMSMembership$ManagerImpl.joinDistributedSystem(GMSMembership.java:1754) at org.apache.geode.distributed.internal.membership.gms.Services.start(Services.java:247) at org.apache.geode.distributed.internal.membership.gms.GMSMembership.start(GMSMembership.java:1644) at org.apache.geode.distributed.internal.DistributionImpl.start(DistributionImpl.java:171) ... 39 more Caused by: java.net.SocketTimeoutException: Read timed out at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:115) at java.net.SocketInputStream.read(SocketInputStream.java:168) at java.net.SocketInputStream.read(SocketInputStream.java:140) at java.io.BufferedInputStream.fill(BufferedInputStream.java:252) at java.io.BufferedInputStream.read(BufferedInputStream.java:271) at java.io.FilterInputStream.read(FilterInputStream.java:83) at java.io.FilterInputStream.read(FilterInputStream.java:83) at org.apache.geode.distributed.internal.tcpserver.TcpClient.assertNotSslAlert(TcpClient.java:298) at org.apache.geode.distributed.internal.tcpserver.TcpClient.getServerVersion(TcpClient.java:274) at org.apache.geode.distributed.internal.tcpserver.TcpClient.getServerVersion(TcpClient.java:265) at org.apache.geode.distributed.internal.tcpserver.TcpClient.lambda$getServerVersion$0(TcpClient.java:252) at org.apache.geode.internal.lang.utils.function.Checked.lambda$rethrowFunction$0(Checked.java:42) at java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) at org.apache.geode.internal.lang.utils.JavaWorkarounds.computeIfAbsent(JavaWorkarounds.java:34) at org.apache.geode.distributed.internal.tcpserver.TcpClient.getServerVersion(TcpClient.java:247) at org.apache.geode.distributed.internal.tcpserver.TcpClient.requestToServer(TcpClient.java:144) at org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.findCoordinator(GMSJoinLeave.java:1182) at org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.join(GMSJoinLeave.java:343) ... 45 more {noformat} > ClientAuthenticationDUnitTest > testCredentialsForNotifications[Client > VmConfiguration{java=11, geode=1.7.0}] fails with SocketTimeoutException > after MembershipConfigurationException > -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- > > Key: GEODE-10332 > URL: https://issues.apache.org/jira/browse/GEODE-10332 > Project: Geode > Issue Type: Bug > Components: membership, tests > Reporter: Kirk Lund > Priority: Major > Labels: needsTriage > > NOTE: ClientAuthenticationDUnitTest is a dunit test that runs during > geode-core:upgradeTest which is running tests in parallel. To be well-behaved > all ports must come from AvailablePortHelper and be well below the range of > ephemeral ports. > This failure was seen during repeated precheckin runs for PR #7571. Initial > analysis of output suggests that some of the following test problems might be > contributing: > # Some ports are using AvailablePortHelper while others seem to be using > default ports. Any use of default ports will eventually cause failures of one > or more tests when they run in parallel. > # The membership port range is too high. It's up in the ephemeral port range > which means it's eventually going to collide with ephemeral ports being used > by other processes. > # The assertions in ClientAuthenticationTestCase don't provide enough info so > debugging this test will require a lot of digging through log files and > studying test code in multiple class files. > {noformat} > org.apache.geode.test.dunit.RMIException: While invoking > org.apache.geode.security.ClientAuthenticationTestCase$$Lambda$490/0x000000010041e040.call > in VM 1, Host Host > heavy-lifter-afcd3383-60cd-5694-909f-3bc9af935683.c.apachegeode-ci.internal > with 4 VMs, Geode 10240.0.0, Java 11 > at org.apache.geode.test.dunit.VM.executeMethodOnObject(VM.java:688) > at org.apache.geode.test.dunit.VM.invoke(VM.java:525) > at > org.apache.geode.security.ClientAuthenticationTestCase.doTestCredentialsForNotifications(ClientAuthenticationTestCase.java:595) > at > org.apache.geode.security.ClientAuthenticationDUnitTest.testCredentialsForNotifications(ClientAuthenticationDUnitTest.java:104) > at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:566) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) > at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) > at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) > at > org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) > at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) > at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) > at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) > at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) > at org.junit.runners.ParentRunner.run(ParentRunner.java:413) > at org.junit.runners.Suite.runChild(Suite.java:128) > at org.junit.runners.Suite.runChild(Suite.java:27) > at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) > at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) > at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) > at > org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) > at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) > at org.junit.runners.ParentRunner.run(ParentRunner.java:413) > at org.junit.runner.JUnitCore.run(JUnitCore.java:137) > at org.junit.runner.JUnitCore.run(JUnitCore.java:115) > at > org.junit.vintage.engine.execution.RunnerExecutor.execute(RunnerExecutor.java:42) > at > org.junit.vintage.engine.VintageTestEngine.executeAllChildren(VintageTestEngine.java:80) > at > org.junit.vintage.engine.VintageTestEngine.execute(VintageTestEngine.java:72) > at > org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:108) > at > org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88) > at > org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54) > at > org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67) > at > org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52) > at > org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:96) > at > org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:75) > at > org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.processAllTestClasses(JUnitPlatformTestClassProcessor.java:99) > at > org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.access$000(JUnitPlatformTestClassProcessor.java:79) > at > org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor.stop(JUnitPlatformTestClassProcessor.java:75) > at > org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.stop(SuiteTestClassProcessor.java:61) > at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:566) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33) > at > org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94) > at com.sun.proxy.$Proxy2.stop(Unknown Source) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.stop(TestWorker.java:133) > at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:566) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:182) > at > org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:164) > at > org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:414) > at > org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64) > at > org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:48) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > at > org.gradle.internal.concurrent.ThreadFactoryImpl$ManagedThreadRunnable.run(ThreadFactoryImpl.java:56) > at java.lang.Thread.run(Thread.java:829) > Caused by: java.lang.AssertionError: Got unexpected exception when starting > peer > at org.apache.geode.test.dunit.Assert.fail(Assert.java:66) > at > org.apache.geode.security.SecurityTestUtils.createCacheServer(SecurityTestUtils.java:220) > at > org.apache.geode.security.SecurityTestUtils.createCacheServer(SecurityTestUtils.java:185) > at > org.apache.geode.security.ClientAuthenticationTestUtils.createCacheServer(ClientAuthenticationTestUtils.java:67) > at > org.apache.geode.security.ClientAuthenticationTestUtils.createCacheServer(ClientAuthenticationTestUtils.java:43) > at > org.apache.geode.security.ClientAuthenticationTestCase.lambda$doTestCredentialsForNotifications$4695cb09$1(ClientAuthenticationTestCase.java:596) > at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:566) > at > org.apache.geode.test.dunit.internal.MethodInvoker.executeObject(MethodInvoker.java:123) > at > org.apache.geode.test.dunit.internal.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:78) > at jdk.internal.reflect.GeneratedMethodAccessor293.invoke(Unknown > Source) > at > jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:566) > at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:359) > at sun.rmi.transport.Transport$1.run(Transport.java:200) > at sun.rmi.transport.Transport$1.run(Transport.java:197) > at java.security.AccessController.doPrivileged(Native Method) > at sun.rmi.transport.Transport.serviceCall(Transport.java:196) > at > sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:562) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:796) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:677) > at java.security.AccessController.doPrivileged(Native Method) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:676) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) > ... 1 more > Caused by: org.apache.geode.GemFireConfigException: Problem configuring > membership services > at > org.apache.geode.distributed.internal.DistributionImpl.start(DistributionImpl.java:184) > at > org.apache.geode.distributed.internal.DistributionImpl.createDistribution(DistributionImpl.java:222) > at > org.apache.geode.distributed.internal.ClusterDistributionManager.<init>(ClusterDistributionManager.java:488) > at > org.apache.geode.distributed.internal.ClusterDistributionManager.<init>(ClusterDistributionManager.java:521) > at > org.apache.geode.distributed.internal.ClusterDistributionManager.create(ClusterDistributionManager.java:350) > at > org.apache.geode.distributed.internal.InternalDistributedSystem$DefaultClusterDistributionManagerConstructor.create(InternalDistributedSystem.java:2993) > at > org.apache.geode.distributed.internal.InternalDistributedSystem.initialize(InternalDistributedSystem.java:758) > at > org.apache.geode.distributed.internal.InternalDistributedSystem$Builder.build(InternalDistributedSystem.java:3048) > at > org.apache.geode.distributed.internal.InternalDistributedSystem.connectInternal(InternalDistributedSystem.java:283) > at > org.apache.geode.distributed.internal.InternalDistributedSystem.connectInternal(InternalDistributedSystem.java:209) > at > org.apache.geode.distributed.DistributedSystem.connect(DistributedSystem.java:161) > at > org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase.getSystem(JUnit4DistributedTestCase.java:182) > at > org.apache.geode.security.SecurityTestUtils.createSystem(SecurityTestUtils.java:1760) > at > org.apache.geode.security.SecurityTestUtils.createCacheServer(SecurityTestUtils.java:202) > ... 26 more > Caused by: > org.apache.geode.distributed.internal.membership.api.MembershipConfigurationException: > Unable to join the distributed system. Could not contact any of the > locators: [localhost:27917] > at > org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.join(GMSJoinLeave.java:415) > at > org.apache.geode.distributed.internal.membership.gms.GMSMembership.join(GMSMembership.java:536) > at > org.apache.geode.distributed.internal.membership.gms.GMSMembership.access$1200(GMSMembership.java:75) > at > org.apache.geode.distributed.internal.membership.gms.GMSMembership$ManagerImpl.joinDistributedSystem(GMSMembership.java:1754) > at > org.apache.geode.distributed.internal.membership.gms.Services.start(Services.java:247) > at > org.apache.geode.distributed.internal.membership.gms.GMSMembership.start(GMSMembership.java:1644) > at > org.apache.geode.distributed.internal.DistributionImpl.start(DistributionImpl.java:171) > ... 39 more > Caused by: java.net.SocketTimeoutException: Read timed out > at java.net.SocketInputStream.socketRead0(Native Method) > at java.net.SocketInputStream.socketRead(SocketInputStream.java:115) > at java.net.SocketInputStream.read(SocketInputStream.java:168) > at java.net.SocketInputStream.read(SocketInputStream.java:140) > at java.io.BufferedInputStream.fill(BufferedInputStream.java:252) > at java.io.BufferedInputStream.read(BufferedInputStream.java:271) > at java.io.FilterInputStream.read(FilterInputStream.java:83) > at java.io.FilterInputStream.read(FilterInputStream.java:83) > at > org.apache.geode.distributed.internal.tcpserver.TcpClient.assertNotSslAlert(TcpClient.java:298) > at > org.apache.geode.distributed.internal.tcpserver.TcpClient.getServerVersion(TcpClient.java:274) > at > org.apache.geode.distributed.internal.tcpserver.TcpClient.getServerVersion(TcpClient.java:265) > at > org.apache.geode.distributed.internal.tcpserver.TcpClient.lambda$getServerVersion$0(TcpClient.java:252) > at > org.apache.geode.internal.lang.utils.function.Checked.lambda$rethrowFunction$0(Checked.java:42) > at > java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1705) > at > org.apache.geode.internal.lang.utils.JavaWorkarounds.computeIfAbsent(JavaWorkarounds.java:34) > at > org.apache.geode.distributed.internal.tcpserver.TcpClient.getServerVersion(TcpClient.java:247) > at > org.apache.geode.distributed.internal.tcpserver.TcpClient.requestToServer(TcpClient.java:144) > at > org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.findCoordinator(GMSJoinLeave.java:1182) > at > org.apache.geode.distributed.internal.membership.gms.membership.GMSJoinLeave.join(GMSJoinLeave.java:343) > ... 45 more > {noformat} -- This message was sent by Atlassian Jira (v8.20.7#820007)