[ https://issues.apache.org/jira/browse/GEODE-1155?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15813037#comment-15813037 ]
Bruce Schuchardt commented on GEODE-1155: ----------------------------------------- It looks to me like this was being caused by a lingering DistributionMessageObserver. This test class and a number of others install message observers that disconnect the DistributedSystem if they see a certain class of message or some other condition. The logging from these observers looks exactly like what is seen in the logs that I posted in my last comment. I think that this was fixed in Kirk's refactoring of tearDown, possibly aided by some work that Jason did in November to add a try/catch that ensures that cleanUpAllVMs is invoked. > CI failure: NetSearchMessagingDUnitTest.testNetSearchNormals > ------------------------------------------------------------ > > Key: GEODE-1155 > URL: https://issues.apache.org/jira/browse/GEODE-1155 > Project: Geode > Issue Type: Bug > Reporter: Jianxia Chen > Labels: CI, Flaky > > https://brazil.gemstone.com:8080/job/Geode_develop_DistributedTests/2087/testReport/junit/com.gemstone.gemfire.internal.cache/NetSearchMessagingDUnitTest/testNetSearchNormals/ > com.gemstone.gemfire.test.dunit.RMIException: While invoking > com.gemstone.gemfire.internal.cache.NetSearchMessagingDUnitTest$12.run in VM > 0 running on Host cc2-ub.gemstone.com with 4 VMs > at com.gemstone.gemfire.test.dunit.VM.invoke(VM.java:439) > at com.gemstone.gemfire.test.dunit.VM.invoke(VM.java:381) > at com.gemstone.gemfire.test.dunit.VM.invoke(VM.java:317) > at > com.gemstone.gemfire.internal.cache.NetSearchMessagingDUnitTest.createNormal(NetSearchMessagingDUnitTest.java:398) > at > com.gemstone.gemfire.internal.cache.NetSearchMessagingDUnitTest.testNetSearchNormals(NetSearchMessagingDUnitTest.java:122) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at junit.framework.TestCase.runTest(TestCase.java:176) > at junit.framework.TestCase.runBare(TestCase.java:141) > at junit.framework.TestResult$1.protect(TestResult.java:122) > at junit.framework.TestResult.runProtected(TestResult.java:142) > at junit.framework.TestResult.run(TestResult.java:125) > at junit.framework.TestCase.run(TestCase.java:129) > at junit.framework.TestSuite.runTest(TestSuite.java:252) > at junit.framework.TestSuite.run(TestSuite.java:247) > at > org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:86) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.runTestClass(JUnitTestClassExecuter.java:105) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.execute(JUnitTestClassExecuter.java:56) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassProcessor.processTestClass(JUnitTestClassProcessor.java:64) > at > org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:50) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.messaging.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32) > at > org.gradle.messaging.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93) > at com.sun.proxy.$Proxy2.processTestClass(Unknown Source) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:106) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.messaging.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.messaging.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:360) > at > org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:54) > at > org.gradle.internal.concurrent.StoppableExecutorImpl$1.run(StoppableExecutorImpl.java:40) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Caused by: > com.gemstone.gemfire.distributed.DistributedSystemDisconnectedException: This > connection to a distributed system has been disconnected. > at > com.gemstone.gemfire.distributed.internal.InternalDistributedSystem.checkConnected(InternalDistributedSystem.java:827) > at > com.gemstone.gemfire.distributed.internal.InternalDistributedSystem.getDistributionManager(InternalDistributedSystem.java:1421) > at > com.gemstone.gemfire.pdx.internal.PeerTypeRegistration.<init>(PeerTypeRegistration.java:116) > at > com.gemstone.gemfire.pdx.internal.TypeRegistry.<init>(TypeRegistry.java:67) > at > com.gemstone.gemfire.internal.cache.GemFireCacheImpl.initializePdxRegistry(GemFireCacheImpl.java:1412) > at > com.gemstone.gemfire.internal.cache.GemFireCacheImpl.initializeDeclarativeCache(GemFireCacheImpl.java:1348) > at > com.gemstone.gemfire.internal.cache.GemFireCacheImpl.initialize(GemFireCacheImpl.java:1176) > at > com.gemstone.gemfire.internal.cache.GemFireCacheImpl.basicCreate(GemFireCacheImpl.java:796) > at > com.gemstone.gemfire.internal.cache.GemFireCacheImpl.create(GemFireCacheImpl.java:784) > at com.gemstone.gemfire.cache.CacheFactory.create(CacheFactory.java:178) > at com.gemstone.gemfire.cache.CacheFactory.create(CacheFactory.java:169) > at > com.gemstone.gemfire.test.dunit.cache.internal.JUnit4CacheTestCase.createCache(JUnit4CacheTestCase.java:114) > at > com.gemstone.gemfire.test.dunit.cache.internal.JUnit4CacheTestCase.getCache(JUnit4CacheTestCase.java:255) > at > com.gemstone.gemfire.test.dunit.cache.internal.JUnit4CacheTestCase.getCache(JUnit4CacheTestCase.java:234) > at > com.gemstone.gemfire.test.dunit.cache.internal.JUnit4CacheTestCase.getCache(JUnit4CacheTestCase.java:226) > at > com.gemstone.gemfire.test.dunit.cache.internal.JUnit3CacheTestCase.getCache(JUnit3CacheTestCase.java:84) > at > com.gemstone.gemfire.internal.cache.NetSearchMessagingDUnitTest$12.run(NetSearchMessagingDUnitTest.java:401) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at hydra.MethExecutor.executeObject(MethExecutor.java:268) > at > com.gemstone.gemfire.test.dunit.standalone.RemoteDUnitVM.executeMethodOnObject(RemoteDUnitVM.java:83) > at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323) > at sun.rmi.transport.Transport$1.run(Transport.java:200) > at sun.rmi.transport.Transport$1.run(Transport.java:197) > at java.security.AccessController.doPrivileged(Native Method) > at sun.rmi.transport.Transport.serviceCall(Transport.java:196) > at > sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683) > at java.security.AccessController.doPrivileged(Native Method) > at > sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682) > ... 3 more -- This message was sent by Atlassian JIRA (v6.3.4#6332)