Donal Evans created GEODE-10329:
-----------------------------------
Summary: CI Failure: PersistentPartitionedRegionDistributedTest >
testCacheCloseDuringBucketMoveDoesntCauseDataLoss
Key: GEODE-10329
URL: https://issues.apache.org/jira/browse/GEODE-10329
Project: Geode
Issue Type: Bug
Affects Versions: 1.16.0
Reporter: Donal Evans
{code:java}
> Task :geode-core:distributedTest
PersistentPartitionedRegionDistributedTest >
testCacheCloseDuringBucketMoveDoesntCauseDataLoss FAILED
java.lang.AssertionError: Suspicious strings were written to the log during
this run.
Fix the strings or use IgnoredException.addIgnoredException to ignore.
-----------------------------------------------------------------------
Found suspect string in 'dunit_suspect-vm0.log' at line 662
[fatal 2022/05/23 17:31:45.980 UTC <Geode Failure Detection thread 4>
tid=257] Uncaught exception in thread Thread[Geode Failure Detection thread
4,5,RMI Runtime]
java.util.concurrent.RejectedExecutionException: Task
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$$Lambda$604/0x000000080119f4f8@2f733640
rejected from java.util.concurrent.ThreadPoolExecutor@2aaf4890[Shutting down,
pool size = 6, active threads = 5, queued tasks = 0, completed tasks = 7]
at
java.base/java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065)
at
java.base/java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833)
at
java.base/java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365)
at
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.checkIfAvailable(GMSHealthMonitor.java:1241)
at
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.processMessage(GMSHealthMonitor.java:1173)
at
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.sendSuspectRequest(GMSHealthMonitor.java:1425)
at
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.initiateSuspicion(GMSHealthMonitor.java:486)
at
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.lambda$checkMember$1(GMSHealthMonitor.java:470)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
at org.junit.Assert.fail(Assert.java:89)
at
org.apache.geode.test.dunit.internal.DUnitLauncher.closeAndCheckForSuspects(DUnitLauncher.java:422)
at
org.apache.geode.test.dunit.internal.DUnitLauncher.closeAndCheckForSuspects(DUnitLauncher.java:438)
at
org.apache.geode.test.dunit.rules.DistributedRule$TearDown.doTearDown(DistributedRule.java:230)
at
org.apache.geode.test.dunit.rules.DistributedRule$TearDown.access$100(DistributedRule.java:211)
at
org.apache.geode.test.dunit.rules.DistributedRule.after(DistributedRule.java:151)
at
org.apache.geode.test.dunit.rules.AbstractDistributedRule.afterDistributedTest(AbstractDistributedRule.java:81)
at
org.apache.geode.test.dunit.rules.AbstractDistributedRule$1.evaluate(AbstractDistributedRule.java:61)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at
org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at junitparams.JUnitParamsRunner.runChild(JUnitParamsRunner.java:449)
at junitparams.JUnitParamsRunner.runChild(JUnitParamsRunner.java:393)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at
org.junit.vintage.engine.execution.RunnerExecutor.execute(RunnerExecutor.java:42)
at
org.junit.vintage.engine.VintageTestEngine.executeAllChildren(VintageTestEngine.java:80)
at
org.junit.vintage.engine.VintageTestEngine.execute(VintageTestEngine.java:72)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:108)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52)
at
org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:96)
at
org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:75)
at
org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.processAllTestClasses(JUnitPlatformTestClassProcessor.java:99)
at
org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.access$000(JUnitPlatformTestClassProcessor.java:79)
at
org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor.stop(JUnitPlatformTestClassProcessor.java:75)
at
org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.stop(SuiteTestClassProcessor.java:61)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:568)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33)
at
org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94)
at jdk.proxy1.$Proxy2.stop(Unknown Source)
at
org.gradle.api.internal.tasks.testing.worker.TestWorker.stop(TestWorker.java:133)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:568)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
at
org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at
org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:182)
at
org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:164)
at
org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:414)
at
org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64)
at
org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:48)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at
org.gradle.internal.concurrent.ThreadFactoryImpl$ManagedThreadRunnable.run(ThreadFactoryImpl.java:56)
at java.lang.Thread.run(Thread.java:833)
8374 tests completed, 1 failed, 413 skipped
{code}
={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}= Test Results URI
={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}=
[http://files.apachegeode-ci.info/builds/apache-develop-main/1.16.0-build.0242/test-results/distributedTest/1653332261/]
={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}=
Test report artifacts from this job are available at:
[http://files.apachegeode-ci.info/builds/apache-develop-main/1.16.0-build.0242/test-artifacts/1653332261/distributedtestfiles-openjdk17-1.16.0-build.0242.tgz]
The exception was thrown as the member was shutting down:
{code:java}
[vm0] [info 2022/05/23 17:31:45.550 UTC <Pooled High Priority Message
Processor 6> tid=0xeb] Now closing distribution for
heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(287702)<v17>:51001
[vm0] [info 2022/05/23 17:31:45.600 UTC <Pooled High Priority Message
Processor 6> tid=0xeb] Stopping membership services
[vm0] [info 2022/05/23 17:31:45.500 UTC <Geode Failure Detection thread 2>
tid=0xfc] received suspect message from myself for
heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(287718)<v20>:51002: Member
isn't responding to heartbeat requests
[vm1] [info 2022/05/23 17:31:45.767 UTC <unicast
receiver,heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10-31261> tid=0x139]
received leave request from 10.0.2.97<v17>:51001 for
heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(287702)<v17>:51001
[vm1] [info 2022/05/23 17:31:45.771 UTC <unicast
receiver,heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10-31261> tid=0x139]
JoinLeave.processLeaveRequestMessage(LeaveRequestMessage) invoked.
isCoordinator=false; isStopping=false; cancelInProgress=false
[vm1] [info 2022/05/23 17:31:45.772 UTC <unicast
receiver,heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10-31261> tid=0x139]
ignoring leave request from old member
[vm0] [info 2022/05/23 17:31:45.780 UTC <Geode Failure Detection thread 5>
tid=0x102] No longer suspecting
heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(285325:locator)<ec><v0>:51000
[vm0] [info 2022/05/23 17:31:45.742 UTC <Geode Failure Detection thread 3>
tid=0xff] Performing availability check for suspect member
heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(285325:locator)<ec><v0>:51000
reason=Member isn't responding to heartbeat requests
[vm0] [info 2022/05/23 17:31:45.868 UTC <Geode Failure Detection thread 5>
tid=0x102] All other members are suspect at this point
[vm0] [info 2022/05/23 17:31:45.873 UTC <Geode Failure Detection Server thread
1> tid=0xc6] GMSHealthMonitor server thread exiting
[vm0] [info 2022/05/23 17:31:45.941 UTC <Geode Failure Detection thread 7>
tid=0x103] Performing availability check for suspect member
heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(287718)<v20>:51002
reason=Member isn't responding to heartbeat requests
[vm0] [info 2022/05/23 17:31:46.003 UTC <Geode Failure Detection thread 7>
tid=0x103] All other members are suspect at this point
[vm0] [info 2022/05/23 17:31:46.009 UTC <Geode Failure Detection thread 7>
tid=0x103] JGroupsMessenger channel is closed - messaging is not possible
[vm0] [info 2022/05/23 17:31:46.069 UTC <Geode Failure Detection thread 7>
tid=0x103] No longer suspecting
heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(287718)<v20>:51002
[vm0] [info 2022/05/23 17:31:46.069 UTC <Geode Failure Detection thread 3>
tid=0xff] JGroupsMessenger channel is closed - messaging is not possible
[vm0] [info 2022/05/23 17:31:46.096 UTC <Geode Failure Detection thread 3>
tid=0xff] No longer suspecting
heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(285325:locator)<ec><v0>:51000
[locator] [info 2022/05/23 17:31:45.774 UTC <unicast
receiver,heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10-56595> tid=0x34]
received suspect message from
heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10<v17>:51001 for
heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(285325:locator)<ec><v0>:51000:
Member isn't responding to heartbeat requests
[vm0] [info 2022/05/23 17:31:46.416 UTC <Pooled High Priority Message
Processor 6> tid=0xeb] DistributionManager stopped in 26323ms.
[vm0] [info 2022/05/23 17:31:46.416 UTC <Pooled High Priority Message
Processor 6> tid=0xeb] Marking DistributionManager
heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(287702)<v17>:51001 as closed.
[locator] [info 2022/05/23 17:31:46.420 UTC <unicast
receiver,heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10-56595> tid=0x34]
Membership ignoring suspect request for SuspectMembersMessage
[suspectRequests=[SuspectRequest
[member=heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(285325:locator)<ec><v0>:51000,
reason=Member isn't responding to heartbeat requests]]] from non-member
heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10<v17>:51001
[vm0] [fatal 2022/05/23 17:31:45.980 UTC <Geode Failure Detection thread 4>
tid=0x101] Uncaught exception in thread Thread[Geode Failure Detection thread
4,5,RMI Runtime]
[vm0] java.util.concurrent.RejectedExecutionException: Task
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$$Lambda$604/0x000000080119f4f8@2f733640
rejected from java.util.concurrent.ThreadPoolExecutor@2aaf4890[Shutting down,
pool size = 6, active threads = 5, queued tasks = 0, completed tasks = 7]
[vm0] at
java.base/java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065)
[vm0] at
java.base/java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833)
[vm0] at
java.base/java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365)
[vm0] at
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.checkIfAvailable(GMSHealthMonitor.java:1241)
[vm0] at
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.processMessage(GMSHealthMonitor.java:1173)
[vm0] at
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.sendSuspectRequest(GMSHealthMonitor.java:1425)
[vm0] at
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.initiateSuspicion(GMSHealthMonitor.java:486)
[vm0] at
org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.lambda$checkMember$1(GMSHealthMonitor.java:470)
[vm0] at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
[vm0] at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
[vm0] at java.base/java.lang.Thread.run(Thread.java:833) {code}
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
