[
https://issues.apache.org/jira/browse/GEODE-10329?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Anthony Baker updated GEODE-10329:
----------------------------------
Labels: pull-request-available (was: needsTriage pull-request-available)
> CI Failure: PersistentPartitionedRegionDistributedTest >
> testCacheCloseDuringBucketMoveDoesntCauseDataLoss fails due to
> RejectedExecutionException during member availability check
> -----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
>
> Key: GEODE-10329
> URL: https://issues.apache.org/jira/browse/GEODE-10329
> Project: Geode
> Issue Type: Bug
> Affects Versions: 1.16.0
> Reporter: Donal Evans
> Assignee: Donal Evans
> Priority: Major
> Labels: pull-request-available
>
> {code:java}
> > Task :geode-core:distributedTest
> PersistentPartitionedRegionDistributedTest >
> testCacheCloseDuringBucketMoveDoesntCauseDataLoss FAILED
> java.lang.AssertionError: Suspicious strings were written to the log
> during this run.
> Fix the strings or use IgnoredException.addIgnoredException to ignore.
> -----------------------------------------------------------------------
> Found suspect string in 'dunit_suspect-vm0.log' at line 662
> [fatal 2022/05/23 17:31:45.980 UTC <Geode Failure Detection thread 4>
> tid=257] Uncaught exception in thread Thread[Geode Failure Detection thread
> 4,5,RMI Runtime]
> java.util.concurrent.RejectedExecutionException: Task
> org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$$Lambda$604/0x000000080119f4f8@2f733640
> rejected from java.util.concurrent.ThreadPoolExecutor@2aaf4890[Shutting
> down, pool size = 6, active threads = 5, queued tasks = 0, completed tasks =
> 7]
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365)
> at
> org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.checkIfAvailable(GMSHealthMonitor.java:1241)
> at
> org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.processMessage(GMSHealthMonitor.java:1173)
> at
> org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.sendSuspectRequest(GMSHealthMonitor.java:1425)
> at
> org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.initiateSuspicion(GMSHealthMonitor.java:486)
> at
> org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.lambda$checkMember$1(GMSHealthMonitor.java:470)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> at java.base/java.lang.Thread.run(Thread.java:833)
> at org.junit.Assert.fail(Assert.java:89)
> at
> org.apache.geode.test.dunit.internal.DUnitLauncher.closeAndCheckForSuspects(DUnitLauncher.java:422)
> at
> org.apache.geode.test.dunit.internal.DUnitLauncher.closeAndCheckForSuspects(DUnitLauncher.java:438)
> at
> org.apache.geode.test.dunit.rules.DistributedRule$TearDown.doTearDown(DistributedRule.java:230)
> at
> org.apache.geode.test.dunit.rules.DistributedRule$TearDown.access$100(DistributedRule.java:211)
> at
> org.apache.geode.test.dunit.rules.DistributedRule.after(DistributedRule.java:151)
> at
> org.apache.geode.test.dunit.rules.AbstractDistributedRule.afterDistributedTest(AbstractDistributedRule.java:81)
> at
> org.apache.geode.test.dunit.rules.AbstractDistributedRule$1.evaluate(AbstractDistributedRule.java:61)
> at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
> at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
> at
> org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
> at junitparams.JUnitParamsRunner.runChild(JUnitParamsRunner.java:449)
> at junitparams.JUnitParamsRunner.runChild(JUnitParamsRunner.java:393)
> at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
> at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
> at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
> at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
> at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
> at
> org.junit.vintage.engine.execution.RunnerExecutor.execute(RunnerExecutor.java:42)
> at
> org.junit.vintage.engine.VintageTestEngine.executeAllChildren(VintageTestEngine.java:80)
> at
> org.junit.vintage.engine.VintageTestEngine.execute(VintageTestEngine.java:72)
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:108)
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88)
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54)
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67)
> at
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52)
> at
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:96)
> at
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:75)
> at
> org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.processAllTestClasses(JUnitPlatformTestClassProcessor.java:99)
> at
> org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor$CollectAllTestClassesExecutor.access$000(JUnitPlatformTestClassProcessor.java:79)
> at
> org.gradle.api.internal.tasks.testing.junitplatform.JUnitPlatformTestClassProcessor.stop(JUnitPlatformTestClassProcessor.java:75)
> at
> org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.stop(SuiteTestClassProcessor.java:61)
> at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> at
> jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
> at
> jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:568)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
> at
> org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33)
> at
> org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94)
> at jdk.proxy1.$Proxy2.stop(Unknown Source)
> at
> org.gradle.api.internal.tasks.testing.worker.TestWorker.stop(TestWorker.java:133)
> at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> at
> jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
> at
> jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:568)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
> at
> org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
> at
> org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:182)
> at
> org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:164)
> at
> org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:414)
> at
> org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64)
> at
> org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:48)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> at
> org.gradle.internal.concurrent.ThreadFactoryImpl$ManagedThreadRunnable.run(ThreadFactoryImpl.java:56)
> at java.lang.Thread.run(Thread.java:833)
> 8374 tests completed, 1 failed, 413 skipped
> {code}
>
> ={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}= Test Results URI
> ={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}=
> [http://files.apachegeode-ci.info/builds/apache-develop-main/1.16.0-build.0242/test-results/distributedTest/1653332261/]
> ={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}={-}=
> Test report artifacts from this job are available at:
> [http://files.apachegeode-ci.info/builds/apache-develop-main/1.16.0-build.0242/test-artifacts/1653332261/distributedtestfiles-openjdk17-1.16.0-build.0242.tgz]
>
> The exception was thrown as the member was shutting down:
> {code:java}
> [vm0] [info 2022/05/23 17:31:45.550 UTC <Pooled High Priority Message
> Processor 6> tid=0xeb] Now closing distribution for
> heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(287702)<v17>:51001
> [vm0] [info 2022/05/23 17:31:45.600 UTC <Pooled High Priority Message
> Processor 6> tid=0xeb] Stopping membership services
> [vm0] [info 2022/05/23 17:31:45.500 UTC <Geode Failure Detection thread 2>
> tid=0xfc] received suspect message from myself for
> heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(287718)<v20>:51002: Member
> isn't responding to heartbeat requests
> [vm1] [info 2022/05/23 17:31:45.767 UTC <unicast
> receiver,heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10-31261> tid=0x139]
> received leave request from 10.0.2.97<v17>:51001 for
> heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(287702)<v17>:51001
> [vm1] [info 2022/05/23 17:31:45.771 UTC <unicast
> receiver,heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10-31261> tid=0x139]
> JoinLeave.processLeaveRequestMessage(LeaveRequestMessage) invoked.
> isCoordinator=false; isStopping=false; cancelInProgress=false
> [vm1] [info 2022/05/23 17:31:45.772 UTC <unicast
> receiver,heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10-31261> tid=0x139]
> ignoring leave request from old member
> [vm0] [info 2022/05/23 17:31:45.780 UTC <Geode Failure Detection thread 5>
> tid=0x102] No longer suspecting
> heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(285325:locator)<ec><v0>:51000
> [vm0] [info 2022/05/23 17:31:45.742 UTC <Geode Failure Detection thread 3>
> tid=0xff] Performing availability check for suspect member
> heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(285325:locator)<ec><v0>:51000
> reason=Member isn't responding to heartbeat requests
> [vm0] [info 2022/05/23 17:31:45.868 UTC <Geode Failure Detection thread 5>
> tid=0x102] All other members are suspect at this point
> [vm0] [info 2022/05/23 17:31:45.873 UTC <Geode Failure Detection Server
> thread 1> tid=0xc6] GMSHealthMonitor server thread exiting
> [vm0] [info 2022/05/23 17:31:45.941 UTC <Geode Failure Detection thread 7>
> tid=0x103] Performing availability check for suspect member
> heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(287718)<v20>:51002
> reason=Member isn't responding to heartbeat requests
> [vm0] [info 2022/05/23 17:31:46.003 UTC <Geode Failure Detection thread 7>
> tid=0x103] All other members are suspect at this point
> [vm0] [info 2022/05/23 17:31:46.009 UTC <Geode Failure Detection thread 7>
> tid=0x103] JGroupsMessenger channel is closed - messaging is not possible
> [vm0] [info 2022/05/23 17:31:46.069 UTC <Geode Failure Detection thread 7>
> tid=0x103] No longer suspecting
> heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(287718)<v20>:51002
> [vm0] [info 2022/05/23 17:31:46.069 UTC <Geode Failure Detection thread 3>
> tid=0xff] JGroupsMessenger channel is closed - messaging is not possible
> [vm0] [info 2022/05/23 17:31:46.096 UTC <Geode Failure Detection thread 3>
> tid=0xff] No longer suspecting
> heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(285325:locator)<ec><v0>:51000
> [locator] [info 2022/05/23 17:31:45.774 UTC <unicast
> receiver,heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10-56595> tid=0x34]
> received suspect message from
> heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10<v17>:51001 for
> heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(285325:locator)<ec><v0>:51000:
> Member isn't responding to heartbeat requests
> [vm0] [info 2022/05/23 17:31:46.416 UTC <Pooled High Priority Message
> Processor 6> tid=0xeb] DistributionManager stopped in 26323ms.
> [vm0] [info 2022/05/23 17:31:46.416 UTC <Pooled High Priority Message
> Processor 6> tid=0xeb] Marking DistributionManager
> heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(287702)<v17>:51001 as
> closed.
> [locator] [info 2022/05/23 17:31:46.420 UTC <unicast
> receiver,heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10-56595> tid=0x34]
> Membership ignoring suspect request for SuspectMembersMessage
> [suspectRequests=[SuspectRequest
> [member=heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10(285325:locator)<ec><v0>:51000,
> reason=Member isn't responding to heartbeat requests]]] from non-member
> heavy-lifter-a90ffecf-c229-56fe-bfbd-e005a2422a10<v17>:51001
> [vm0] [fatal 2022/05/23 17:31:45.980 UTC <Geode Failure Detection thread 4>
> tid=0x101] Uncaught exception in thread Thread[Geode Failure Detection thread
> 4,5,RMI Runtime]
> [vm0] java.util.concurrent.RejectedExecutionException: Task
> org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor$$Lambda$604/0x000000080119f4f8@2f733640
> rejected from java.util.concurrent.ThreadPoolExecutor@2aaf4890[Shutting
> down, pool size = 6, active threads = 5, queued tasks = 0, completed tasks =
> 7]
> [vm0] at
> java.base/java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065)
> [vm0] at
> java.base/java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833)
> [vm0] at
> java.base/java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365)
> [vm0] at
> org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.checkIfAvailable(GMSHealthMonitor.java:1241)
> [vm0] at
> org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.processMessage(GMSHealthMonitor.java:1173)
> [vm0] at
> org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.sendSuspectRequest(GMSHealthMonitor.java:1425)
> [vm0] at
> org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.initiateSuspicion(GMSHealthMonitor.java:486)
> [vm0] at
> org.apache.geode.distributed.internal.membership.gms.fd.GMSHealthMonitor.lambda$checkMember$1(GMSHealthMonitor.java:470)
> [vm0] at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
> [vm0] at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
> [vm0] at java.base/java.lang.Thread.run(Thread.java:833) {code}
--
This message was sent by Atlassian Jira
(v8.20.7#820007)
