Leon Finker created GEODE-10453:
-----------------------------------
Summary: Infinite/slow indexing on reconnect and register interest
replay
Key: GEODE-10453
URL: https://issues.apache.org/jira/browse/GEODE-10453
Project: Geode
Issue Type: Bug
Affects Versions: 1.15.1
Reporter: Leon Finker
Cache server was restarted. Client side upon reconnect went into infinite/slow
indexing loop. This has not recovered even after multiple days. The thread
stack for thread taking 100% CPU was:
{code}
Thread Name <poolTimer-Server-21659> state <BLOCKED>
Waiting on <org.apache.geode.cache.client.internal.ConnectionImpl@293d172>
Owned By <queueTimer-Server1> with ID <140>
Executor Group <ScheduledThreadPoolExecutorWithKeepAlive>
Monitored metric <ResourceManagerStats.numThreadsStuck>
Thread stack for "poolTimer-Server-21659" (0x128275):
java.lang.ThreadState: BLOCKED
at
app//org.apache.geode.cache.client.internal.ConnectionImpl.execute(ConnectionImpl.java:283)
at
app//org.apache.geode.cache.client.internal.QueueConnectionImpl.execute(QueueConnectionImpl.java:191)
at
app//org.apache.geode.cache.client.internal.OpExecutorImpl.executeWithPossibleReAuthentication(OpExecutorImpl.java:760)
at
app//org.apache.geode.cache.client.internal.OpExecutorImpl.executeOnServer(OpExecutorImpl.java:343)
at
app//org.apache.geode.cache.client.internal.OpExecutorImpl.executeOn(OpExecutorImpl.java:312)
at
app//org.apache.geode.cache.client.internal.PoolImpl.executeOn(PoolImpl.java:848)
at app//org.apache.geode.cache.client.internal.PingOp.execute(PingOp.java:40)
at
app//org.apache.geode.cache.client.internal.LiveServerPinger$PingTask.run2(LiveServerPinger.java:128)
at
app//org.apache.geode.cache.client.internal.PoolImpl$PoolTask.run(PoolImpl.java:1340)
at
java.base@17.0.6/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at
java.base@17.0.6/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
at
app//org.apache.geode.internal.ScheduledThreadPoolExecutorWithKeepAlive$DelegatingScheduledFuture.run(ScheduledThreadPoolExecutorWithKeepAlive.java:285)
at
java.base@17.0.6/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base@17.0.6/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base@17.0.6/java.lang.Thread.run(Thread.java:833)
Locked ownable synchronizers:
- None
Lock owner thread stack for "queueTimer-Server1" (0x6a):
java.lang.ThreadState: RUNNABLE
at
app//org.apache.geode.cache.query.internal.types.TypeUtils$ComparisonStrategy$4.execute(TypeUtils.java:90)
at
app//org.apache.geode.cache.query.internal.types.TypeUtils.compare(TypeUtils.java:499)
at
app//org.apache.geode.cache.query.internal.index.MemoryIndexStore.getOldKey(MemoryIndexStore.java:275)
at
app//org.apache.geode.cache.query.internal.index.MemoryIndexStore.updateMapping(MemoryIndexStore.java:122)
at
app//org.apache.geode.cache.query.internal.index.CompactRangeIndex$IMQEvaluator.applyProjection(CompactRangeIndex.java:1563)
at
app//org.apache.geode.cache.query.internal.index.CompactRangeIndex$IMQEvaluator.doNestedIterations(CompactRangeIndex.java:1519)
at
app//org.apache.geode.cache.query.internal.index.CompactRangeIndex$IMQEvaluator.evaluate(CompactRangeIndex.java:1372)
at
app//org.apache.geode.cache.query.internal.index.CompactRangeIndex.addMapping(CompactRangeIndex.java:143)
at
app//org.apache.geode.cache.query.internal.index.AbstractIndex.addIndexMapping(AbstractIndex.java:488)
at
app//org.apache.geode.cache.query.internal.index.IndexManager.addIndexMapping(IndexManager.java:1143)
at
app//org.apache.geode.cache.query.internal.index.IndexManager.processAction(IndexManager.java:1089)
at
app//org.apache.geode.cache.query.internal.index.IndexManager.updateIndexes(IndexManager.java:982)
at
app//org.apache.geode.cache.query.internal.index.IndexManager.updateIndexes(IndexManager.java:956)
at
app//org.apache.geode.internal.cache.AbstractRegionMap.initialImagePut(AbstractRegionMap.java:839)
at
app//org.apache.geode.internal.cache.LocalRegion.refreshEntriesFromServerKeys(LocalRegion.java:4348)
at
app//org.apache.geode.cache.client.internal.RegisterInterestOp$RegisterInterestOpImpl.processResponse(RegisterInterestOp.java:217)
at
app//org.apache.geode.cache.client.internal.RegisterInterestOp$RegisterInterestOpImpl.processResponse(RegisterInterestOp.java:121)
at
app//org.apache.geode.cache.client.internal.AbstractOp.attemptReadResponse(AbstractOp.java:209)
at
app//org.apache.geode.cache.client.internal.AbstractOp.attempt(AbstractOp.java:394)
at
app//org.apache.geode.cache.client.internal.ConnectionImpl.execute(ConnectionImpl.java:284)
at
app//org.apache.geode.cache.client.internal.QueueConnectionImpl.execute(QueueConnectionImpl.java:191)
at
app//org.apache.geode.cache.client.internal.OpExecutorImpl.executeWithPossibleReAuthentication(OpExecutorImpl.java:760)
at
app//org.apache.geode.cache.client.internal.OpExecutorImpl.executeOn(OpExecutorImpl.java:475)
at
app//org.apache.geode.cache.client.internal.OpExecutorImpl.executeOn(OpExecutorImpl.java:488)
at
app//org.apache.geode.cache.client.internal.PoolImpl.executeOn(PoolImpl.java:861)
at
app//org.apache.geode.cache.client.internal.RegisterInterestOp.executeOn(RegisterInterestOp.java:113)
at
app//org.apache.geode.cache.client.internal.ServerRegionProxy.registerInterestOn(ServerRegionProxy.java:506)
at
app//org.apache.geode.cache.client.internal.QueueManagerImpl.recoverSingleKey(QueueManagerImpl.java:1236)
at
app//org.apache.geode.cache.client.internal.QueueManagerImpl.recoverSingleRegion(QueueManagerImpl.java:1183)
at
app//org.apache.geode.cache.client.internal.QueueManagerImpl.recoverSingleList(QueueManagerImpl.java:1129)
at
app//org.apache.geode.cache.client.internal.QueueManagerImpl.recoverInterestList(QueueManagerImpl.java:1250)
at
app//org.apache.geode.cache.client.internal.QueueManagerImpl.recoverAllInterestTypes(QueueManagerImpl.java:1264)
at
app//org.apache.geode.cache.client.internal.QueueManagerImpl.recoverInterest(QueueManagerImpl.java:1094)
at
app//org.apache.geode.cache.client.internal.QueueManagerImpl.recoverPrimary(QueueManagerImpl.java:938)
at
app//org.apache.geode.cache.client.internal.QueueManagerImpl$RedundancySatisfierTask.run2(QueueManagerImpl.java:1475)
at
app//org.apache.geode.cache.client.internal.PoolImpl$PoolTask.run(PoolImpl.java:1340)
at
java.base@17.0.6/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at java.base@17.0.6/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base@17.0.6/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at
app//org.apache.geode.cache.query.internal.index.CompactRangeIndex$IMQEvaluator.doNestedIterations(CompactRangeIndex.java:1509)
{code}
After client stop attempt and cache close, the following stack trace was logged:
{code}
The index is corrupted and
marked as invalid.
org.apache.geode.cache.CacheClosedException: The cache is closed.
at
org.apache.geode.internal.cache.GemFireCacheImpl$Stopper.generateCancelledException(GemFireCacheImpl.java:5207)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.CancelCriterion.checkCancelInProgress(CancelCriterion.java:83)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.internal.cache.LocalRegion.checkRegionDestroyed(LocalRegion.java:7382)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.internal.cache.LocalRegion.checkReadiness(LocalRegion.java:2788)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.internal.cache.LocalRegion.values(LocalRegion.java:1970)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.query.internal.QRegion.<init>(QRegion.java:81)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.query.internal.index.DummyQRegion.<init>(DummyQRegion.java:52)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.query.internal.index.CompactRangeIndex$IMQEvaluator.evaluate(CompactRangeIndex.java:1342)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.query.internal.index.CompactRangeIndex.addMapping(CompactRangeIndex.java:143)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.query.internal.index.AbstractIndex.addIndexMapping(AbstractIndex.java:488)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.query.internal.index.IndexManager.addIndexMapping(IndexManager.java:1143)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.query.internal.index.IndexManager.processAction(IndexManager.java:1089)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.query.internal.index.IndexManager.updateIndexes(IndexManager.java:982)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.query.internal.index.IndexManager.updateIndexes(IndexManager.java:956)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.internal.cache.AbstractRegionMap.initialImagePut(AbstractRegionMap.java:839)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.internal.cache.LocalRegion.refreshEntriesFromServerKeys(LocalRegion.java:4348)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.RegisterInterestOp$RegisterInterestOpImpl.processResponse(RegisterInterestOp.java:217)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.RegisterInterestOp$RegisterInterestOpImpl.processResponse(RegisterInterestOp.java:121)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.AbstractOp.attemptReadResponse(AbstractOp.java:209)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.AbstractOp.attempt(AbstractOp.java:394)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.ConnectionImpl.execute(ConnectionImpl.java:284)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.QueueConnectionImpl.execute(QueueConnectionImpl.java:191)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.OpExecutorImpl.executeWithPossibleReAuthentication(OpExecutorImpl.java:760)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.OpExecutorImpl.executeOn(OpExecutorImpl.java:475)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.OpExecutorImpl.executeOn(OpExecutorImpl.java:488)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.PoolImpl.executeOn(PoolImpl.java:861)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.RegisterInterestOp.executeOn(RegisterInterestOp.java:113)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.ServerRegionProxy.registerInterestOn(ServerRegionProxy.java:506)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.QueueManagerImpl.recoverSingleKey(QueueManagerImpl.java:1236)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.QueueManagerImpl.recoverSingleRegion(QueueManagerImpl.java:1183)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.QueueManagerImpl.recoverSingleList(QueueManagerImpl.java:1129)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.QueueManagerImpl.recoverInterestList(QueueManagerImpl.java:1250)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.QueueManagerImpl.recoverAllInterestTypes(QueueManagerImpl.java:1264)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.QueueManagerImpl.recoverInterest(QueueManagerImpl.java:1094)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.QueueManagerImpl.recoverPrimary(QueueManagerImpl.java:938)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.QueueManagerImpl$RedundancySatisfierTask.run2(QueueManagerImpl.java:1475)
~[geode-core-1.15.1.jar:?]
at
org.apache.geode.cache.client.internal.PoolImpl$PoolTask.run(PoolImpl.java:1340)
~[geode-core-1.15.1.jar:?]
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
~[?:?]
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
~[?:?]
{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
