IGNITE-1265 - Limit the node log output in a case of partition exchange timeout.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/8ced2073 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/8ced2073 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/8ced2073 Branch: refs/heads/master Commit: 8ced20733da65bbd224d7dea61ab09d8dcfa2efc Parents: d384d29 Author: Alexey Goncharuk <agoncha...@gridgain.com> Authored: Wed Aug 19 12:21:24 2015 -0700 Committer: Alexey Goncharuk <agoncha...@gridgain.com> Committed: Wed Aug 19 12:21:24 2015 -0700 ---------------------------------------------------------------------- .../apache/ignite/IgniteSystemProperties.java | 3 ++ .../GridDhtPartitionsExchangeFuture.java | 32 +++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8ced2073/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 7e96b29..7c808df 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -354,6 +354,9 @@ public final class IgniteSystemProperties { /** Number of cache operation retries in case of topology exceptions. */ public static final String IGNITE_CACHE_RETRIES_COUNT = "IGNITE_CACHE_RETRIES_COUNT"; + /** Number of times pending cache objects will be dumped to the log in case of partition exchange timeout. */ + public static final String IGNITE_DUMP_PENDING_OBJECTS_THRESHOLD = "IGNITE_DUMP_PENDING_OBJECTS_THRESHOLD"; + /** * Enforces singleton. */ http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8ced2073/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java index 5701749..4971ca6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java @@ -54,6 +54,10 @@ import static org.apache.ignite.internal.managers.communication.GridIoPolicy.*; public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityTopologyVersion> implements Comparable<GridDhtPartitionsExchangeFuture>, GridDhtTopologyFuture { /** */ + private static final int DUMP_PENDING_OBJECTS_THRESHOLD = + IgniteSystemProperties.getInteger(IgniteSystemProperties.IGNITE_DUMP_PENDING_OBJECTS_THRESHOLD, 10); + + /** */ private static final long serialVersionUID = 0L; /** Dummy flag. */ @@ -722,6 +726,8 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT if (log.isDebugEnabled()) log.debug("Before waiting for partition release future: " + this); + int dumpedObjects = 0; + while (true) { try { partReleaseFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS); @@ -730,7 +736,11 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT } catch (IgniteFutureTimeoutCheckedException ignored) { // Print pending transactions and locks that might have led to hang. - dumpPendingObjects(); + if (dumpedObjects < DUMP_PENDING_OBJECTS_THRESHOLD) { + dumpPendingObjects(); + + dumpedObjects++; + } } } @@ -742,6 +752,8 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT IgniteInternalFuture<?> locksFut = cctx.mvcc().finishLocks(exchId.topologyVersion()); + dumpedObjects = 0; + while (true) { try { locksFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS); @@ -749,16 +761,20 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT break; } catch (IgniteFutureTimeoutCheckedException ignored) { - U.warn(log, "Failed to wait for locks release future. " + - "Dumping pending objects that might be the cause: " + cctx.localNodeId()); + if (dumpedObjects < DUMP_PENDING_OBJECTS_THRESHOLD) { + U.warn(log, "Failed to wait for locks release future. " + + "Dumping pending objects that might be the cause: " + cctx.localNodeId()); - U.warn(log, "Locked entries:"); + U.warn(log, "Locked entries:"); - Map<IgniteTxKey, Collection<GridCacheMvccCandidate>> locks = - cctx.mvcc().unfinishedLocks(exchId.topologyVersion()); + Map<IgniteTxKey, Collection<GridCacheMvccCandidate>> locks = + cctx.mvcc().unfinishedLocks(exchId.topologyVersion()); - for (Map.Entry<IgniteTxKey, Collection<GridCacheMvccCandidate>> e : locks.entrySet()) - U.warn(log, "Locked entry [key=" + e.getKey() + ", mvcc=" + e.getValue() + ']'); + for (Map.Entry<IgniteTxKey, Collection<GridCacheMvccCandidate>> e : locks.entrySet()) + U.warn(log, "Locked entry [key=" + e.getKey() + ", mvcc=" + e.getValue() + ']'); + + dumpedObjects++; + } } }
