Repository: accumulo Updated Branches: refs/heads/ACCUMULO-4004 [created] f7d22c395
ACCUMULO-4004: Add new property for WALog max age, close log when age is reached. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f7d22c39 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f7d22c39 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f7d22c39 Branch: refs/heads/ACCUMULO-4004 Commit: f7d22c395a27ced9fa382b44162371d8b464988d Parents: 1d49030 Author: Dave Marion <dlmar...@apache.org> Authored: Tue Mar 29 16:02:38 2016 -0400 Committer: Dave Marion <dlmar...@apache.org> Committed: Tue Mar 29 16:02:38 2016 -0400 ---------------------------------------------------------------------- .../java/org/apache/accumulo/core/conf/Property.java | 1 + .../java/org/apache/accumulo/tserver/TabletServer.java | 3 ++- .../apache/accumulo/tserver/log/TabletServerLogger.java | 11 ++++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7d22c39/core/src/main/java/org/apache/accumulo/core/conf/Property.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index b8311a9..e8537ac 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -254,6 +254,7 @@ public enum Property { + "must be made, which is slower. However opening too many files at once can cause problems."), TSERV_WALOG_MAX_SIZE("tserver.walog.max.size", "1G", PropertyType.MEMORY, "The maximum size for each write-ahead log. See comment for property tserver.memory.maps.max"), + TSERV_WALOG_MAX_AGE("tserver.walog.max.age", "24h", PropertyType.TIMEDURATION, "The maximum age for each write-ahead log."), TSERV_WALOG_TOLERATED_CREATION_FAILURES("tserver.walog.tolerated.creation.failures", "50", PropertyType.COUNT, "The maximum number of failures tolerated when creating a new WAL file within the period specified by tserver.walog.failures.period." http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7d22c39/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index d19dfa9..be52d5b 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@ -354,6 +354,7 @@ public class TabletServer extends AccumuloServerContext implements Runnable { }, 5000, 5000); final long walogMaxSize = aconf.getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE); + final long walogMaxAge = aconf.getTimeInMillis(Property.TSERV_WALOG_MAX_AGE); final long minBlockSize = CachedConfiguration.getInstance().getLong("dfs.namenode.fs-limits.min-block-size", 0); if (minBlockSize != 0 && minBlockSize > walogMaxSize) throw new RuntimeException("Unable to start TabletServer. Logger is set to use blocksize " + walogMaxSize + " but hdfs minimum block size is " @@ -367,7 +368,7 @@ public class TabletServer extends AccumuloServerContext implements Runnable { final RetryFactory walCreationRetryFactory = new RetryFactory(toleratedWalCreationFailures, walCreationFailureRetryIncrement, walCreationFailureRetryIncrement, walCreationFailureRetryMax); - logger = new TabletServerLogger(this, walogMaxSize, syncCounter, flushCounter, walCreationRetryFactory); + logger = new TabletServerLogger(this, walogMaxSize, syncCounter, flushCounter, walCreationRetryFactory, walogMaxAge); this.resourceManager = new TabletServerResourceManager(this, fs); this.security = AuditedSecurityOperation.getInstance(this); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7d22c39/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java index b76b9cd..7b52857 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java @@ -74,6 +74,7 @@ public class TabletServerLogger { private final AtomicLong logSizeEstimate = new AtomicLong(); private final long maxSize; + private final long maxAge; private final TabletServer tserver; @@ -102,6 +103,8 @@ public class TabletServerLogger { private final RetryFactory retryFactory; private Retry retry = null; + private long createTime = 0; + static private abstract class TestCallWithWriteLock { abstract boolean test(); @@ -145,13 +148,14 @@ public class TabletServerLogger { } } - public TabletServerLogger(TabletServer tserver, long maxSize, AtomicLong syncCounter, AtomicLong flushCounter, RetryFactory retryFactory) { + public TabletServerLogger(TabletServer tserver, long maxSize, AtomicLong syncCounter, AtomicLong flushCounter, RetryFactory retryFactory, long maxAge) { this.tserver = tserver; this.maxSize = maxSize; this.syncCounter = syncCounter; this.flushCounter = flushCounter; this.retryFactory = retryFactory; this.retry = null; + this.maxAge = maxAge; } private DfsLogger initializeLoggers(final AtomicInteger logIdOut) throws IOException { @@ -224,6 +228,7 @@ public class TabletServerLogger { retry = null; } + this.createTime = System.currentTimeMillis(); return; } else { throw new RuntimeException("Error: unexpected type seen: " + next); @@ -434,12 +439,12 @@ public class TabletServerLogger { }); } } - // if the log gets too big, reset it .. grab the write lock first + // if the log gets too big or too old, reset it .. grab the write lock first logSizeEstimate.addAndGet(4 * 3); // event, tid, seq overhead testLockAndRun(logIdLock, new TestCallWithWriteLock() { @Override boolean test() { - return logSizeEstimate.get() > maxSize; + return (logSizeEstimate.get() > maxSize) || ((System.currentTimeMillis() - createTime) > maxAge); } @Override
