Repository: accumulo Updated Branches: refs/heads/master 173d80e11 -> 02450e4f6
ACCUMULO-4004: Added new property to close walogs after some time period. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0206d784 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0206d784 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0206d784 Branch: refs/heads/master Commit: 0206d7845bf8bba08e18373f1dbb8e9aaf6d6d60 Parents: f6bfe90 Author: Dave Marion <dlmar...@apache.org> Authored: Thu Mar 31 15:53:04 2016 -0400 Committer: Dave Marion <dlmar...@apache.org> Committed: Thu Mar 31 15:53:04 2016 -0400 ---------------------------------------------------------------------- .../java/org/apache/accumulo/core/conf/Property.java | 1 + .../java/org/apache/accumulo/tserver/TabletServer.java | 3 ++- .../apache/accumulo/tserver/log/TabletServerLogger.java | 11 ++++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/0206d784/core/src/main/java/org/apache/accumulo/core/conf/Property.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 9243494..2149ad9 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -209,6 +209,7 @@ public enum Property { + "must be made, which is slower. However opening too many files at once can cause problems."), TSERV_WALOG_MAX_SIZE("tserver.walog.max.size", "1G", PropertyType.MEMORY, "The maximum size for each write-ahead log. See comment for property tserver.memory.maps.max"), + TSERV_WALOG_MAX_AGE("tserver.walog.max.age", "24h", PropertyType.TIMEDURATION, "The maximum age for each write-ahead log."), TSERV_MAJC_DELAY("tserver.compaction.major.delay", "30s", PropertyType.TIMEDURATION, "Time a tablet server will sleep between checking which tablets need compaction."), TSERV_MAJC_THREAD_MAXOPEN("tserver.compaction.major.thread.files.open.max", "10", PropertyType.COUNT, http://git-wip-us.apache.org/repos/asf/accumulo/blob/0206d784/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index ac8f2ec..38bd8ac 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@ -3567,11 +3567,12 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu security = AuditedSecurityOperation.getInstance(); clientAddress = HostAndPort.fromParts(hostname, 0); long walogMaxSize = getSystemConfiguration().getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE); + long walogMaxAge = getSystemConfiguration().getTimeInMillis(Property.TSERV_WALOG_MAX_AGE); long minBlockSize = CachedConfiguration.getInstance().getLong("dfs.namenode.fs-limits.min-block-size", 0); if (minBlockSize != 0 && minBlockSize > walogMaxSize) throw new RuntimeException("Unable to start TabletServer. Logger is set to use blocksize " + walogMaxSize + " but hdfs minimum block size is " + minBlockSize + ". Either increase the " + Property.TSERV_WALOG_MAX_SIZE + " or decrease dfs.namenode.fs-limits.min-block-size in hdfs-site.xml."); - logger = new TabletServerLogger(this, walogMaxSize); + logger = new TabletServerLogger(this, walogMaxSize, walogMaxAge); try { AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig(new Iterable<Entry<String,String>>() { http://git-wip-us.apache.org/repos/asf/accumulo/blob/0206d784/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java index c864c78..158fdbd 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java @@ -56,6 +56,7 @@ public class TabletServerLogger { private final AtomicLong logSizeEstimate = new AtomicLong(); private final long maxSize; + private final long maxAge; private final TabletServer tserver; @@ -76,6 +77,8 @@ public class TabletServerLogger { private final AtomicInteger seqGen = new AtomicInteger(); + private long createTime = 0; + private static boolean enabled(Tablet tablet) { return tablet.getTableConfiguration().getBoolean(Property.TABLE_WALOG_ENABLED); } @@ -127,9 +130,10 @@ public class TabletServerLogger { } } - public TabletServerLogger(TabletServer tserver, long maxSize) { + public TabletServerLogger(TabletServer tserver, long maxSize, long maxAge) { this.tserver = tserver; this.maxSize = maxSize; + this.maxAge = maxAge; } private int initializeLoggers(final List<DfsLogger> copy) throws IOException { @@ -185,6 +189,7 @@ public class TabletServerLogger { alog.open(tserver.getClientAddressString()); loggers.add(alog); logSetId.incrementAndGet(); + this.createTime = System.currentTimeMillis(); return; } catch (Exception t) { throw new RuntimeException(t); @@ -312,11 +317,11 @@ public class TabletServerLogger { }); } } - // if the log gets too big, reset it .. grab the write lock first + // if the log gets too big or too old, reset it .. grab the write lock first logSizeEstimate.addAndGet(4 * 3); // event, tid, seq overhead testLockAndRun(logSetLock, new TestCallWithWriteLock() { boolean test() { - return logSizeEstimate.get() > maxSize; + return (logSizeEstimate.get() > maxSize) || ((System.currentTimeMillis() - createTime) > maxAge); } void withWriteLock() throws IOException {
