This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 26f316d188 Removes deprecated minor compaction idle time (#4888)
26f316d188 is described below
commit 26f316d188873383a7322a64d64bf1d0c94dc0d5
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Tue Sep 17 12:12:18 2024 -0400
Removes deprecated minor compaction idle time (#4888)
---
.../org/apache/accumulo/core/conf/Property.java | 8 --
.../tserver/TabletServerResourceManager.java | 17 +--
.../tserver/memory/LargestFirstMemoryManager.java | 52 ++-------
.../tserver/memory/TabletMemoryReport.java | 10 +-
.../memory/LargestFirstMemoryManagerTest.java | 129 +++++++++------------
.../accumulo/test/ScanServerMaxLatencyIT.java | 23 +---
.../test/functional/ManyWriteAheadLogsIT.java | 5 +-
7 files changed, 77 insertions(+), 167 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 670ff7d292..1947ea376e 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -865,14 +865,6 @@ public enum Property {
+ "server to see a write to a tablet server. The default value of
this property is set to such a "
+ "high value that is should never cause a minor compaction.",
"3.1.0"),
- @Deprecated(since = "3.1.0")
- @ReplacedBy(property = TABLE_MINC_COMPACT_MAXAGE)
- TABLE_MINC_COMPACT_IDLETIME("table.compaction.minor.idle", "5m",
PropertyType.TIMEDURATION,
- "When the age of the youngest key value in a tablets in memory map
exceeds this configuration, then"
- + " a minor compaction may be initiated. There is no guarantee an
idle tablet will be compacted."
- + "This property was deprecated because the new property
table.compaction.minor.age can offer the "
- + " same functionality although it may cause more minor compactions
than this property would have.",
- "1.3.5"),
TABLE_COMPACTION_DISPATCHER("table.compaction.dispatcher",
SimpleCompactionDispatcher.class.getName(), PropertyType.CLASSNAME,
"A configurable dispatcher that decides what compaction service a table
should use.",
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
index fe8f33da5e..37037d2f6c 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
@@ -577,10 +577,9 @@ public class TabletServerResourceManager {
}
}
- public void updateMemoryUsageStats(Tablet tablet, long size, long
lastCommitTime, long mincSize,
+ public void updateMemoryUsageStats(Tablet tablet, long size, long mincSize,
Timer firstWriteTimer) {
- memUsageReports
- .add(new TabletMemoryReport(tablet, lastCommitTime, size, mincSize,
firstWriteTimer));
+ memUsageReports.add(new TabletMemoryReport(tablet, size, mincSize,
firstWriteTimer));
}
public void tabletClosed(KeyExtent extent) {
@@ -686,7 +685,6 @@ public class TabletServerResourceManager {
private final AtomicLong lastReportedSize = new AtomicLong();
private final AtomicLong lastReportedMincSize = new AtomicLong();
private final AtomicReference<Timer> firstReportedCommitTimer = new
AtomicReference<>(null);
- private volatile long lastReportedCommitTime = 0;
public void updateMemoryUsageStats(Tablet tablet, long size, long
mincSize) {
@@ -721,19 +719,12 @@ public class TabletServerResourceManager {
report = true;
}
- long currentTime = System.currentTimeMillis();
-
- if ((delta > 32000 || delta < 0 || (currentTime - lastReportedCommitTime
> 1000))
- && lastReportedSize.compareAndSet(lrs, totalSize)) {
- if (delta > 0) {
- lastReportedCommitTime = currentTime;
- }
+ if ((delta > 32000 || delta < 0) && lastReportedSize.compareAndSet(lrs,
totalSize)) {
report = true;
}
if (report) {
- memMgmt.updateMemoryUsageStats(tablet, size, lastReportedCommitTime,
mincSize,
- firstReportedCommitTimer.get());
+ memMgmt.updateMemoryUsageStats(tablet, size, mincSize,
firstReportedCommitTimer.get());
}
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/memory/LargestFirstMemoryManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/memory/LargestFirstMemoryManager.java
index 42623f7027..8a6d1b939c 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/memory/LargestFirstMemoryManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/memory/LargestFirstMemoryManager.java
@@ -45,7 +45,6 @@ import org.slf4j.LoggerFactory;
public class LargestFirstMemoryManager {
private static final Logger log =
LoggerFactory.getLogger(LargestFirstMemoryManager.class);
- static final long ZERO_TIME = System.currentTimeMillis();
private static final int TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER = 2;
private static final double MAX_FLUSH_AT_ONCE_PERCENT = 0.20;
@@ -63,14 +62,10 @@ public class LargestFirstMemoryManager {
private static class TabletInfo {
final KeyExtent extent;
final long memTableSize;
- final long idleTime;
- final long load;
- public TabletInfo(KeyExtent extent, long memTableSize, long idleTime, long
load) {
+ public TabletInfo(KeyExtent extent, long memTableSize) {
this.extent = extent;
this.memTableSize = memTableSize;
- this.idleTime = idleTime;
- this.load = load;
}
}
@@ -138,13 +133,6 @@ public class LargestFirstMemoryManager {
maxObserved = 0;
}
- @SuppressWarnings("deprecation")
- protected long getMinCIdleThreshold(KeyExtent extent) {
- TableId tableId = extent.tableId();
- return mincIdleThresholds.computeIfAbsent(tableId, tid ->
context.getTableConfiguration(tid)
- .getTimeInMillis(Property.TABLE_MINC_COMPACT_IDLETIME));
- }
-
protected long getMaxAge(KeyExtent extent) {
TableId tableId = extent.tableId();
return mincAgeThresholds.computeIfAbsent(tableId, tid ->
context.getTableConfiguration(tid)
@@ -174,8 +162,7 @@ public class LargestFirstMemoryManager {
final List<KeyExtent> tabletsToMinorCompact = new ArrayList<>();
LargestMap largestMemTablets = new LargestMap(maxMinCs);
- final LargestMap largestIdleMemTablets = new
LargestMap(maxConcurrentMincs);
- final long now = currentTimeMillis();
+ final LargestMap oldMemTablets = new LargestMap(maxConcurrentMincs);
long ingestMemory = 0;
long compactionMemory = 0;
@@ -191,17 +178,15 @@ public class LargestFirstMemoryManager {
}
final long memTabletSize = ts.getMemTableSize();
+
final long minorCompactingSize = ts.getMinorCompactingMemTableSize();
- final long idleTime = now - Math.max(ts.getLastCommitTime(), ZERO_TIME);
- final long timeMemoryLoad = timeMemoryLoad(memTabletSize, idleTime);
ingestMemory += memTabletSize;
if (minorCompactingSize == 0 && memTabletSize > 0) {
- TabletInfo tabletInfo = new TabletInfo(tablet, memTabletSize,
idleTime, timeMemoryLoad);
+ TabletInfo tabletInfo = new TabletInfo(tablet, memTabletSize);
try {
// If the table was deleted, getMinCIdleThreshold will throw an
exception
- if (idleTime > getMinCIdleThreshold(tablet)
- || ts.getElapsedSinceFirstWrite(TimeUnit.MILLISECONDS) >
getMaxAge(tablet)) {
- largestIdleMemTablets.put(timeMemoryLoad, tabletInfo);
+ if (ts.getElapsedSinceFirstWrite(TimeUnit.MILLISECONDS) >
getMaxAge(tablet)) {
+ oldMemTablets.put(memTabletSize, tabletInfo);
}
} catch (IllegalArgumentException e) {
Throwable cause = e.getCause();
@@ -216,7 +201,7 @@ public class LargestFirstMemoryManager {
throw e;
}
// Only place the tablet into largestMemTablets map when the table
still exists
- largestMemTablets.put(timeMemoryLoad, tabletInfo);
+ largestMemTablets.put(memTabletSize, tabletInfo);
}
compactionMemory += minorCompactingSize;
@@ -241,10 +226,9 @@ public class LargestFirstMemoryManager {
// compaction
if (memoryChange >= 0 && ingestMemory + memoryChange >
compactionThreshold * maxMemory) {
startMinC = true;
- } else if (!largestIdleMemTablets.isEmpty()) {
+ } else if (!oldMemTablets.isEmpty()) {
startMinC = true;
- // switch largestMemTablets to largestIdleMemTablets
- largestMemTablets = largestIdleMemTablets;
+ largestMemTablets = oldMemTablets;
log.debug("IDLE minor compaction chosen");
}
}
@@ -257,10 +241,9 @@ public class LargestFirstMemoryManager {
for (TabletInfo largest : lastEntry.getValue()) {
toBeCompacted += largest.memTableSize;
tabletsToMinorCompact.add(largest.extent);
- log.debug(String.format("COMPACTING %s total = %,d ingestMemory =
%,d",
- largest.extent.toString(), (ingestMemory + compactionMemory),
ingestMemory));
- log.debug(String.format("chosenMem = %,d chosenIT = %.2f load %,d",
largest.memTableSize,
- largest.idleTime / 1000.0, largest.load));
+ log.debug(String.format("COMPACTING %s total = %,d ingestMemory =
%,d chosenMem = %,d",
+ largest.extent.toString(), (ingestMemory + compactionMemory),
ingestMemory,
+ largest.memTableSize));
if (toBeCompacted > ingestMemory * MAX_FLUSH_AT_ONCE_PERCENT) {
break outer;
}
@@ -297,15 +280,4 @@ public class LargestFirstMemoryManager {
return tabletsToMinorCompact;
}
-
- protected long currentTimeMillis() {
- return System.currentTimeMillis();
- }
-
- // The load function: memory times the idle time, doubling every 15 mins
- static long timeMemoryLoad(long mem, long time) {
- double minutesIdle = time / 60000.0;
-
- return (long) (mem * Math.pow(2, minutesIdle / 15.0));
- }
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/memory/TabletMemoryReport.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/memory/TabletMemoryReport.java
index 00339d9711..1f1f7cc9fe 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/memory/TabletMemoryReport.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/memory/TabletMemoryReport.java
@@ -27,15 +27,13 @@ import org.apache.accumulo.tserver.tablet.Tablet;
public class TabletMemoryReport implements Cloneable {
private final Tablet tablet;
- private final long lastCommitTime;
private final long memTableSize;
private final long minorCompactingMemTableSize;
private final Timer firstWriteTimer;
- public TabletMemoryReport(Tablet tablet, long lastCommitTime, long
memTableSize,
- long minorCompactingMemTableSize, Timer firstWriteTimer) {
+ public TabletMemoryReport(Tablet tablet, long memTableSize, long
minorCompactingMemTableSize,
+ Timer firstWriteTimer) {
this.tablet = tablet;
- this.lastCommitTime = lastCommitTime;
this.memTableSize = memTableSize;
this.minorCompactingMemTableSize = minorCompactingMemTableSize;
this.firstWriteTimer = firstWriteTimer;
@@ -49,10 +47,6 @@ public class TabletMemoryReport implements Cloneable {
return tablet;
}
- public long getLastCommitTime() {
- return lastCommitTime;
- }
-
public long getElapsedSinceFirstWrite(TimeUnit unit) {
return firstWriteTimer == null ? 0 : firstWriteTimer.elapsed(unit);
}
diff --git
a/server/tserver/src/test/java/org/apache/accumulo/tserver/memory/LargestFirstMemoryManagerTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/memory/LargestFirstMemoryManagerTest.java
index 35b2459e95..b24ff2ab0b 100644
---
a/server/tserver/src/test/java/org/apache/accumulo/tserver/memory/LargestFirstMemoryManagerTest.java
+++
b/server/tserver/src/test/java/org/apache/accumulo/tserver/memory/LargestFirstMemoryManagerTest.java
@@ -43,8 +43,6 @@ import org.junit.jupiter.api.Timeout;
@Timeout(60)
public class LargestFirstMemoryManagerTest {
- private static final long ZERO = LargestFirstMemoryManager.ZERO_TIME;
- private static final long LATER = ZERO + MINUTES.toMillis(20);
private static final long ONE_GIG = 1024 * 1024 * 1024;
private static final long ONE_MEG = 1024 * 1024;
private static final long HALF_GIG = ONE_GIG / 2;
@@ -69,108 +67,99 @@ public class LargestFirstMemoryManagerTest {
List<KeyExtent> tabletsToMinorCompact;
// nothing to do
tabletsToMinorCompact =
- mgr.tabletsToMinorCompact(tablets(t(k("x"), ZERO, 1000, 0), t(k("y"),
ZERO, 2000, 0)));
+ mgr.tabletsToMinorCompact(tablets(t(k("x"), 1000, 0), t(k("y"), 2000,
0)));
assertEquals(0, tabletsToMinorCompact.size());
// one tablet is really big
tabletsToMinorCompact =
- mgr.tabletsToMinorCompact(tablets(t(k("x"), ZERO, ONE_GIG, 0),
t(k("y"), ZERO, 2000, 0)));
+ mgr.tabletsToMinorCompact(tablets(t(k("x"), ONE_GIG, 0), t(k("y"),
2000, 0)));
assertEquals(1, tabletsToMinorCompact.size());
assertEquals(k("x"), tabletsToMinorCompact.get(0));
- // one tablet is idle
- mgr.currentTime = LATER;
tabletsToMinorCompact =
- mgr.tabletsToMinorCompact(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"),
LATER, 2000, 0)));
- assertEquals(1, tabletsToMinorCompact.size());
- assertEquals(k("x"), tabletsToMinorCompact.get(0));
- // one tablet is idle, but one is really big
+ mgr.tabletsToMinorCompact(tablets(t(k("x"), 1000, 0), t(k("y"), 2000,
0)));
+ assertEquals(0, tabletsToMinorCompact.size());
tabletsToMinorCompact =
- mgr.tabletsToMinorCompact(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"),
LATER, ONE_GIG, 0)));
+ mgr.tabletsToMinorCompact(tablets(t(k("x"), 1001, 0), t(k("y"),
ONE_GIG, 0)));
assertEquals(1, tabletsToMinorCompact.size());
assertEquals(k("y"), tabletsToMinorCompact.get(0));
// lots of work to do
mgr = new LargestFirstMemoryManagerUnderTest();
mgr.init(context);
- tabletsToMinorCompact = mgr.tabletsToMinorCompact(tablets(t(k("a"), ZERO,
HALF_GIG, 0),
- t(k("b"), ZERO, HALF_GIG + ONE_MEG, 0), t(k("c"), ZERO, HALF_GIG + (2
* ONE_MEG), 0),
- t(k("d"), ZERO, HALF_GIG + (3 * ONE_MEG), 0), t(k("e"), ZERO, HALF_GIG
+ (4 * ONE_MEG), 0),
- t(k("f"), ZERO, HALF_GIG + (5 * ONE_MEG), 0), t(k("g"), ZERO, HALF_GIG
+ (6 * ONE_MEG), 0),
- t(k("h"), ZERO, HALF_GIG + (7 * ONE_MEG), 0),
- t(k("i"), ZERO, HALF_GIG + (8 * ONE_MEG), 0)));
+ tabletsToMinorCompact = mgr.tabletsToMinorCompact(tablets(t(k("a"),
HALF_GIG, 0),
+ t(k("b"), HALF_GIG + ONE_MEG, 0), t(k("c"), HALF_GIG + (2 * ONE_MEG),
0),
+ t(k("d"), HALF_GIG + (3 * ONE_MEG), 0), t(k("e"), HALF_GIG + (4 *
ONE_MEG), 0),
+ t(k("f"), HALF_GIG + (5 * ONE_MEG), 0), t(k("g"), HALF_GIG + (6 *
ONE_MEG), 0),
+ t(k("h"), HALF_GIG + (7 * ONE_MEG), 0), t(k("i"), HALF_GIG + (8 *
ONE_MEG), 0)));
assertEquals(2, tabletsToMinorCompact.size());
assertEquals(k("i"), tabletsToMinorCompact.get(0));
assertEquals(k("h"), tabletsToMinorCompact.get(1));
// one finished, one in progress, one filled up
mgr = new LargestFirstMemoryManagerUnderTest();
mgr.init(context);
- tabletsToMinorCompact = mgr.tabletsToMinorCompact(tablets(t(k("a"), ZERO,
HALF_GIG, 0),
- t(k("b"), ZERO, HALF_GIG + ONE_MEG, 0), t(k("c"), ZERO, HALF_GIG + (2
* ONE_MEG), 0),
- t(k("d"), ZERO, HALF_GIG + (3 * ONE_MEG), 0), t(k("e"), ZERO, HALF_GIG
+ (4 * ONE_MEG), 0),
- t(k("f"), ZERO, HALF_GIG + (5 * ONE_MEG), 0), t(k("g"), ZERO, ONE_GIG,
0),
- t(k("h"), ZERO, 0, HALF_GIG + (7 * ONE_MEG)), t(k("i"), ZERO, 0, 0)));
+ tabletsToMinorCompact =
+ mgr.tabletsToMinorCompact(tablets(t(k("a"), HALF_GIG, 0), t(k("b"),
HALF_GIG + ONE_MEG, 0),
+ t(k("c"), HALF_GIG + (2 * ONE_MEG), 0), t(k("d"), HALF_GIG + (3 *
ONE_MEG), 0),
+ t(k("e"), HALF_GIG + (4 * ONE_MEG), 0), t(k("f"), HALF_GIG + (5 *
ONE_MEG), 0),
+ t(k("g"), ONE_GIG, 0), t(k("h"), 0, HALF_GIG + (7 * ONE_MEG)),
t(k("i"), 0, 0)));
assertEquals(1, tabletsToMinorCompact.size());
assertEquals(k("g"), tabletsToMinorCompact.get(0));
// memory is very full, lots of candidates
- tabletsToMinorCompact = mgr.tabletsToMinorCompact(tablets(t(k("a"), ZERO,
HALF_GIG, 0),
- t(k("b"), ZERO, ONE_GIG + ONE_MEG, 0), t(k("c"), ZERO, ONE_GIG + (2 *
ONE_MEG), 0),
- t(k("d"), ZERO, ONE_GIG + (3 * ONE_MEG), 0), t(k("e"), ZERO, ONE_GIG +
(4 * ONE_MEG), 0),
- t(k("f"), ZERO, ONE_GIG + (5 * ONE_MEG), 0), t(k("g"), ZERO, ONE_GIG +
(6 * ONE_MEG), 0),
- t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
+ tabletsToMinorCompact =
+ mgr.tabletsToMinorCompact(tablets(t(k("a"), HALF_GIG, 0), t(k("b"),
ONE_GIG + ONE_MEG, 0),
+ t(k("c"), ONE_GIG + (2 * ONE_MEG), 0), t(k("d"), ONE_GIG + (3 *
ONE_MEG), 0),
+ t(k("e"), ONE_GIG + (4 * ONE_MEG), 0), t(k("f"), ONE_GIG + (5 *
ONE_MEG), 0),
+ t(k("g"), ONE_GIG + (6 * ONE_MEG), 0), t(k("h"), 0, 0), t(k("i"),
0, 0)));
assertEquals(2, tabletsToMinorCompact.size());
assertEquals(k("g"), tabletsToMinorCompact.get(0));
assertEquals(k("f"), tabletsToMinorCompact.get(1));
// only have two compactors, still busy
- tabletsToMinorCompact = mgr.tabletsToMinorCompact(tablets(t(k("a"), ZERO,
HALF_GIG, 0),
- t(k("b"), ZERO, ONE_GIG + ONE_MEG, 0), t(k("c"), ZERO, ONE_GIG + (2 *
ONE_MEG), 0),
- t(k("d"), ZERO, ONE_GIG + (3 * ONE_MEG), 0), t(k("e"), ZERO, ONE_GIG +
(4 * ONE_MEG), 0),
- t(k("f"), ZERO, ONE_GIG, ONE_GIG + (5 * ONE_MEG)),
- t(k("g"), ZERO, ONE_GIG, ONE_GIG + (6 * ONE_MEG)), t(k("h"), ZERO, 0,
0),
- t(k("i"), ZERO, 0, 0)));
+ tabletsToMinorCompact =
+ mgr.tabletsToMinorCompact(tablets(t(k("a"), HALF_GIG, 0), t(k("b"),
ONE_GIG + ONE_MEG, 0),
+ t(k("c"), ONE_GIG + (2 * ONE_MEG), 0), t(k("d"), ONE_GIG + (3 *
ONE_MEG), 0),
+ t(k("e"), ONE_GIG + (4 * ONE_MEG), 0), t(k("f"), ONE_GIG, ONE_GIG
+ (5 * ONE_MEG)),
+ t(k("g"), ONE_GIG, ONE_GIG + (6 * ONE_MEG)), t(k("h"), 0, 0),
t(k("i"), 0, 0)));
assertEquals(0, tabletsToMinorCompact.size());
// finished one
- tabletsToMinorCompact = mgr.tabletsToMinorCompact(tablets(t(k("a"), ZERO,
HALF_GIG, 0),
- t(k("b"), ZERO, ONE_GIG + ONE_MEG, 0), t(k("c"), ZERO, ONE_GIG + (2 *
ONE_MEG), 0),
- t(k("d"), ZERO, ONE_GIG + (3 * ONE_MEG), 0), t(k("e"), ZERO, ONE_GIG +
(4 * ONE_MEG), 0),
- t(k("f"), ZERO, ONE_GIG, ONE_GIG + (5 * ONE_MEG)), t(k("g"), ZERO,
ONE_GIG, 0),
- t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0)));
+ tabletsToMinorCompact =
+ mgr.tabletsToMinorCompact(tablets(t(k("a"), HALF_GIG, 0), t(k("b"),
ONE_GIG + ONE_MEG, 0),
+ t(k("c"), ONE_GIG + (2 * ONE_MEG), 0), t(k("d"), ONE_GIG + (3 *
ONE_MEG), 0),
+ t(k("e"), ONE_GIG + (4 * ONE_MEG), 0), t(k("f"), ONE_GIG, ONE_GIG
+ (5 * ONE_MEG)),
+ t(k("g"), ONE_GIG, 0), t(k("h"), 0, 0), t(k("i"), 0, 0)));
assertEquals(1, tabletsToMinorCompact.size());
assertEquals(k("e"), tabletsToMinorCompact.get(0));
// many are running: do nothing
mgr = new LargestFirstMemoryManagerUnderTest();
mgr.init(context);
- tabletsToMinorCompact = mgr.tabletsToMinorCompact(tablets(t(k("a"), ZERO,
HALF_GIG, 0),
- t(k("b"), ZERO, HALF_GIG + ONE_MEG, 0), t(k("c"), ZERO, HALF_GIG + (2
* ONE_MEG), 0),
- t(k("d"), ZERO, 0, HALF_GIG), t(k("e"), ZERO, 0, HALF_GIG), t(k("f"),
ZERO, 0, HALF_GIG),
- t(k("g"), ZERO, 0, HALF_GIG), t(k("i"), ZERO, 0, HALF_GIG), t(k("j"),
ZERO, 0, HALF_GIG),
- t(k("k"), ZERO, 0, HALF_GIG), t(k("l"), ZERO, 0, HALF_GIG), t(k("m"),
ZERO, 0, HALF_GIG)));
+ tabletsToMinorCompact = mgr.tabletsToMinorCompact(tablets(t(k("a"),
HALF_GIG, 0),
+ t(k("b"), HALF_GIG + ONE_MEG, 0), t(k("c"), HALF_GIG + (2 * ONE_MEG),
0),
+ t(k("d"), 0, HALF_GIG), t(k("e"), 0, HALF_GIG), t(k("f"), 0, HALF_GIG),
+ t(k("g"), 0, HALF_GIG), t(k("i"), 0, HALF_GIG), t(k("j"), 0, HALF_GIG),
+ t(k("k"), 0, HALF_GIG), t(k("l"), 0, HALF_GIG), t(k("m"), 0,
HALF_GIG)));
assertEquals(0, tabletsToMinorCompact.size());
// observe adjustment:
mgr = new LargestFirstMemoryManagerUnderTest();
mgr.init(context);
// compact the largest
- tabletsToMinorCompact = mgr.tabletsToMinorCompact(tablets(t(k("a"), ZERO,
QGIG, 0),
- t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, QGIG + 2, 0)));
+ tabletsToMinorCompact = mgr.tabletsToMinorCompact(
+ tablets(t(k("a"), QGIG, 0), t(k("b"), QGIG + 1, 0), t(k("c"), QGIG +
2, 0)));
assertEquals(1, tabletsToMinorCompact.size());
assertEquals(k("c"), tabletsToMinorCompact.get(0));
// show that it is compacting... do nothing
- tabletsToMinorCompact = mgr.tabletsToMinorCompact(tablets(t(k("a"), ZERO,
QGIG, 0),
- t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2)));
+ tabletsToMinorCompact = mgr.tabletsToMinorCompact(
+ tablets(t(k("a"), QGIG, 0), t(k("b"), QGIG + 1, 0), t(k("c"), 0, QGIG
+ 2)));
assertEquals(0, tabletsToMinorCompact.size());
// not going to bother compacting any more
- mgr.currentTime += MINUTES.toMillis(1);
- tabletsToMinorCompact = mgr.tabletsToMinorCompact(tablets(t(k("a"), ZERO,
QGIG, 0),
- t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2)));
+ tabletsToMinorCompact = mgr.tabletsToMinorCompact(
+ tablets(t(k("a"), QGIG, 0), t(k("b"), QGIG + 1, 0), t(k("c"), 0, QGIG
+ 2)));
assertEquals(0, tabletsToMinorCompact.size());
// now do nothing
- mgr.currentTime += MINUTES.toMillis(1);
- tabletsToMinorCompact = mgr.tabletsToMinorCompact(
- tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, 0, 0), t(k("c"),
ZERO, 0, 0)));
+ tabletsToMinorCompact =
+ mgr.tabletsToMinorCompact(tablets(t(k("a"), QGIG, 0), t(k("b"), 0, 0),
t(k("c"), 0, 0)));
assertEquals(0, tabletsToMinorCompact.size());
// on no! more data, this time we compact because we've adjusted
- mgr.currentTime += MINUTES.toMillis(1);
tabletsToMinorCompact = mgr.tabletsToMinorCompact(
- tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0),
t(k("c"), ZERO, 0, 0)));
+ tablets(t(k("a"), QGIG, 0), t(k("b"), QGIG + 1, 0), t(k("c"), 0, 0)));
assertEquals(1, tabletsToMinorCompact.size());
assertEquals(k("b"), tabletsToMinorCompact.get(0));
}
@@ -190,8 +179,8 @@ public class LargestFirstMemoryManagerTest {
// one tablet is really big and the other is for a nonexistent table
KeyExtent extent = new KeyExtent(TableId.of("3"), new Text("j"), null);
KeyExtent extent2 = new KeyExtent(TableId.of("2"), new Text("j"), null);
- tabletsToMinorCompact = mgr.tabletsToMinorCompact(tablets(t(extent, ZERO,
ONE_GIG, 0),
- t(extent2, ZERO, ONE_GIG, 0), t(k("j"), ZERO, ONE_GIG, 0)));
+ tabletsToMinorCompact = mgr.tabletsToMinorCompact(
+ tablets(t(extent, ONE_GIG, 0), t(extent2, ONE_GIG, 0), t(k("j"),
ONE_GIG, 0)));
assertEquals(1, tabletsToMinorCompact.size());
assertEquals(extent, tabletsToMinorCompact.get(0));
}
@@ -204,31 +193,19 @@ public class LargestFirstMemoryManagerTest {
// nothing to do
tabletsToMinorCompact =
- mgr.tabletsToMinorCompact(tablets(t(k("x"), ZERO, 1000, 0), t(k("y"),
ZERO, 2000, 0)));
+ mgr.tabletsToMinorCompact(tablets(t(k("x"), 1000, 0), t(k("y"), 2000,
0)));
assertEquals(0, tabletsToMinorCompact.size());
// a tablet that exceeds the configured max age should need to compact
tabletsToMinorCompact =
- mgr.tabletsToMinorCompact(tablets(t(k("x"), ZERO, 1000, 0,
Duration.ofMinutes(14)),
- t(k("y"), ZERO, 2000, 0, Duration.ofMinutes(16))));
+ mgr.tabletsToMinorCompact(tablets(t(k("x"), 1000, 0,
Duration.ofMinutes(14)),
+ t(k("y"), 2000, 0, Duration.ofMinutes(16))));
assertEquals(1, tabletsToMinorCompact.size());
assertEquals(k("y"), tabletsToMinorCompact.get(0));
}
private static class LargestFirstMemoryManagerUnderTest extends
LargestFirstMemoryManager {
- public long currentTime = ZERO;
-
- @Override
- protected long currentTimeMillis() {
- return currentTime;
- }
-
- @Override
- protected long getMinCIdleThreshold(KeyExtent extent) {
- return MINUTES.toMillis(15);
- }
-
@Override
protected long getMaxAge(KeyExtent extent) {
return MINUTES.toMillis(15);
@@ -272,8 +249,8 @@ public class LargestFirstMemoryManagerTest {
return new KeyExtent(TableId.of("1"), new Text(endRow), null);
}
- private TabletMemoryReport t(KeyExtent ke, long lastCommit, long memSize,
long compactingSize) {
- return new TabletMemoryReport(null, lastCommit, memSize, compactingSize,
null) {
+ private TabletMemoryReport t(KeyExtent ke, long memSize, long
compactingSize) {
+ return new TabletMemoryReport(null, memSize, compactingSize, null) {
@Override
public KeyExtent getExtent() {
return ke;
@@ -281,9 +258,9 @@ public class LargestFirstMemoryManagerTest {
};
}
- private TabletMemoryReport t(KeyExtent ke, long lastCommit, long memSize,
long compactingSize,
+ private TabletMemoryReport t(KeyExtent ke, long memSize, long compactingSize,
Duration elapsedSinceFirstWrite) {
- return new TabletMemoryReport(null, lastCommit, memSize, compactingSize,
null) {
+ return new TabletMemoryReport(null, memSize, compactingSize, null) {
@Override
public KeyExtent getExtent() {
return ke;
diff --git
a/test/src/main/java/org/apache/accumulo/test/ScanServerMaxLatencyIT.java
b/test/src/main/java/org/apache/accumulo/test/ScanServerMaxLatencyIT.java
index b6a0b81894..8e2fb7b796 100644
--- a/test/src/main/java/org/apache/accumulo/test/ScanServerMaxLatencyIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ScanServerMaxLatencyIT.java
@@ -57,16 +57,12 @@ public class ScanServerMaxLatencyIT extends
ConfigurableMacBase {
cfg.setProperty(Property.SSERV_CACHED_TABLET_METADATA_EXPIRATION, "2s");
}
- @SuppressWarnings("deprecation")
- private static Property IDLE_MINC_PROP =
Property.TABLE_MINC_COMPACT_IDLETIME;
-
@Test
public void testMaxLatency() throws Exception {
- final String[] tables = this.getUniqueNames(4);
+ final String[] tables = this.getUniqueNames(3);
final String table1 = tables[0];
final String table2 = tables[1];
final String table3 = tables[2];
- final String table4 = tables[3];
ExecutorService executor = Executors.newCachedThreadPool();
try (var client =
Accumulo.newClient().from(getClientProperties()).build()) {
@@ -81,30 +77,24 @@ public class ScanServerMaxLatencyIT extends
ConfigurableMacBase {
ntc.withInitialTabletAvailability(TabletAvailability.HOSTED);
client.tableOperations().create(table2, ntc);
ntc = new NewTableConfiguration();
- ntc.setProperties(Map.of(IDLE_MINC_PROP.getKey(), "2s"));
- ntc.withInitialTabletAvailability(TabletAvailability.HOSTED);
- client.tableOperations().create(table3, ntc);
- ntc = new NewTableConfiguration();
ntc.setProperties(Map.of(Property.TABLE_MINC_COMPACT_MAXAGE.getKey(),
"3s"));
ntc.withInitialTabletAvailability(TabletAvailability.HOSTED);
- client.tableOperations().create(table4, ntc);
+ client.tableOperations().create(table3, ntc);
Timer timer = Timer.startNew();
// Write to table4 once, this is different than the other tables that
are continually being
// written to. table4 should minor compact 3 seconds after this write.
- writeElapsed(new SecureRandom(), client, table4, timer);
+ writeElapsed(new SecureRandom(), client, table3, timer);
boolean sawDataInTable4 = false;
List<Future<Void>> futures = new ArrayList<>();
futures.add(executor.submit(createWriterTask(client, table1, timer)));
futures.add(executor.submit(createWriterTask(client, table2, timer)));
- futures.add(executor.submit(createWriterTask(client, table3, timer)));
// wait for some data to be written
Wait.waitFor(() -> readMaxElapsed(client, IMMEDIATE, table1) > 0
- && readMaxElapsed(client, IMMEDIATE, table2) > 0
- && readMaxElapsed(client, IMMEDIATE, table3) > 0);
+ && readMaxElapsed(client, IMMEDIATE, table2) > 0);
long lastMaxSeen = -1;
int changes = 0;
@@ -131,11 +121,8 @@ public class ScanServerMaxLatencyIT extends
ConfigurableMacBase {
// The other table does not have the setting to minor compact based on
age, so should never
// see any data for it from the scan server.
assertEquals(-1, readMaxElapsed(client, EVENTUAL, table2));
- // The background thread is writing to this table every 100ms so it
should not be considered
- // idle and therefor should not minor compact.
- assertEquals(-1, readMaxElapsed(client, EVENTUAL, table3));
- if (!sawDataInTable4 && readMaxElapsed(client, EVENTUAL, table4) !=
-1) {
+ if (!sawDataInTable4 && readMaxElapsed(client, EVENTUAL, table3) !=
-1) {
assertTrue(
timer.elapsed(TimeUnit.MILLISECONDS) > 3000
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/ManyWriteAheadLogsIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/ManyWriteAheadLogsIT.java
index f497430ba1..aa2d54d298 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/ManyWriteAheadLogsIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/ManyWriteAheadLogsIT.java
@@ -55,9 +55,6 @@ public class ManyWriteAheadLogsIT extends
AccumuloClusterHarness {
private String walSize;
- @SuppressWarnings("deprecation")
- private static Property IDLE_MINC_PROP =
Property.TABLE_MINC_COMPACT_IDLETIME;
-
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration
hadoopCoreSite) {
// configure a smaller wal size so the wals will roll frequently in the
test
@@ -69,7 +66,7 @@ public class ManyWriteAheadLogsIT extends
AccumuloClusterHarness {
// idle compactions may addess the problem this test is creating, however
they will not prevent
// lots of closed WALs for all write patterns. This test ensures code that
directly handles many
// tablets referencing many different WALs is working.
- cfg.setProperty(IDLE_MINC_PROP, "1h");
+ cfg.setProperty(Property.TABLE_MINC_COMPACT_MAXAGE, "1h");
cfg.getClusterServerConfiguration().setNumDefaultTabletServers(1);
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
