This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new a3444f7fda Fix MemoryStarved ITs (#4024)
a3444f7fda is described below
commit a3444f7fda056bed6e36f8b3bc6252abbe1807cb
Author: Christopher Tubbs <ctubb...@apache.org>
AuthorDate: Wed Dec 6 14:46:38 2023 -0500
Fix MemoryStarved ITs (#4024)
Rewrite MemoryConsumingIterator's method to compute the amount of memory
to consume, so that:
* The implementation is more comprehensible
* Replace exception with bounds checking
* Avoid allocating more than necessary (a single byte is sufficient)
* The log message includes the amount of used memory detected
* The waiting message appears, even when memory was allocated, because
that's the behavior
* Give the GC more time to detect the changed GC condition before trying
to detect the low memory condition
Also, remove hard-coded comments for size of heap and incorrect interval
frequency, and increase the configured free memory threshold, so that
the memory percentage isn't so low, it doesn't get lower than the
minimum that G1GC needs to do its job by default on a 256K VM.
This fixes #3868
Also, trivially remove an unused Logger
---
.../org/apache/accumulo/test/CloseScannerIT.java | 4 ---
.../test/functional/MemoryConsumingIterator.java | 33 +++++++++-------------
.../test/functional/MemoryStarvedMajCIT.java | 2 --
.../test/functional/MemoryStarvedMinCIT.java | 2 --
.../test/functional/MemoryStarvedScanIT.java | 4 +--
5 files changed, 14 insertions(+), 31 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/CloseScannerIT.java
b/test/src/main/java/org/apache/accumulo/test/CloseScannerIT.java
index 96b4b930c2..4dc89cac9a 100644
--- a/test/src/main/java/org/apache/accumulo/test/CloseScannerIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/CloseScannerIT.java
@@ -32,13 +32,9 @@ import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.test.functional.ReadWriteIT;
import org.apache.accumulo.test.util.Wait;
import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class CloseScannerIT extends AccumuloClusterHarness {
- private static final Logger LOG =
LoggerFactory.getLogger(CloseScannerIT.class);
-
static final int ROWS = 1000;
static final int COLS = 1000;
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryConsumingIterator.java
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryConsumingIterator.java
index 3b98f1c801..2b23fb1cff 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryConsumingIterator.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryConsumingIterator.java
@@ -39,8 +39,6 @@ public class MemoryConsumingIterator extends WrappingIterator
{
private static final List<byte[]> BUFFERS = new ArrayList<>();
- private static final int TEN_MiB = 10 * 1024 * 1024;
-
public static void freeBuffers() {
BUFFERS.clear();
}
@@ -50,23 +48,17 @@ public class MemoryConsumingIterator extends
WrappingIterator {
System.gc();
Runtime runtime = Runtime.getRuntime();
long maxConfiguredMemory = runtime.maxMemory();
- long allocatedMemory = runtime.totalMemory();
- long allocatedFreeMemory = runtime.freeMemory();
- long freeMemory = maxConfiguredMemory - (allocatedMemory -
allocatedFreeMemory);
- long minimumFreeMemoryThreshold =
- (long) (maxConfiguredMemory *
MemoryStarvedScanIT.FREE_MEMORY_THRESHOLD);
+ long usedMemory = runtime.totalMemory() - runtime.freeMemory();
+ long freeMemory = maxConfiguredMemory - usedMemory;
+ long minFreeMemory = (long) (maxConfiguredMemory *
MemoryStarvedScanIT.FREE_MEMORY_THRESHOLD);
- int amountToConsume = 0;
- if (freeMemory > minimumFreeMemoryThreshold) {
- amountToConsume = (int) (freeMemory - (minimumFreeMemoryThreshold -
TEN_MiB));
- }
- if (amountToConsume < 0) {
- throw new IllegalStateException(
- "Overflow. Unsupported memory size for tablet server when using this
iterator");
- }
- LOG.info("max: {}, free: {}, minFree: {}, amountToConsume: {}",
maxConfiguredMemory, freeMemory,
- minimumFreeMemoryThreshold, amountToConsume);
- return amountToConsume;
+ // consume free memory, and exceed the minimum threshold by just a little
bit
+ // don't exceed typical JDK byte array limit
+ long amountToConsume =
+ Math.min(Math.max(0, freeMemory + 1 - minFreeMemory),
Integer.MAX_VALUE - 8);
+ LOG.info("max: {}, used: {}, free: {}, minFree: {}, amountToConsume: {}",
maxConfiguredMemory,
+ usedMemory, freeMemory, minFreeMemory, amountToConsume);
+ return (int) amountToConsume;
}
@Override
@@ -80,10 +72,11 @@ public class MemoryConsumingIterator extends
WrappingIterator {
BUFFERS.add(new byte[amountToConsume]);
LOG.info("memory allocated");
} else {
- LOG.info("Waiting for LowMemoryDetector to recognize low on memory
condition.");
+ LOG.info("consumed enough; no more memory allocated");
}
+ LOG.info("Waiting for LowMemoryDetector to recognize low on memory
condition.");
try {
- Thread.sleep(SECONDS.toMillis(1));
+ Thread.sleep(SECONDS.toMillis(5));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IOException("interrupted during sleep", ex);
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java
index 366b62f935..771d74d588 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java
@@ -58,8 +58,6 @@ public class MemoryStarvedMajCIT extends
SharedMiniClusterBase {
cfg.setNumTservers(1);
cfg.setMemory(ServerType.TABLET_SERVER, 256, MemoryUnit.MEGABYTE);
// Configure the LowMemoryDetector in the TabletServer
- // check on 1s intervals and set low mem condition if more than 80% of
- // the heap is used.
cfg.setProperty(Property.GENERAL_LOW_MEM_DETECTOR_INTERVAL, "5s");
cfg.setProperty(Property.GENERAL_LOW_MEM_DETECTOR_THRESHOLD,
Double.toString(MemoryStarvedScanIT.FREE_MEMORY_THRESHOLD));
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java
index 0a001047cc..edb4cd03db 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMinCIT.java
@@ -56,8 +56,6 @@ public class MemoryStarvedMinCIT extends
SharedMiniClusterBase {
cfg.setNumTservers(1);
cfg.setMemory(ServerType.TABLET_SERVER, 256, MemoryUnit.MEGABYTE);
// Configure the LowMemoryDetector in the TabletServer
- // check on 1s intervals and set low mem condition if more than 80% of
- // the heap is used.
cfg.setProperty(Property.GENERAL_LOW_MEM_DETECTOR_INTERVAL, "5s");
cfg.setProperty(Property.GENERAL_LOW_MEM_DETECTOR_THRESHOLD,
Double.toString(MemoryStarvedScanIT.FREE_MEMORY_THRESHOLD));
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
index d06cef1308..59b9a535b8 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
@@ -70,8 +70,6 @@ public class MemoryStarvedScanIT extends
SharedMiniClusterBase {
cfg.setNumTservers(1);
cfg.setMemory(ServerType.TABLET_SERVER, 256, MemoryUnit.MEGABYTE);
// Configure the LowMemoryDetector in the TabletServer
- // check on 1s intervals and set low mem condition if more than 80% of
- // the heap is used.
cfg.setProperty(Property.GENERAL_LOW_MEM_DETECTOR_INTERVAL, "5s");
cfg.setProperty(Property.GENERAL_LOW_MEM_DETECTOR_THRESHOLD,
Double.toString(FREE_MEMORY_THRESHOLD));
@@ -87,7 +85,7 @@ public class MemoryStarvedScanIT extends
SharedMiniClusterBase {
}
}
- public static final double FREE_MEMORY_THRESHOLD = 0.20D;
+ public static final double FREE_MEMORY_THRESHOLD = 0.40D;
private static final Logger LOG =
LoggerFactory.getLogger(MemoryStarvedScanIT.class);
private static final DoubleAdder SCAN_START_DELAYED = new DoubleAdder();
