This is an automated email from the ASF dual-hosted git repository.
DomGarguilo pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 1bbee0531f Add memory used to scan trace spans (#6524)
1bbee0531f is described below
commit 1bbee0531f89e8e462c96a223f733366bc147db9
Author: Dom G. <[email protected]>
AuthorDate: Thu Sep 3 11:41:14 2026 -0400
Add memory used to scan trace spans (#6524)
* Add memory used to scan trace spans
* Add public constant for mem overhead
---
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 4 +++-
.../main/java/org/apache/accumulo/core/trace/TraceAttributes.java | 2 ++
.../src/main/java/org/apache/accumulo/tserver/tablet/KVEntry.java | 8 +++++++-
.../main/java/org/apache/accumulo/tserver/tablet/TabletBase.java | 3 +++
.../main/java/org/apache/accumulo/test/tracing/ScanTracingIT.java | 8 ++++++++
5 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 6a4caafbda..42c6f6b3a0 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -1211,7 +1211,9 @@ public enum Property {
"Options for the table scan dispatcher.", "2.0.0"),
TABLE_SCAN_MAXMEM("table.scan.max.memory", "512k", PropertyType.BYTES,
"The maximum amount of memory that will be used to cache results of a
client query/scan. "
- + "Once this limit is reached, the buffered data is sent to the
client.",
+ + "Once this limit is reached, the buffered data is sent to the
client. This is an "
+ + "estimate of heap usage and includes per entry object overhead, so
it will always "
+ + "exceed the size of the key value data actually sent to the
client.",
"1.3.5"),
TABLE_SHUFFLE_SOURCES("table.shuffle.sources", "false", PropertyType.BOOLEAN,
"Shuffle the opening order for Rfiles to reduce thread contention on
file open operations.",
diff --git
a/core/src/main/java/org/apache/accumulo/core/trace/TraceAttributes.java
b/core/src/main/java/org/apache/accumulo/core/trace/TraceAttributes.java
index e2c5b078b7..fe311417be 100644
--- a/core/src/main/java/org/apache/accumulo/core/trace/TraceAttributes.java
+++ b/core/src/main/java/org/apache/accumulo/core/trace/TraceAttributes.java
@@ -25,6 +25,8 @@ public class TraceAttributes {
AttributeKey.longKey("accumulo.scan.entries.returned");
public static final AttributeKey<Long> BYTES_RETURNED_KEY =
AttributeKey.longKey("accumulo.scan.bytes.returned");
+ public static final AttributeKey<Long> MEMORY_USED_KEY =
+ AttributeKey.longKey("accumulo.scan.memory.used");
public static final AttributeKey<Long> BYTES_READ_KEY =
AttributeKey.longKey("accumulo.scan.bytes.read");
public static final AttributeKey<Long> BYTES_READ_FILE_KEY =
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/KVEntry.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/KVEntry.java
index 1f085e9630..e99e24bf7a 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/KVEntry.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/KVEntry.java
@@ -25,6 +25,12 @@ import org.apache.accumulo.core.data.KeyValue;
import org.apache.accumulo.core.data.Value;
public class KVEntry extends KeyValue {
+
+ /**
+ * Estimated overhead for nine objects at 32 bytes each.
+ */
+ public static final int MEMORY_OVERHEAD = 9 * 32;
+
private static final long serialVersionUID = 1L;
public KVEntry(Key k, Value v) {
@@ -36,6 +42,6 @@ public class KVEntry extends KeyValue {
}
int estimateMemoryUsed() {
- return getKey().getSize() + getValue().get().length + (9 * 32); //
overhead is 32 per object
+ return getKey().getSize() + getValue().get().length + MEMORY_OVERHEAD;
}
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
index 0ac8f6bd14..52ac5f4dfa 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
@@ -253,10 +253,13 @@ public abstract class TabletBase {
if (span.isRecording()) {
span.setAttribute(TraceAttributes.ENTRIES_RETURNED_KEY, batch.size());
long bytesReturned = 0;
+ long memoryUsed = 0;
for (var e : batch) {
bytesReturned += e.getKey().getLength() + e.getValue().get().length;
+ memoryUsed += e.estimateMemoryUsed();
}
span.setAttribute(TraceAttributes.BYTES_RETURNED_KEY, bytesReturned);
+ span.setAttribute(TraceAttributes.MEMORY_USED_KEY, memoryUsed);
span.setAttribute(TraceAttributes.EXECUTOR_KEY,
scanParameters.getScanDispatch().getExecutorName());
span.setAttribute(TraceAttributes.TABLE_ID_KEY,
getExtent().tableId().canonical());
diff --git
a/test/src/main/java/org/apache/accumulo/test/tracing/ScanTracingIT.java
b/test/src/main/java/org/apache/accumulo/test/tracing/ScanTracingIT.java
index 70f22eb483..72c791fac4 100644
--- a/test/src/main/java/org/apache/accumulo/test/tracing/ScanTracingIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/tracing/ScanTracingIT.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.test.tracing;
import static org.apache.accumulo.core.trace.TraceAttributes.EXECUTOR_KEY;
import static org.apache.accumulo.core.trace.TraceAttributes.EXTENT_KEY;
+import static org.apache.accumulo.tserver.tablet.KVEntry.MEMORY_OVERHEAD;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -45,6 +46,7 @@ import
org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.accumulo.server.util.PortUtils;
import org.apache.accumulo.test.TestIngest;
import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.accumulo.tserver.tablet.KVEntry;
import org.apache.hadoop.conf.Configuration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -231,6 +233,8 @@ class ScanTracingIT extends ConfigurableMacBase {
double colMultiplier = 10.0 / expectedColumns;
assertClose((long) (results.scanSize * colMultiplier),
stats.getBytesRead(), .05);
assertClose(results.scanSize, stats.getBytesReturned(), .05);
+ assertEquals(stats.getBytesReturned() + KVEntry.MEMORY_OVERHEAD *
stats.getEntriesReturned(),
+ stats.getMemoryUsed(), stats::toString);
if (secondScanFitsInCache && i == 1) {
assertEquals(0, stats.getFileBytesRead(), stats::toString);
} else {
@@ -335,6 +339,10 @@ class ScanTracingIT extends ConfigurableMacBase {
return
scanStats.getOrDefault(TraceAttributes.BYTES_RETURNED_KEY.getKey(), 0L);
}
+ long getMemoryUsed() {
+ return scanStats.getOrDefault(TraceAttributes.MEMORY_USED_KEY.getKey(),
0L);
+ }
+
long getDataCacheHits() {
return scanStats.getOrDefault(TraceAttributes.DATA_HITS_KEY.getKey(),
0L);
}
