This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 1dc7596f2b consider fate table in compaction prioritization (#4164)
1dc7596f2b is described below
commit 1dc7596f2b1f9c3d62942838ea2f5783523c3186
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Wed Jan 17 12:01:18 2024 -0500
consider fate table in compaction prioritization (#4164)
---
.../util/compaction/CompactionJobPrioritizer.java | 44 ++++++++++++++--------
.../util/compaction/CompactionPrioritizerTest.java | 28 +++++++++++++-
2 files changed, 54 insertions(+), 18 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
index 9e81cce598..755d21e075 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
@@ -21,7 +21,7 @@ package org.apache.accumulo.core.util.compaction;
import java.util.Comparator;
import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
@@ -41,8 +41,14 @@ public class CompactionJobPrioritizer {
private static final short METADATA_USER_MIN = METADATA_USER_MAX - 1000;
private static final short METADATA_SYSTEM_MAX = METADATA_USER_MIN - 1;
private static final short METADATA_SYSTEM_MIN = METADATA_SYSTEM_MAX - 1000;
- private static final short USER_USER_MAX = METADATA_SYSTEM_MIN - 1;
- private static final short USER_USER_MIN = USER_USER_MAX - 30768;
+ // ACCUMULO_OTHER is intended for tables in the accumulo namespace that are
not accumulo.root or
+ // accumulo.metadata
+ private static final short ACCUMULO_OTHER_USER_MAX = METADATA_SYSTEM_MIN - 1;
+ private static final short ACCUMULO_OTHER_USER_MIN = ACCUMULO_OTHER_USER_MAX
- 1000;
+ private static final short ACCUMULO_OTHER_SYSTEM_MAX =
ACCUMULO_OTHER_USER_MIN - 1;
+ private static final short ACCUMULO_OTHER_SYSTEM_MIN =
ACCUMULO_OTHER_SYSTEM_MAX - 1000;
+ private static final short USER_USER_MAX = ACCUMULO_OTHER_SYSTEM_MIN - 1;
+ private static final short USER_USER_MIN = (USER_USER_MAX + Short.MIN_VALUE)
/ 2;
private static final short USER_SYSTEM_MAX = USER_USER_MIN - 1;
private static final short USER_SYSTEM_MIN = Short.MIN_VALUE;
@@ -56,8 +62,9 @@ public class CompactionJobPrioritizer {
int min;
int max;
- switch (Ample.DataLevel.of(tableId)) {
- case ROOT:
+ // Check if the table is in the accumulo namespace
+ if (AccumuloTable.allTableIds().contains(tableId)) {
+ if (tableId.equals(AccumuloTable.ROOT.tableId())) {
if (kind == CompactionKind.USER) {
min = ROOT_USER_MIN;
max = ROOT_USER_MAX;
@@ -65,8 +72,7 @@ public class CompactionJobPrioritizer {
min = ROOT_SYSTEM_MIN;
max = ROOT_SYSTEM_MAX;
}
- break;
- case METADATA:
+ } else if (tableId.equals(AccumuloTable.METADATA.tableId())) {
if (kind == CompactionKind.USER) {
min = METADATA_USER_MIN;
max = METADATA_USER_MAX;
@@ -74,18 +80,24 @@ public class CompactionJobPrioritizer {
min = METADATA_SYSTEM_MIN;
max = METADATA_SYSTEM_MAX;
}
- break;
- case USER:
+ } else {
+ // This is a table in the accumulo namespace that is not root or
metadata table.
if (kind == CompactionKind.USER) {
- min = USER_USER_MIN;
- max = USER_USER_MAX;
+ min = ACCUMULO_OTHER_USER_MIN;
+ max = ACCUMULO_OTHER_USER_MAX;
} else {
- min = USER_SYSTEM_MIN;
- max = USER_SYSTEM_MAX;
+ min = ACCUMULO_OTHER_SYSTEM_MIN;
+ max = ACCUMULO_OTHER_SYSTEM_MAX;
}
- break;
- default:
- throw new IllegalStateException("Unknown data level" +
Ample.DataLevel.of(tableId));
+ }
+ } else {
+ if (kind == CompactionKind.USER) {
+ min = USER_USER_MIN;
+ max = USER_USER_MAX;
+ } else {
+ min = USER_SYSTEM_MIN;
+ max = USER_SYSTEM_MAX;
+ }
}
return (short) Math.min(max, min + totalFiles + compactingFiles);
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
b/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
index 15d96a36f2..2e6a03ef41 100644
---
a/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
@@ -87,11 +87,28 @@ public class CompactionPrioritizerTest {
short pm8 = createPriority(AccumuloTable.METADATA.tableId(),
CompactionKind.SYSTEM, 1, 1);
assertTrue(pm7 > pm8);
+ short pf1 = createPriority(AccumuloTable.FATE.tableId(),
CompactionKind.USER, 10000, 1);
+ assertTrue(pm8 > pf1);
+ short pf2 = createPriority(AccumuloTable.FATE.tableId(),
CompactionKind.USER, 100, 30);
+ assertTrue(pf1 > pf2);
+ short pf3 = createPriority(AccumuloTable.FATE.tableId(),
CompactionKind.USER, 100, 1);
+ assertTrue(pf2 > pf3);
+ short pf4 = createPriority(AccumuloTable.FATE.tableId(),
CompactionKind.USER, 1, 1);
+ assertTrue(pf3 > pf4);
+ short pf5 = createPriority(AccumuloTable.FATE.tableId(),
CompactionKind.SYSTEM, 10000, 1);
+ assertTrue(pf4 > pf5);
+ short pf6 = createPriority(AccumuloTable.FATE.tableId(),
CompactionKind.SYSTEM, 100, 30);
+ assertTrue(pf5 > pf6);
+ short pf7 = createPriority(AccumuloTable.FATE.tableId(),
CompactionKind.SYSTEM, 100, 1);
+ assertTrue(pf6 > pf7);
+ short pf8 = createPriority(AccumuloTable.FATE.tableId(),
CompactionKind.SYSTEM, 1, 1);
+ assertTrue(pm7 > pf8);
+
var userTable1 = TableId.of("1");
var userTable2 = TableId.of("2");
short pu1 = createPriority(userTable1, CompactionKind.USER, 10000, 1);
- assertTrue(pm8 > pu1);
+ assertTrue(pf8 > pu1);
short pu2 = createPriority(userTable2, CompactionKind.USER, 1000, 30);
assertTrue(pu1 > pu2);
short pu3 = createPriority(userTable1, CompactionKind.USER, 1000, 1);
@@ -118,6 +135,8 @@ public class CompactionPrioritizerTest {
short minMetaUser = createPriority(AccumuloTable.METADATA.tableId(),
CompactionKind.USER, 1, 1);
short minMetaSystem =
createPriority(AccumuloTable.METADATA.tableId(),
CompactionKind.SYSTEM, 1, 1);
+ short minFateUser = createPriority(AccumuloTable.FATE.tableId(),
CompactionKind.USER, 1, 1);
+ short minFateSystem = createPriority(AccumuloTable.FATE.tableId(),
CompactionKind.SYSTEM, 1, 1);
short minUserUser = createPriority(userTable, CompactionKind.USER, 1, 1);
// Test the boundary condition around the max number of files to encode.
Ensure the next level
@@ -132,8 +151,13 @@ public class CompactionPrioritizerTest {
short metaSystem =
createPriority(AccumuloTable.METADATA.tableId(),
CompactionKind.SYSTEM, files, 1);
assertTrue(minMetaUser > metaSystem);
+ short fateUser = createPriority(AccumuloTable.FATE.tableId(),
CompactionKind.USER, files, 1);
+ assertTrue(minMetaSystem > fateUser);
+ short fateSystem =
+ createPriority(AccumuloTable.FATE.tableId(), CompactionKind.SYSTEM,
files, 1);
+ assertTrue(minFateUser > fateSystem);
short userUser = createPriority(userTable, CompactionKind.USER, files,
1);
- assertTrue(minMetaSystem > userUser);
+ assertTrue(minFateSystem > userUser);
short userSystem = createPriority(userTable, CompactionKind.SYSTEM,
files, 1);
assertTrue(minUserUser > userSystem);
}
