This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new ddff638 Use numFilesCompacting+totalTabletFiles for compaction
priority (#2330)
ddff638 is described below
commit ddff6381197fd74f352700e13418c090ad3022c9
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Mon Nov 1 09:52:35 2021 -0400
Use numFilesCompacting+totalTabletFiles for compaction priority (#2330)
---
.../spi/compaction/DefaultCompactionPlanner.java | 9 +++++---
.../util/compaction/CompactionJobPrioritizer.java | 13 +++++++-----
.../util/compaction/CompactionPrioritizerTest.java | 24 +++++++++++++---------
3 files changed, 28 insertions(+), 18 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index edb9b37..7046bd8 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@ -303,15 +303,18 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
// determine which executor to use based on the size of the files
var ceid = getExecutor(group);
- return params.createPlanBuilder().addJob(createPriority(params), ceid,
group).build();
+ return params.createPlanBuilder().addJob(createPriority(params,
group), ceid, group)
+ .build();
}
} catch (RuntimeException e) {
throw e;
}
}
- private static short createPriority(PlanningParameters params) {
- return CompactionJobPrioritizer.createPriority(params.getKind(),
params.getAll().size());
+ private static short createPriority(PlanningParameters params,
+ Collection<CompactableFile> group) {
+ return CompactionJobPrioritizer.createPriority(params.getKind(),
params.getAll().size(),
+ group.size());
}
private long getMaxSizeToCompact(CompactionKind kind) {
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
index ddd3a0b..efc3e22 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
@@ -30,25 +30,28 @@ public class CompactionJobPrioritizer {
Comparator.comparingInt(CompactionJob::getPriority)
.thenComparingInt(job -> job.getFiles().size()).reversed();
- public static short createPriority(CompactionKind kind, int totalFiles) {
+ public static short createPriority(CompactionKind kind, int totalFiles, int
compactingFiles) {
+
+ int prio = totalFiles + compactingFiles;
+
switch (kind) {
case USER:
case CHOP:
// user-initiated compactions will have a positive priority
// based on number of files
- if (totalFiles > Short.MAX_VALUE) {
+ if (prio > Short.MAX_VALUE) {
return Short.MAX_VALUE;
}
- return (short) totalFiles;
+ return (short) prio;
case SELECTOR:
case SYSTEM:
// system-initiated compactions will have a negative priority
// starting at -32768 and increasing based on number of files
// maxing out at -1
- if (totalFiles > Short.MAX_VALUE) {
+ if (prio > Short.MAX_VALUE) {
return -1;
} else {
- return (short) (Short.MIN_VALUE + totalFiles);
+ return (short) (Short.MIN_VALUE + prio);
}
default:
throw new AssertionError("Unknown kind " + kind);
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
b/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
index 5ff5fb3..a3299e7 100644
---
a/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
@@ -42,26 +42,30 @@ public class CompactionPrioritizerTest {
files.add(CompactableFile
.create(URI.create("hdfs://foonn/accumulo/tables/5/" + tablet + "/"
+ i + ".rf"), 4, 4));
}
- return new CompactionJobImpl(CompactionJobPrioritizer.createPriority(kind,
totalFiles),
+ // TODO pass numFiles
+ return new CompactionJobImpl(
+ CompactionJobPrioritizer.createPriority(kind, totalFiles, numFiles),
CompactionExecutorIdImpl.externalId("test"), files, kind,
Optional.of(false));
}
@Test
public void testPrioritizer() throws Exception {
- assertEquals((short) 0,
CompactionJobPrioritizer.createPriority(CompactionKind.USER, 0));
+ assertEquals((short) 0,
CompactionJobPrioritizer.createPriority(CompactionKind.USER, 0, 0));
assertEquals((short) 10000,
- CompactionJobPrioritizer.createPriority(CompactionKind.USER, 10000));
+ CompactionJobPrioritizer.createPriority(CompactionKind.USER, 10000,
0));
assertEquals((short) 32767,
- CompactionJobPrioritizer.createPriority(CompactionKind.USER, 32767));
+ CompactionJobPrioritizer.createPriority(CompactionKind.USER, 32767,
0));
assertEquals((short) 32767,
- CompactionJobPrioritizer.createPriority(CompactionKind.USER,
Integer.MAX_VALUE));
+ CompactionJobPrioritizer.createPriority(CompactionKind.USER,
Integer.MAX_VALUE, 0));
- assertEquals((short) -32768,
CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM, 0));
+ assertEquals((short) -32768,
+ CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM, 0, 0));
assertEquals((short) -22768,
- CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM, 10000));
- assertEquals((short) -1,
CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM, 32767));
+ CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM, 10000,
0));
assertEquals((short) -1,
- CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM,
Integer.MAX_VALUE));
+ CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM, 32767,
0));
+ assertEquals((short) -1,
+ CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM,
Integer.MAX_VALUE, 0));
}
@Test
@@ -74,7 +78,7 @@ public class CompactionPrioritizerTest {
var j6 = createJob(CompactionKind.CHOP, "t-014", 5, 40);
var j7 = createJob(CompactionKind.CHOP, "t-015", 5, 7);
var j8 = createJob(CompactionKind.SELECTOR, "t-014", 5, 21);
- var j9 = createJob(CompactionKind.SELECTOR, "t-015", 7, 21);
+ var j9 = createJob(CompactionKind.SELECTOR, "t-015", 7, 20);
var expected = List.of(j6, j2, j3, j1, j7, j4, j9, j8, j5);
