This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 3.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit f6c0da3f72af01ded77e743c03bac60fd5aaa3e1
Merge: 51633db8df ba9c376ecc
Author: Dave Marion <dlmar...@apache.org>
AuthorDate: Fri Nov 8 20:16:45 2024 +0000
Merge branch '2.1' into 3.1
.../core/spi/compaction/CompactionPlanner.java | 9 +
.../spi/compaction/DefaultCompactionPlanner.java | 21 ++-
.../util/compaction/CompactionJobPrioritizer.java | 143 +++++++++++++---
.../compaction/DefaultCompactionPlannerTest.java | 8 +
.../util/compaction/CompactionPrioritizerTest.java | 185 ++++++++++++++++++---
.../tserver/compactions/CompactionService.java | 7 +
6 files changed, 324 insertions(+), 49 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index ee07bc2fc0,6c60d5095c..7455625034
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@@ -32,8 -28,8 +32,9 @@@ import java.util.HashSet
import java.util.List;
import java.util.Objects;
import java.util.Set;
+import java.util.stream.Collectors;
+ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.Property;
@@@ -297,120 -233,121 +298,128 @@@ public class DefaultCompactionPlanner i
determineMaxFilesToCompact(params);
}
- @SuppressWarnings("removal")
+ @SuppressWarnings("deprecation")
private void determineMaxFilesToCompact(InitParameters params) {
- String fqo = params.getFullyQualifiedOption("maxOpen");
- if (!params.getServiceEnvironment().getConfiguration().isSet(fqo)
- && params.getServiceEnvironment().getConfiguration()
- .isSet(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey())) {
- log.warn("The property " + Property.TSERV_MAJC_THREAD_MAXOPEN.getKey()
- + " was set, it is deprecated. Set the " + fqo + " option
instead.");
- this.maxFilesToCompact =
Integer.parseInt(params.getServiceEnvironment().getConfiguration()
- .get(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey()));
- } else {
- this.maxFilesToCompact =
Integer.parseInt(params.getOptions().getOrDefault("maxOpen",
-
Property.TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN.getDefaultValue()));
+
+ String maxOpen = params.getOptions().get("maxOpen");
+ if (maxOpen == null) {
+ maxOpen =
Property.TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN.getDefaultValue();
+ log.trace("default maxOpen not set, defaulting to {}", maxOpen);
}
+ this.maxFilesToCompact = Integer.parseInt(maxOpen);
}
- @Override
- public CompactionPlan makePlan(PlanningParameters params) {
- int maxTabletFiles = 0;
- try {
+ private void validateConfig(JsonElement json, List<String> fields, String
className) {
- if (params.getCandidates().isEmpty()) {
- return params.createPlanBuilder().build();
- }
+ JsonObject jsonObject = GSON.get().fromJson(json, JsonObject.class);
- Set<CompactableFile> filesCopy = new HashSet<>(params.getCandidates());
+ List<String> objectProperties = new ArrayList<>(jsonObject.keySet());
+ HashSet<String> classFieldNames = new HashSet<>(fields);
- long maxSizeToCompact = getMaxSizeToCompact(params.getKind());
+ if (!classFieldNames.containsAll(objectProperties)) {
+ objectProperties.removeAll(classFieldNames);
+ throw new JsonParseException(
+ "Invalid fields: " + objectProperties + " provided for class: " +
className);
+ }
+ }
- Collection<CompactableFile> group;
- if (params.getRunningCompactions().isEmpty()) {
- group = findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact,
- maxSizeToCompact);
+ @Override
+ public CompactionPlan makePlan(PlanningParameters params) {
+ if (params.getCandidates().isEmpty()) {
+ return params.createPlanBuilder().build();
+ }
+
+ Set<CompactableFile> filesCopy = new HashSet<>(params.getCandidates());
- if (!group.isEmpty() && group.size() < params.getCandidates().size()
- && params.getCandidates().size() <= maxFilesToCompact
- && (params.getKind() == CompactionKind.USER
- || params.getKind() == CompactionKind.SELECTOR)) {
- // USER and SELECTOR compactions must eventually compact all files.
When a subset of files
- // that meets the compaction ratio is selected, look ahead and see
if the next compaction
- // would also meet the compaction ratio. If not then compact
everything to avoid doing
- // more than logarithmic work across multiple comapctions.
+ long maxSizeToCompact = getMaxSizeToCompact(params.getKind());
- filesCopy.removeAll(group);
- filesCopy.add(getExpected(group, 0));
+ Collection<CompactableFile> group;
+ if (params.getRunningCompactions().isEmpty()) {
+ group =
+ findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact, maxSizeToCompact);
- if (findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact,
- maxSizeToCompact).isEmpty()) {
- // The next possible compaction does not meet the compaction
ratio, so compact
- // everything.
- group = Set.copyOf(params.getCandidates());
- }
+ if (!group.isEmpty() && group.size() < params.getCandidates().size()
+ && params.getCandidates().size() <= maxFilesToCompact
+ && (params.getKind() == CompactionKind.USER
+ || params.getKind() == DeprecatedCompactionKind.SELECTOR)) {
+ // USER and SELECTOR compactions must eventually compact all files.
When a subset of files
+ // that meets the compaction ratio is selected, look ahead and see if
the next compaction
+ // would also meet the compaction ratio. If not then compact
everything to avoid doing
+ // more than logarithmic work across multiple comapctions.
+ filesCopy.removeAll(group);
+ filesCopy.add(getExpected(group, 0));
+
+ if (findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact,
+ maxSizeToCompact).isEmpty()) {
+ // The next possible compaction does not meet the compaction ratio,
so compact
+ // everything.
+ group = Set.copyOf(params.getCandidates());
}
- } else if (params.getKind() == CompactionKind.SYSTEM) {
- // This code determines if once the files compacting finish would
they be included in a
- // compaction with the files smaller than them? If so, then wait for
the running compaction
- // to complete.
+ }
- // The set of files running compactions may produce
- var expectedFiles = getExpected(params.getRunningCompactions());
+ } else if (params.getKind() == CompactionKind.SYSTEM) {
+ // This code determines if once the files compacting finish would they
be included in a
+ // compaction with the files smaller than them? If so, then wait for
the running compaction
+ // to complete.
- if (!Collections.disjoint(filesCopy, expectedFiles)) {
- throw new AssertionError();
- }
+ // The set of files running compactions may produce
+ var expectedFiles = getExpected(params.getRunningCompactions());
- filesCopy.addAll(expectedFiles);
+ if (!Collections.disjoint(filesCopy, expectedFiles)) {
+ throw new AssertionError();
+ }
- group = findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact,
- maxSizeToCompact);
+ filesCopy.addAll(expectedFiles);
- if (!Collections.disjoint(group, expectedFiles)) {
- // file produced by running compaction will eventually compact with
existing files, so
- // wait.
- group = Set.of();
- }
- } else {
+ group =
+ findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact, maxSizeToCompact);
+
+ if (!Collections.disjoint(group, expectedFiles)) {
+ // file produced by running compaction will eventually compact with
existing files, so
+ // wait.
group = Set.of();
}
+ } else {
+ group = Set.of();
+ }
- if (group.isEmpty()) {
-
- if ((params.getKind() == CompactionKind.USER || params.getKind() ==
CompactionKind.SELECTOR
- || params.getKind() == CompactionKind.CHOP)
- && params.getRunningCompactions().stream()
- .noneMatch(job -> job.getKind() == params.getKind())) {
- group = findMaximalRequiredSetToCompact(params.getCandidates(),
maxFilesToCompact);
- } else if (params.getKind() == CompactionKind.SYSTEM
- && params.getRunningCompactions().isEmpty()
- && params.getAll().size() == params.getCandidates().size()) {
- maxTabletFiles = getMaxTabletFiles(
-
params.getServiceEnvironment().getConfiguration(params.getTableId()));
- if (params.getAll().size() > maxTabletFiles) {
- // The tablet is above its max files, there are no compactions
running, all files are
- // candidates for a system compaction, and no files were found to
compact. Attempt to
- // find a set of files to compact by lowering the compaction
ratio.
- group = findFilesToCompactWithLowerRatio(params,
maxSizeToCompact, maxTabletFiles);
- }
++ int maxTabletFiles = 0;
+ if (group.isEmpty()) {
+ if ((params.getKind() == CompactionKind.USER
+ || params.getKind() == DeprecatedCompactionKind.SELECTOR)
+ && params.getRunningCompactions().stream()
+ .noneMatch(job -> job.getKind() == params.getKind())) {
+ group = findMaximalRequiredSetToCompact(params.getCandidates(),
maxFilesToCompact);
+ } else if (params.getKind() == CompactionKind.SYSTEM
+ && params.getRunningCompactions().isEmpty()
+ && params.getAll().size() == params.getCandidates().size()) {
- int maxTabletFiles =
++ maxTabletFiles =
+
getMaxTabletFiles(params.getServiceEnvironment().getConfiguration(params.getTableId()));
+ if (params.getAll().size() > maxTabletFiles) {
+ // The tablet is above its max files, there are no compactions
running, all files are
+ // candidates for a system compaction, and no files were found to
compact. Attempt to
+ // find a set of files to compact by lowering the compaction ratio.
+ group = findFilesToCompactWithLowerRatio(params, maxSizeToCompact,
maxTabletFiles);
}
}
+ }
- if (group.isEmpty()) {
- return params.createPlanBuilder().build();
- } else {
- // determine which executor to use based on the size of the files
- var ceid = getExecutor(group);
+ if (group.isEmpty()) {
+ return params.createPlanBuilder().build();
+ } else {
+ // determine which executor to use based on the size of the files
+ var ceid = getExecutor(group);
- return params.createPlanBuilder().addJob(createPriority(params, group),
ceid, group).build();
++ try {
+ return params.createPlanBuilder()
+ .addJob(createPriority(params, group, maxTabletFiles), ceid,
group).build();
++ } catch (RuntimeException e) {
++ throw e;
++ } catch (TableNotFoundException e) {
++ throw new RuntimeException("Error getting namespace for table: " +
params.getTableId(), e);
+ }
- } catch (RuntimeException e) {
- throw e;
- } catch (TableNotFoundException e) {
- throw new RuntimeException("Error getting namespace for table: " +
params.getTableId(), e);
}
}
diff --cc
core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
index 039201b7c0,686e589f5c..608027cb3f
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
@@@ -19,9 -19,22 +19,21 @@@
package org.apache.accumulo.core.util.compaction;
import java.util.Comparator;
+ import java.util.HashMap;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.function.Function;
+ import org.apache.accumulo.core.clientImpl.Namespace;
+ import org.apache.accumulo.core.data.NamespaceId;
+ import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
++import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
+ import org.apache.accumulo.core.util.Pair;
+ import org.apache.commons.lang3.Range;
+
+ import com.google.common.base.Preconditions;
public class CompactionJobPrioritizer {
@@@ -29,32 -42,113 +41,115 @@@
Comparator.comparingInt(CompactionJob::getPriority)
.thenComparingInt(job -> job.getFiles().size()).reversed();
+ private static final Map<Pair<TableId,CompactionKind>,Range<Short>>
SYSTEM_TABLE_RANGES =
+ new HashMap<>();
+ private static final Map<Pair<NamespaceId,CompactionKind>,
+ Range<Short>> ACCUMULO_NAMESPACE_RANGES = new HashMap<>();
+
+ // Create ranges of possible priority values where each range has
+ // 2000 possible values. Priority order is:
+ // root table user initiated
+ // root table system initiated
+ // metadata table user initiated
+ // metadata table system initiated
+ // other tables in accumulo namespace user initiated
+ // other tables in accumulo namespace system initiated
+ // user tables that have more files that configured system initiated
+ // user tables user initiated
+ // user tables system initiated
+ static final Range<Short> ROOT_TABLE_USER = Range.of((short) 30768, (short)
32767);
+ static final Range<Short> ROOT_TABLE_SYSTEM = Range.of((short) 28768,
(short) 30767);
+
+ static final Range<Short> METADATA_TABLE_USER = Range.of((short) 26768,
(short) 28767);
+ static final Range<Short> METADATA_TABLE_SYSTEM = Range.of((short) 24768,
(short) 26767);
+
+ static final Range<Short> SYSTEM_NS_USER = Range.of((short) 22768, (short)
24767);
+ static final Range<Short> SYSTEM_NS_SYSTEM = Range.of((short) 20768,
(short) 22767);
+
+ static final Range<Short> TABLE_OVER_SIZE = Range.of((short) 18768, (short)
20767);
+
+ static final Range<Short> USER_TABLE_USER = Range.of((short) 1, (short)
18767);
+ static final Range<Short> USER_TABLE_SYSTEM = Range.of((short) -32768,
(short) 0);
+
+ static {
+ // root table
- SYSTEM_TABLE_RANGES.put(new Pair<>(RootTable.ID, CompactionKind.USER),
ROOT_TABLE_USER);
- SYSTEM_TABLE_RANGES.put(new Pair<>(RootTable.ID, CompactionKind.SYSTEM),
ROOT_TABLE_SYSTEM);
++ SYSTEM_TABLE_RANGES.put(new Pair<>(AccumuloTable.ROOT.tableId(),
CompactionKind.USER),
++ ROOT_TABLE_USER);
++ SYSTEM_TABLE_RANGES.put(new Pair<>(AccumuloTable.ROOT.tableId(),
CompactionKind.SYSTEM),
++ ROOT_TABLE_SYSTEM);
+
+ // metadata table
- SYSTEM_TABLE_RANGES.put(new Pair<>(MetadataTable.ID,
CompactionKind.USER), METADATA_TABLE_USER);
- SYSTEM_TABLE_RANGES.put(new Pair<>(MetadataTable.ID,
CompactionKind.SYSTEM),
++ SYSTEM_TABLE_RANGES.put(new Pair<>(AccumuloTable.METADATA.tableId(),
CompactionKind.USER),
++ METADATA_TABLE_USER);
++ SYSTEM_TABLE_RANGES.put(new Pair<>(AccumuloTable.METADATA.tableId(),
CompactionKind.SYSTEM),
+ METADATA_TABLE_SYSTEM);
+
+ // metadata table
+ ACCUMULO_NAMESPACE_RANGES.put(new Pair<>(Namespace.ACCUMULO.id(),
CompactionKind.USER),
+ SYSTEM_NS_USER);
+ ACCUMULO_NAMESPACE_RANGES.put(new Pair<>(Namespace.ACCUMULO.id(),
CompactionKind.SYSTEM),
+ SYSTEM_NS_SYSTEM);
+ }
+
+ @SuppressWarnings("deprecation")
- public static short createPriority(CompactionKind kind, int totalFiles, int
compactingFiles) {
-
- int prio = totalFiles + compactingFiles;
-
- switch (kind) {
- case USER:
- // user-initiated compactions will have a positive priority
- // based on number of files
- if (prio > Short.MAX_VALUE) {
- return Short.MAX_VALUE;
- }
- return (short) prio;
- case SELECTOR:
- case SYSTEM:
- // system-initiated compactions will have a negative priority
- // starting at -32768 and increasing based on number of files
- // maxing out at -1
- if (prio > Short.MAX_VALUE) {
- return -1;
- } else {
- return (short) (Short.MIN_VALUE + prio);
- }
- default:
- throw new AssertionError("Unknown kind " + kind);
+ public static short createPriority(final NamespaceId nsId, final TableId
tableId,
+ final CompactionKind kind, final int totalFiles, final int
compactingFiles,
+ final int maxFilesPerTablet) {
+
+ Objects.requireNonNull(nsId, "nsId cannot be null");
+ Objects.requireNonNull(tableId, "tableId cannot be null");
+ Preconditions.checkArgument(totalFiles >= 0, "totalFiles is negative %s",
totalFiles);
+ Preconditions.checkArgument(compactingFiles >= 0, "compactingFiles is
negative %s",
+ compactingFiles);
+
+ final Function<Range<Short>,Short> normalPriorityFunction = new
Function<>() {
+ @Override
+ public Short apply(Range<Short> f) {
+ return (short) Math.min(f.getMaximum(), f.getMinimum() + totalFiles +
compactingFiles);
+ }
+ };
+
+ final Function<Range<Short>,Short> tabletOverSizeFunction = new
Function<>() {
+ @Override
+ public Short apply(Range<Short> f) {
+ return (short) Math.min(f.getMaximum(),
+ f.getMinimum() + compactingFiles + (totalFiles -
maxFilesPerTablet));
+ }
+ };
+
+ // Handle the case of a CHOP compaction. For the purposes of determining
+ // a priority, treat them as a USER compaction.
+ CompactionKind calculationKind = kind;
- if (kind == CompactionKind.CHOP) {
- calculationKind = CompactionKind.USER;
- } else if (kind == CompactionKind.SELECTOR) {
++ if (kind == CompactionKind.SELECTOR) {
+ calculationKind = CompactionKind.SYSTEM;
}
+
+ Range<Short> range = null;
+ Function<Range<Short>,Short> func = normalPriorityFunction;
+ if (Namespace.ACCUMULO.id() == nsId) {
+ // Handle system tables
+ range = SYSTEM_TABLE_RANGES.get(new Pair<>(tableId, calculationKind));
+ if (range == null) {
+ range = ACCUMULO_NAMESPACE_RANGES.get(new Pair<>(nsId,
calculationKind));
+ }
+ } else {
+ // Handle user tables
+ if (totalFiles > maxFilesPerTablet && calculationKind ==
CompactionKind.SYSTEM) {
+ range = TABLE_OVER_SIZE;
+ func = tabletOverSizeFunction;
+ } else if (calculationKind == CompactionKind.SYSTEM) {
+ range = USER_TABLE_SYSTEM;
+ } else {
+ range = USER_TABLE_USER;
+ }
+ }
+
+ if (range == null) {
+ throw new IllegalStateException(
+ "Error calculating compaction priority for table: " + tableId);
+ }
+ return func.apply(range);
+
}
}
diff --cc
core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
index e0c32dde6f,f3186701ad..55cf3e1f97
---
a/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
@@@ -36,9 -35,15 +36,12 @@@ import java.util.Optional
import java.util.Set;
import java.util.stream.Collectors;
+ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
+ import org.apache.accumulo.core.clientImpl.Namespace;
-import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
-import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.conf.SiteConfiguration;
+ import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.spi.common.ServiceEnvironment;
import org.apache.accumulo.core.spi.common.ServiceEnvironment.Configuration;
diff --cc
core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
index 5eaad1df0b,1aa62c4f6b..3720890f15
---
a/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
@@@ -28,8 -40,13 +40,12 @@@ import java.util.List
import java.util.Optional;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
+ import org.apache.accumulo.core.clientImpl.Namespace;
+ import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
++import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
+ import org.apache.commons.lang3.Range;
import org.junit.jupiter.api.Test;
public class CompactionPrioritizerTest {
@@@ -48,23 -67,161 +66,152 @@@
}
@Test
- public void testPrioritizer() throws Exception {
- assertEquals((short) 0,
CompactionJobPrioritizer.createPriority(CompactionKind.USER, 0, 0));
- assertEquals((short) 10000,
- CompactionJobPrioritizer.createPriority(CompactionKind.USER, 10000,
0));
- assertEquals((short) 32767,
- CompactionJobPrioritizer.createPriority(CompactionKind.USER, 32767,
0));
- assertEquals((short) 32767,
- CompactionJobPrioritizer.createPriority(CompactionKind.USER,
Integer.MAX_VALUE, 0));
-
- assertEquals((short) -32768,
- CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM, 0, 0));
- assertEquals((short) -22768,
- CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM, 10000,
0));
- assertEquals((short) -1,
- CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM, 32767,
0));
- assertEquals((short) -1,
- CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM,
Integer.MAX_VALUE, 0));
+ public void testNonOverlappingRanges() {
+ List<Range<Short>> ranges = new ArrayList<>();
+ ranges.add(ROOT_TABLE_USER);
+ ranges.add(ROOT_TABLE_SYSTEM);
+ ranges.add(METADATA_TABLE_USER);
+ ranges.add(METADATA_TABLE_SYSTEM);
+ ranges.add(SYSTEM_NS_USER);
+ ranges.add(SYSTEM_NS_SYSTEM);
+ ranges.add(TABLE_OVER_SIZE);
+ ranges.add(USER_TABLE_USER);
+ ranges.add(USER_TABLE_SYSTEM);
+
+ for (Range<Short> r1 : ranges) {
+ for (Range<Short> r2 : ranges) {
+ if (r1 == r2) {
+ continue;
+ }
+ assertFalse(r1.isOverlappedBy(r2), r1.toString() + " is overlapped by
" + r2.toString());
+ }
+ }
+
+ Collections.sort(ranges, new Comparator<Range<Short>>() {
+ @Override
+ public int compare(Range<Short> r1, Range<Short> r2) {
+ return Short.compare(r1.getMinimum(), r2.getMinimum());
+ }
+ });
+ assertEquals(Short.MIN_VALUE, ranges.get(0).getMinimum());
+ assertEquals(Short.MAX_VALUE, ranges.get(ranges.size() - 1).getMaximum());
+ // check that the max of the previous range is one less than the
+ // minimum of the current range to make sure there are no holes.
+ short lastMax = Short.MIN_VALUE;
+ for (Range<Short> r : ranges) {
+ if (lastMax != Short.MIN_VALUE) {
+ assertTrue(r.getMinimum() - lastMax == 1);
+ }
+ lastMax = r.getMaximum();
+ }
+ }
+
+ @Test
+ public void testRootTablePriorities() {
- assertEquals(ROOT_TABLE_USER.getMinimum() + 1,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.USER, 0, 1,
TABLET_FILE_MAX));
- assertEquals(ROOT_TABLE_USER.getMinimum() + 1010,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.USER, 1000, 10,
TABLET_FILE_MAX));
- assertEquals(ROOT_TABLE_USER.getMaximum(),
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.USER, 3000,
100, TABLET_FILE_MAX));
-
- assertEquals(ROOT_TABLE_USER.getMinimum() + 2,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.CHOP, 0, 2,
TABLET_FILE_MAX));
- assertEquals(ROOT_TABLE_USER.getMinimum() + 1020,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.CHOP, 1000, 20,
TABLET_FILE_MAX));
- assertEquals(ROOT_TABLE_USER.getMaximum(),
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.CHOP, 3000,
200, TABLET_FILE_MAX));
-
- assertEquals(ROOT_TABLE_SYSTEM.getMinimum() + 3,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.SYSTEM, 0, 3,
TABLET_FILE_MAX));
- assertEquals(ROOT_TABLE_SYSTEM.getMinimum() + 1030,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.SYSTEM, 1000,
30, TABLET_FILE_MAX));
- assertEquals(ROOT_TABLE_SYSTEM.getMaximum(),
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.SYSTEM, 3000,
300, TABLET_FILE_MAX));
++ assertEquals(ROOT_TABLE_USER.getMinimum() + 1,
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.ROOT.tableId(), CompactionKind.USER, 0, 1,
TABLET_FILE_MAX));
++ assertEquals(ROOT_TABLE_USER.getMinimum() + 1010,
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.ROOT.tableId(), CompactionKind.USER, 1000, 10,
TABLET_FILE_MAX));
++ assertEquals(ROOT_TABLE_USER.getMaximum(),
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.ROOT.tableId(), CompactionKind.USER, 3000, 100,
TABLET_FILE_MAX));
++
++ assertEquals(ROOT_TABLE_SYSTEM.getMinimum() + 3,
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.ROOT.tableId(), CompactionKind.SYSTEM, 0, 3,
TABLET_FILE_MAX));
++ assertEquals(ROOT_TABLE_SYSTEM.getMinimum() + 1030,
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.ROOT.tableId(), CompactionKind.SYSTEM, 1000, 30,
TABLET_FILE_MAX));
++ assertEquals(ROOT_TABLE_SYSTEM.getMaximum(),
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.ROOT.tableId(), CompactionKind.SYSTEM, 3000, 300,
TABLET_FILE_MAX));
+ }
+
+ @Test
+ public void testMetaTablePriorities() {
- assertEquals(METADATA_TABLE_USER.getMinimum() + 4,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.USER, 0, 4,
TABLET_FILE_MAX));
- assertEquals(METADATA_TABLE_USER.getMinimum() + 1040,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.USER, 1000,
40, TABLET_FILE_MAX));
++ assertEquals(METADATA_TABLE_USER.getMinimum() + 4,
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.METADATA.tableId(), CompactionKind.USER, 0, 4,
TABLET_FILE_MAX));
++ assertEquals(METADATA_TABLE_USER.getMinimum() + 1040,
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.METADATA.tableId(), CompactionKind.USER, 1000, 40,
TABLET_FILE_MAX));
+ assertEquals(METADATA_TABLE_USER.getMaximum(),
- CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
MetadataTable.ID,
- CompactionKind.USER, 3000, 400, TABLET_FILE_MAX));
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.METADATA.tableId(), CompactionKind.USER, 3000, 400,
TABLET_FILE_MAX));
+
- assertEquals(METADATA_TABLE_USER.getMinimum() + 5,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.CHOP, 0, 5,
TABLET_FILE_MAX));
- assertEquals(METADATA_TABLE_USER.getMinimum() + 1050,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.CHOP, 1000,
50, TABLET_FILE_MAX));
- assertEquals(METADATA_TABLE_USER.getMaximum(),
- CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
MetadataTable.ID,
- CompactionKind.CHOP, 3000, 500, TABLET_FILE_MAX));
-
- assertEquals(METADATA_TABLE_SYSTEM.getMinimum() + 6,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.SYSTEM, 0,
6, TABLET_FILE_MAX));
++ assertEquals(METADATA_TABLE_SYSTEM.getMinimum() + 6,
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.METADATA.tableId(), CompactionKind.SYSTEM, 0, 6,
TABLET_FILE_MAX));
+ assertEquals(METADATA_TABLE_SYSTEM.getMinimum() + 1060,
- CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
MetadataTable.ID,
- CompactionKind.SYSTEM, 1000, 60, TABLET_FILE_MAX));
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.METADATA.tableId(), CompactionKind.SYSTEM, 1000,
60, TABLET_FILE_MAX));
+ assertEquals(METADATA_TABLE_SYSTEM.getMaximum(),
- CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
MetadataTable.ID,
- CompactionKind.SYSTEM, 3000, 600, TABLET_FILE_MAX));
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.METADATA.tableId(), CompactionKind.SYSTEM, 3000,
600, TABLET_FILE_MAX));
+ }
+
+ @Test
+ public void testSystemNamespacePriorities() {
+ TableId tid = TableId.of("someOtherSystemTable");
+ assertEquals(SYSTEM_NS_USER.getMinimum() + 7, CompactionJobPrioritizer
+ .createPriority(Namespace.ACCUMULO.id(), tid, CompactionKind.USER, 0,
7, TABLET_FILE_MAX));
+ assertEquals(SYSTEM_NS_USER.getMinimum() + 1070,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.USER, 1000, 70,
TABLET_FILE_MAX));
+ assertEquals(SYSTEM_NS_USER.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.USER, 3000, 700,
TABLET_FILE_MAX));
+
- assertEquals(SYSTEM_NS_USER.getMinimum() + 8, CompactionJobPrioritizer
- .createPriority(Namespace.ACCUMULO.id(), tid, CompactionKind.CHOP, 0,
8, TABLET_FILE_MAX));
- assertEquals(SYSTEM_NS_USER.getMinimum() + 1080,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), tid, CompactionKind.CHOP, 1000, 80,
TABLET_FILE_MAX));
- assertEquals(SYSTEM_NS_USER.getMaximum(),
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), tid, CompactionKind.CHOP, 3000, 800,
TABLET_FILE_MAX));
-
+ assertEquals(SYSTEM_NS_SYSTEM.getMinimum() + 9,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.SYSTEM, 0, 9,
TABLET_FILE_MAX));
+ assertEquals(SYSTEM_NS_SYSTEM.getMinimum() + 1090,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.SYSTEM, 1000, 90,
TABLET_FILE_MAX));
+ assertEquals(SYSTEM_NS_SYSTEM.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.SYSTEM, 3000, 900,
TABLET_FILE_MAX));
+ }
+
+ @Test
+ public void testUserTablePriorities() {
+ TableId tid = TableId.of("someUserTable");
+ assertEquals(USER_TABLE_USER.getMinimum() + 10, CompactionJobPrioritizer
+ .createPriority(Namespace.DEFAULT.id(), tid, CompactionKind.USER, 0,
10, TABLET_FILE_MAX));
+ assertEquals(USER_TABLE_USER.getMinimum() + 1100,
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.USER, 1000, 100,
TABLET_FILE_MAX));
+ assertEquals(USER_TABLE_USER.getMinimum() + 4000,
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.USER, 3000, 1000,
TABLET_FILE_MAX));
+
+ assertEquals(USER_TABLE_SYSTEM.getMinimum() + 11,
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.SYSTEM, 0, 11,
TABLET_FILE_MAX));
+ assertEquals(USER_TABLE_SYSTEM.getMinimum() + 1110,
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.SYSTEM, 1000, 110,
TABLET_FILE_MAX));
+ assertEquals(USER_TABLE_SYSTEM.getMinimum() + 4100,
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.SYSTEM, 3000, 1100,
TABLET_FILE_MAX));
+ }
+
+ @Test
+ public void testTableOverSize() {
+ final int tabletFileMax = 30;
+ final TableId tid = TableId.of("someTable");
- assertEquals(ROOT_TABLE_SYSTEM.getMinimum() + 150,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.SYSTEM, 100,
50, tabletFileMax));
- assertEquals(METADATA_TABLE_SYSTEM.getMinimum() + 150,
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.SYSTEM,
100, 50, tabletFileMax));
++ assertEquals(ROOT_TABLE_SYSTEM.getMinimum() + 150,
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.ROOT.tableId(), CompactionKind.SYSTEM, 100, 50,
tabletFileMax));
++ assertEquals(METADATA_TABLE_SYSTEM.getMinimum() + 150,
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.METADATA.tableId(), CompactionKind.SYSTEM, 100, 50,
tabletFileMax));
+ assertEquals(SYSTEM_NS_SYSTEM.getMinimum() + 150,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.SYSTEM, 100, 50,
tabletFileMax));
+ assertEquals(TABLE_OVER_SIZE.getMinimum() + 120,
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.SYSTEM, 100, 50,
tabletFileMax));
- assertEquals(ROOT_TABLE_SYSTEM.getMaximum(),
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.SYSTEM, 3000,
50, tabletFileMax));
- assertEquals(METADATA_TABLE_SYSTEM.getMaximum(),
CompactionJobPrioritizer.createPriority(
- Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.SYSTEM,
3000, 50, tabletFileMax));
++ assertEquals(ROOT_TABLE_SYSTEM.getMaximum(),
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.ROOT.tableId(), CompactionKind.SYSTEM, 3000, 50,
tabletFileMax));
++ assertEquals(METADATA_TABLE_SYSTEM.getMaximum(),
++ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
++ AccumuloTable.METADATA.tableId(), CompactionKind.SYSTEM, 3000,
50, tabletFileMax));
+ assertEquals(SYSTEM_NS_SYSTEM.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.SYSTEM, 3000, 50,
tabletFileMax));
+ assertEquals(TABLE_OVER_SIZE.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.SYSTEM, 3000, 50,
tabletFileMax));
}
@Test
