This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 031b9c24b7 Fix DefaultCompactionPlanner SPI
031b9c24b7 is described below
commit 031b9c24b7781e0f54dccc57ed484ad26af4b1e4
Author: Christopher Tubbs <ctubb...@apache.org>
AuthorDate: Wed May 10 09:44:34 2023 -0400
Fix DefaultCompactionPlanner SPI
Backport some changes from PR #3378 for 3.0 into 2.1 to fix issue #3377
Specifically, backport the changes to DefaultCompactionPlanner (its unit
test) to make private some of its internal implementation methods that
were unintentionally left public. This also includes the name changes to
those methods that were part of PR #3378, to reduce merge conflicts.
This fixes #3377
---
.../core/spi/compaction/DefaultCompactionPlanner.java | 18 +++++++++---------
.../spi/compaction/DefaultCompactionPlannerTest.java | 2 +-
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index 040296e140..4657108cc5 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@ -115,7 +115,7 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
private static final Logger log =
LoggerFactory.getLogger(DefaultCompactionPlanner.class);
- public static class ExecutorConfig {
+ private static class ExecutorConfig {
String type;
String name;
String maxSize;
@@ -238,7 +238,7 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
Collection<CompactableFile> group;
if (params.getRunningCompactions().isEmpty()) {
- group = findMapFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact,
+ group = findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact,
maxSizeToCompact);
if (!group.isEmpty() && group.size() < params.getCandidates().size()
@@ -253,7 +253,7 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
filesCopy.removeAll(group);
filesCopy.add(getExpected(group, 0));
- if (findMapFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact,
+ if (findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact,
maxSizeToCompact).isEmpty()) {
// The next possible compaction does not meet the compaction
ratio, so compact
// everything.
@@ -276,7 +276,7 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
filesCopy.addAll(expectedFiles);
- group = findMapFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact,
+ group = findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact,
maxSizeToCompact);
if (!Collections.disjoint(group, expectedFiles)) {
@@ -354,7 +354,7 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
return expected;
}
- public static Collection<CompactableFile>
+ private static Collection<CompactableFile>
findMaximalRequiredSetToCompact(Collection<CompactableFile> files, int
maxFilesToCompact) {
if (files.size() <= maxFilesToCompact) {
@@ -373,7 +373,7 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
return sortedFiles.subList(0, numToCompact);
}
- public static Collection<CompactableFile>
findMapFilesToCompact(Set<CompactableFile> files,
+ static Collection<CompactableFile>
findDataFilesToCompact(Set<CompactableFile> files,
double ratio, int maxFilesToCompact, long maxSizeToCompact) {
if (files.size() <= 1) {
return Collections.emptySet();
@@ -401,7 +401,7 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
var loops = Math.max(1, sortedFiles.size() - maxFilesToCompact + 1);
for (int i = 0; i < loops; i++) {
- var filesToCompact = findMapFilesToCompact(
+ var filesToCompact = findDataFilesToCompact(
sortedFiles.subList(i, Math.min(sortedFiles.size(),
maxFilesToCompact) + i), ratio);
if (!filesToCompact.isEmpty()) {
return filesToCompact;
@@ -441,7 +441,7 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
* smaller set because of it would prevent the future compaction.
*/
private static Collection<CompactableFile>
- findMapFilesToCompact(List<CompactableFile> sortedFiles, double ratio) {
+ findDataFilesToCompact(List<CompactableFile> sortedFiles, double ratio) {
int larsmaIndex = -1;
long larsmaSum = Long.MIN_VALUE;
@@ -500,7 +500,7 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
return executors.get(executors.size() - 1).ceid;
}
- public static List<CompactableFile>
sortByFileSize(Collection<CompactableFile> files) {
+ private static List<CompactableFile>
sortByFileSize(Collection<CompactableFile> files) {
ArrayList<CompactableFile> sortedFiles = new ArrayList<>(files);
// sort from smallest file to largest
diff --git
a/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
b/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
index 9f0a2ef3d6..568b57cd2d 100644
---
a/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
@@ -426,7 +426,7 @@ public class DefaultCompactionPlannerTest {
private static void testFFtC(Set<CompactableFile> expected,
Set<CompactableFile> files,
double ratio, int maxFiles, long maxSize) {
- var result = DefaultCompactionPlanner.findMapFilesToCompact(files, ratio,
maxFiles, maxSize);
+ var result = DefaultCompactionPlanner.findDataFilesToCompact(files, ratio,
maxFiles, maxSize);
var expectedNames =
expected.stream().map(CompactableFile::getUri).map(URI::getPath)
.map(path -> path.split("/")).map(t -> t[t.length -
1]).collect(Collectors.toSet());
var resultNames =
result.stream().map(CompactableFile::getUri).map(URI::getPath)
