This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 257939e609 adds unit test for compactable files w/ ranges (#5981)
257939e609 is described below
commit 257939e609dfdb02bb471498b05c7168734d78db
Author: Keith Turner <[email protected]>
AuthorDate: Thu Nov 20 15:57:55 2025 -0800
adds unit test for compactable files w/ ranges (#5981)
Co-authored-by: Kevin Rathbun <[email protected]>
---
.../client/admin/compaction/CompactableFile.java | 15 +++-
.../org/apache/accumulo/core/data/RowRange.java | 10 +++
.../core/metadata/CompactableFileImpl.java | 11 +--
.../compaction/RatioBasedCompactionPlanner.java | 1 -
.../apache/accumulo/core/util/RowRangeUtil.java | 51 ++++++++++++
.../admin/compaction/CompactableFileTest.java | 93 ++++++++++++++++++++++
.../RatioBasedCompactionPlannerTest.java | 38 +++++++++
.../accumulo/core/util/RowRangeUtilTest.java | 84 +++++++++++++++++++
.../compaction/queue/ResolvedCompactionJob.java | 8 ++
9 files changed, 301 insertions(+), 10 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactableFile.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactableFile.java
index 8f9a9abbc1..e08f12ca39 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactableFile.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactableFile.java
@@ -20,7 +20,7 @@ package org.apache.accumulo.core.client.admin.compaction;
import java.net.URI;
-import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.RowRange;
import org.apache.accumulo.core.metadata.CompactableFileImpl;
/**
@@ -42,22 +42,29 @@ public interface CompactableFile {
*
* @since 4.0.0
*/
- public Range getRange();
+ public RowRange getRange();
public long getEstimatedSize();
public long getEstimatedEntries();
+ /**
+ * Creates a new CompactableFile object that implements this interface. The
returned object
+ * implements equals() and hashCode() based only on the file uri and an
infinite range.
+ */
static CompactableFile create(URI uri, long estimatedSize, long
estimatedEntries) {
return new CompactableFileImpl(uri, estimatedSize, estimatedEntries);
}
/**
- * Creates a new CompactableFile object that implements this interface.
+ * Creates a new CompactableFile object that implements this interface. The
returned object
+ * implements equals() and hashCode() based only on the file uri and range.
*
+ * @param range must be of the form (startRow, endRow]
* @since 4.0.0
*/
- static CompactableFile create(URI uri, Range range, long estimatedSize, long
estimatedEntries) {
+ static CompactableFile create(URI uri, RowRange range, long estimatedSize,
+ long estimatedEntries) {
return new CompactableFileImpl(uri, range, estimatedSize,
estimatedEntries);
}
}
diff --git a/core/src/main/java/org/apache/accumulo/core/data/RowRange.java
b/core/src/main/java/org/apache/accumulo/core/data/RowRange.java
index ed7d1fe511..5895f9f124 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/RowRange.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/RowRange.java
@@ -466,6 +466,16 @@ public class RowRange implements Comparable<RowRange> {
return comp;
}
+ /**
+ * Determines if this row range contains the given row.
+ *
+ * @param row row to check
+ * @return true if the row is contained in the row range, otherwise false
+ */
+ public boolean contains(String row) {
+ return contains(new Text(row));
+ }
+
/**
* Determines if this row range contains the given row.
*
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/CompactableFileImpl.java
b/core/src/main/java/org/apache/accumulo/core/metadata/CompactableFileImpl.java
index c912ad3d30..1e77b6a7cb 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/CompactableFileImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/CompactableFileImpl.java
@@ -22,8 +22,9 @@ import java.net.URI;
import java.util.Objects;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
-import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.RowRange;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.util.RowRangeUtil;
public class CompactableFileImpl implements CompactableFile {
@@ -37,8 +38,8 @@ public class CompactableFileImpl implements CompactableFile {
this.dataFileValue = new DataFileValue(size, entries);
}
- public CompactableFileImpl(URI uri, Range range, long size, long entries) {
- this.storedTabletFile = StoredTabletFile.of(uri, range);
+ public CompactableFileImpl(URI uri, RowRange range, long size, long entries)
{
+ this.storedTabletFile = StoredTabletFile.of(uri, range.asRange());
this.dataFileValue = new DataFileValue(size, entries);
}
@@ -53,8 +54,8 @@ public class CompactableFileImpl implements CompactableFile {
}
@Override
- public Range getRange() {
- return storedTabletFile.getRange();
+ public RowRange getRange() {
+ return RowRangeUtil.toRowRange(storedTabletFile.getRange());
}
@Override
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlanner.java
index 11fcfb9a8d..b71c053d9f 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlanner.java
@@ -126,7 +126,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
* @since 4.0.0
* @see org.apache.accumulo.core.spi.compaction
*/
-
public class RatioBasedCompactionPlanner implements CompactionPlanner {
private final static Logger log =
LoggerFactory.getLogger(RatioBasedCompactionPlanner.class);
diff --git a/core/src/main/java/org/apache/accumulo/core/util/RowRangeUtil.java
b/core/src/main/java/org/apache/accumulo/core/util/RowRangeUtil.java
index dacad9c502..fe2bbedea7 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/RowRangeUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/RowRangeUtil.java
@@ -21,7 +21,9 @@ package org.apache.accumulo.core.util;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.RowRange;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.hadoop.io.Text;
import com.google.common.base.Preconditions;
@@ -83,6 +85,9 @@ public class RowRangeUtil {
return row;
}
+ /**
+ * @throws IllegalArgumentException if the range was not created using row
constructors.
+ */
public static Range requireRowRange(Range range) {
String errorMsg = "Range is not a row range";
@@ -102,4 +107,50 @@ public class RowRangeUtil {
return range;
}
+
+ /**
+ * Converts a {@link Range} created using row constructors to a {@link
RowRange}.
+ *
+ * @throws IllegalArgumentException if the range was not created using row
constructors.
+ */
+ public static RowRange toRowRange(Range range) {
+ requireRowRange(range);
+
+ Text start;
+ boolean startInclusive;
+ Text end;
+ boolean endInclusive;
+
+ // This code reverses the transformations done by the Range row
constructor. Like when that
+ // constructor adds a trailing zero to a row, this code strips it to
recover the original row
+ // passed to the constructor.
+
+ if (range.isInfiniteStartKey()) {
+ start = null;
+ startInclusive = true;
+ } else {
+ if (isRowSuffixZeroByte(range.getStartKey())) {
+ start = new
Text(stripZeroTail(range.getStartKey().getRowData()).toArray());
+ startInclusive = false;
+ } else {
+ start = range.getStartKey().getRow();
+ startInclusive = true;
+ }
+ }
+
+ if (range.isInfiniteStopKey()) {
+ end = null;
+ endInclusive = true;
+ } else {
+ if (isRowSuffixZeroByte(range.getEndKey())) {
+ end = new
Text(stripZeroTail(range.getEndKey().getRowData()).toArray());
+ endInclusive = true;
+ } else {
+ end = range.getEndKey().getRow();
+ endInclusive = false;
+ }
+ }
+
+ return RowRange.range(start, startInclusive, end, endInclusive);
+ }
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/client/admin/compaction/CompactableFileTest.java
b/core/src/test/java/org/apache/accumulo/core/client/admin/compaction/CompactableFileTest.java
new file mode 100644
index 0000000000..47d9f925bc
--- /dev/null
+++
b/core/src/test/java/org/apache/accumulo/core/client/admin/compaction/CompactableFileTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.client.admin.compaction;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+import java.net.URI;
+
+import org.apache.accumulo.core.data.RowRange;
+import org.junit.jupiter.api.Test;
+
+class CompactableFileTest {
+ @Test
+ public void testEqualsHashcode() throws Exception {
+ String prefix = "hdfs://fake/accumulo/tables/1/t-0000000z/";
+ var cf1 = CompactableFile.create(new URI(prefix + "F1.rf"), 100, 10);
+ assertEquals(new URI(prefix + "F1.rf"), cf1.getUri());
+ assertEquals(100, cf1.getEstimatedSize());
+ assertEquals(10, cf1.getEstimatedEntries());
+ assertEquals(RowRange.all(), cf1.getRange());
+
+ // create compactable files with one change
+ var cf2 = CompactableFile.create(new URI(prefix + "F2.rf"), 100, 10);
+ var cf3 = CompactableFile.create(new URI(prefix + "F1.rf"), 101, 10);
+ var cf4 = CompactableFile.create(new URI(prefix + "F1.rf"), 100, 11);
+
+ assertNotEquals(cf1, cf2);
+ assertNotEquals(cf1.hashCode(), cf2.hashCode());
+ assertEquals(RowRange.all(), cf2.getRange());
+ assertEquals(cf1, cf3);
+ assertEquals(cf1.hashCode(), cf3.hashCode());
+ assertEquals(RowRange.all(), cf3.getRange());
+ assertEquals(cf1, cf4);
+ assertEquals(cf1.hashCode(), cf4.hashCode());
+ assertEquals(RowRange.all(), cf4.getRange());
+
+ // This should be exactly the same as cf1
+ var cf5 = CompactableFile.create(new URI(prefix + "F1.rf"),
RowRange.all(), 100, 10);
+ assertEquals(cf1, cf5);
+ assertEquals(cf1.hashCode(), cf5.hashCode());
+ assertEquals(cf1.getRange(), cf5.getRange());
+ assertEquals(cf1.getEstimatedEntries(), cf5.getEstimatedEntries());
+ assertEquals(cf1.getEstimatedSize(), cf5.getEstimatedSize());
+ assertEquals(100, cf5.getEstimatedSize());
+ assertEquals(10, cf5.getEstimatedEntries());
+
+ // the creating with the same file as cf1 and an inf range
+ var cf6 = CompactableFile.create(new URI(prefix + "F1.rf"),
RowRange.all(), 100, 10);
+ assertEquals(cf1, cf6);
+ assertEquals(cf1.hashCode(), cf6.hashCode());
+
+ // same file with different range should not be equals
+ var cf7 =
+ CompactableFile.create(new URI(prefix + "F1.rf"),
RowRange.openClosed("c", "f"), 100, 10);
+ assertNotEquals(cf1, cf7);
+ assertNotEquals(cf1.hashCode(), cf7.hashCode());
+ assertEquals(new URI(prefix + "F1.rf"), cf7.getUri());
+ assertEquals(100, cf7.getEstimatedSize());
+ assertEquals(10, cf7.getEstimatedEntries());
+ assertEquals(RowRange.openClosed("c", "f"), cf7.getRange());
+
+ var cf8 =
+ CompactableFile.create(new URI(prefix + "F1.rf"),
RowRange.openClosed("x", "z"), 100, 10);
+ assertNotEquals(cf7, cf8);
+ assertNotEquals(cf7.hashCode(), cf8.hashCode());
+ assertEquals(RowRange.openClosed("x", "z"), cf8.getRange());
+
+ // test different objects created with same data
+ var cf9 =
+ CompactableFile.create(new URI(prefix + "F1.rf"),
RowRange.openClosed("x", "z"), 100, 10);
+ assertEquals(cf8, cf9);
+ assertEquals(cf8.hashCode(), cf9.hashCode());
+ assertEquals(cf8.getRange(), cf9.getRange());
+ assertEquals(RowRange.openClosed("x", "z"), cf8.getRange());
+ }
+}
diff --git
a/core/src/test/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlannerTest.java
b/core/src/test/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlannerTest.java
index ea3d2a907c..5eb8515d8c 100644
---
a/core/src/test/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlannerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlannerTest.java
@@ -50,6 +50,7 @@ import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.ResourceGroupId;
+import org.apache.accumulo.core.data.RowRange;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.TabletId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
@@ -851,6 +852,43 @@ public class RatioBasedCompactionPlannerTest {
verify(senv);
}
+ /**
+ * Ensures the planner does not drop, modify, or mix up ranges on input
files.
+ */
+ @Test
+ public void testFilesWithRanges() throws Exception {
+ String prefix = "hdfs://fake/accumulo/tables/1/t-0000000z/";
+ var file1 =
+ CompactableFile.create(new URI(prefix + "F1.rf"),
RowRange.openClosed("c", "m"), 1000, 1);
+ var file2 =
+ CompactableFile.create(new URI(prefix + "F1.rf"),
RowRange.openClosed("o", "s"), 1001, 2);
+ var file3 =
+ CompactableFile.create(new URI(prefix + "F2.rf"),
RowRange.openClosed("x", "y"), 1002, 3);
+ var file4 = CompactableFile.create(new URI(prefix + "F3.rf"), 1003, 4);
+ var file5 = CompactableFile.create(new URI(prefix + "F4.rf"),
RowRange.openClosed("abc", "xyz"),
+ 1000000, 5);
+
+ String groups = "[{'group':'small','maxSize':'32M'},
{'group':'medium','maxSize':'128M'},"
+ + "{'group':'large','maxSize':'512M'}, {'group':'huge'}]";
+
+ var systemConf = Map.<String,String>of();
+ var tableConf = Map.<String,String>of();
+ var senv = createMockServiceEnvironment(systemConf, tableConf);
+
+ var planner = createPlanner(senv, groups);
+ var all = Set.of(file1, file2, file3, file4, file5);
+ var params = createPlanningParams(senv, all, all, Set.of(), 3,
CompactionKind.SYSTEM);
+ var plan = planner.makePlan(params);
+ var job = getOnlyElement(plan.getJobs());
+ assertEquals(Set.of(
+ CompactableFile.create(new URI(prefix + "F1.rf"),
RowRange.openClosed("c", "m"), 1000, 1),
+ CompactableFile.create(new URI(prefix + "F1.rf"),
RowRange.openClosed("o", "s"), 1001, 2),
+ CompactableFile.create(new URI(prefix + "F2.rf"),
RowRange.openClosed("x", "y"), 1002, 3),
+ CompactableFile.create(new URI(prefix + "F3.rf"), 1003, 4)),
job.getFiles());
+
+ verify(senv);
+ }
+
private CompactionJob createJob(CompactionKind kind, Set<CompactableFile>
all,
Set<CompactableFile> files) {
return new CompactionPlanImpl.BuilderImpl(kind, all)
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/RowRangeUtilTest.java
b/core/src/test/java/org/apache/accumulo/core/util/RowRangeUtilTest.java
new file mode 100644
index 0000000000..d0b21dd6ea
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/util/RowRangeUtilTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.util.List;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.RowRange;
+import org.junit.jupiter.api.Test;
+
+public class RowRangeUtilTest {
+ @Test
+ public void testConversions() {
+ var ranges = List.of(new Range("a", false, "d", false), new Range("a",
false, "d", true),
+ new Range("a", true, "d", false), new Range("a", true, "d", true), new
Range(),
+ new Range("a", true, null, true), new Range("a", false, null, true),
+ new Range(null, true, "d", true), new Range(null, true, "d", false));
+
+ for (var range : ranges) {
+ var rowRange = RowRangeUtil.toRowRange(range);
+ for (var row : List.of("12", "a", "aa", "c", "ca", "d", "da", "z", "AA",
"ZZ")) {
+ // ensure the row range and key range are in agreement
+ assertEquals(range.contains(new Key(row)), rowRange.contains(row));
+ }
+ }
+
+ // Ensure the transformations done by the range constructor are properly
reversed.
+ assertEquals(RowRange.all(), RowRangeUtil.toRowRange(new Range()));
+ assertEquals(RowRange.atLeast("a"), RowRangeUtil.toRowRange(new Range("a",
true, null, true)));
+ assertEquals(RowRange.greaterThan("a"),
+ RowRangeUtil.toRowRange(new Range("a", false, null, true)));
+ assertEquals(RowRange.atMost("a"), RowRangeUtil.toRowRange(new Range(null,
true, "a", true)));
+ assertEquals(RowRange.lessThan("a"),
+ RowRangeUtil.toRowRange(new Range(null, true, "a", false)));
+ assertEquals(RowRange.closed("a"), RowRangeUtil.toRowRange(new Range("a",
true, "a", true)));
+ assertEquals(RowRange.closed("a", "d"),
+ RowRangeUtil.toRowRange(new Range("a", true, "d", true)));
+ assertEquals(RowRange.open("a", "d"),
+ RowRangeUtil.toRowRange(new Range("a", false, "d", false)));
+ assertEquals(RowRange.openClosed("a", "d"),
+ RowRangeUtil.toRowRange(new Range("a", false, "d", true)));
+ assertEquals(RowRange.closedOpen("a", "d"),
+ RowRangeUtil.toRowRange(new Range("a", true, "d", false)));
+
+ // ensure non row ranges fail
+ assertThrows(IllegalArgumentException.class,
+ () -> RowRangeUtil.toRowRange(new Range(new Key("r", "f"), false, new
Key("x"), false)));
+ assertThrows(IllegalArgumentException.class,
+ () -> RowRangeUtil.toRowRange(new Range(new Key("r", "f"), false, new
Key("x"), true)));
+ assertThrows(IllegalArgumentException.class,
+ () -> RowRangeUtil.toRowRange(new Range(new Key("r", "f"), true, new
Key("x"), false)));
+ assertThrows(IllegalArgumentException.class,
+ () -> RowRangeUtil.toRowRange(new Range(new Key("r", "f"), true, new
Key("x"), true)));
+
+ assertThrows(IllegalArgumentException.class,
+ () -> RowRangeUtil.toRowRange(new Range(new Key("r"), false, new
Key("x", "f"), false)));
+ assertThrows(IllegalArgumentException.class,
+ () -> RowRangeUtil.toRowRange(new Range(new Key("r"), false, new
Key("x", "f"), true)));
+ assertThrows(IllegalArgumentException.class,
+ () -> RowRangeUtil.toRowRange(new Range(new Key("r"), true, new
Key("x", "f"), false)));
+ assertThrows(IllegalArgumentException.class,
+ () -> RowRangeUtil.toRowRange(new Range(new Key("r"), true, new
Key("x", "f"), true)));
+ }
+}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/ResolvedCompactionJob.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/ResolvedCompactionJob.java
index bf27772bc8..3177b3c0fb 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/ResolvedCompactionJob.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/ResolvedCompactionJob.java
@@ -27,6 +27,7 @@ import java.util.stream.Collectors;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.ResourceGroupId;
+import org.apache.accumulo.core.data.RowRange;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.fate.FateId;
import org.apache.accumulo.core.metadata.CompactableFileImpl;
@@ -60,6 +61,13 @@ public class ResolvedCompactionJob implements CompactionJob {
private final String tabletDir;
private final boolean overlapsSelectedFiles;
+ private static long weigh(RowRange rowRange) {
+ if (rowRange != null) {
+ return weigh(rowRange.asRange());
+ }
+ return 0;
+ }
+
private static long weigh(Range range) {
long estDataSize = 0;
if (range != null) {
