This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch no-chop-merge
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 341ece512e2ad5111e2afe3801cc080f066ed487
Author: Christopher L. Shannon (cshannon) <christopher.l.shan...@gmail.com>
AuthorDate: Fri Sep 15 12:31:33 2023 -0400
Update DFV when multiple ranges are created during delete
This updates the data file values during deletion when 2 ranges are
created to divide the values in half and make sure they total the
original
---
.../accumulo/manager/TabletGroupWatcher.java | 41 ++++++++++++++--
.../accumulo/manager/TabletGroupWatcherTest.java | 55 ++++++++++++++++++++++
2 files changed, 92 insertions(+), 4 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index b2384972c3..c0b4e3b785 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -110,6 +110,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.thrift.TException;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterators;
@@ -1080,8 +1081,8 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
// Go through and fence each of the files that are part of the tablet
for (Entry<StoredTabletFile,DataFileValue> entry :
tabletMetadata.getFilesMap()
.entrySet()) {
- StoredTabletFile existing = entry.getKey();
- Value value = entry.getValue().encodeAsValue();
+ final StoredTabletFile existing = entry.getKey();
+ final DataFileValue value = entry.getValue();
final Mutation m = new Mutation(keyExtent.toMetaRow());
@@ -1119,9 +1120,28 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
// If the existingFile is not contained in the newFiles set then
we can delete it
Sets.difference(existingFile, newFiles).forEach(
delete -> m.putDelete(DataFileColumnFamily.NAME,
existing.getMetadataText()));
+
// Add any new files that don't match the existingFile
- Sets.difference(newFiles, existingFile).forEach(
- newFile -> m.put(DataFileColumnFamily.NAME,
newFile.getMetadataText(), value));
+ // As of now we will only have at most 2 files as up to 2 ranges
are created
+ final List<StoredTabletFile> filesToAdd =
+ new ArrayList<>(Sets.difference(newFiles, existingFile));
+ Preconditions.checkArgument(filesToAdd.size() <= 2,
+ "There should only be at most 2 StoredTabletFiles after
computing new ranges.");
+
+ // If more than 1 new file then re-calculate the num entries and
size
+ if (filesToAdd.size() == 2) {
+ // This splits up the values in half and makes sure they total
the original
+ // values
+ final Pair<DataFileValue,DataFileValue> newDfvs =
computeNewDfv(value);
+ m.put(DataFileColumnFamily.NAME,
filesToAdd.get(0).getMetadataText(),
+ newDfvs.getFirst().encodeAsValue());
+ m.put(DataFileColumnFamily.NAME,
filesToAdd.get(1).getMetadataText(),
+ newDfvs.getSecond().encodeAsValue());
+ } else {
+ // Will be 0 or 1 files
+ filesToAdd.forEach(newFile -> m.put(DataFileColumnFamily.NAME,
+ newFile.getMetadataText(), value.encodeAsValue()));
+ }
if (!m.getUpdates().isEmpty()) {
bw.addMutation(m);
@@ -1142,6 +1162,19 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
}
}
+ // Divide each new DFV in half and make sure the sum equals the original
+ @VisibleForTesting
+ protected static Pair<DataFileValue,DataFileValue>
computeNewDfv(DataFileValue value) {
+ final DataFileValue file1Value = new DataFileValue(Math.max(1,
value.getSize() / 2),
+ Math.max(1, value.getNumEntries() / 2), value.getTime());
+
+ final DataFileValue file2Value =
+ new DataFileValue(Math.max(1, value.getSize() - file1Value.getSize()),
+ Math.max(1, value.getNumEntries() - file1Value.getNumEntries()),
value.getTime());
+
+ return new Pair<>(file1Value, file2Value);
+ }
+
private Optional<TabletMetadata> loadTabletMetadata(TableId tabletId, final
Text row,
ColumnType... columns) {
try (TabletsMetadata tabletsMetadata =
manager.getContext().getAmple().readTablets()
diff --git
a/server/manager/src/test/java/org/apache/accumulo/manager/TabletGroupWatcherTest.java
b/server/manager/src/test/java/org/apache/accumulo/manager/TabletGroupWatcherTest.java
new file mode 100644
index 0000000000..c5d5ac17c1
--- /dev/null
+++
b/server/manager/src/test/java/org/apache/accumulo/manager/TabletGroupWatcherTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.util.Pair;
+import org.junit.jupiter.api.Test;
+
+public class TabletGroupWatcherTest {
+
+ @Test
+ public void testComputeNewDfvEven() {
+ DataFileValue original = new DataFileValue(20, 10, 100);
+ Pair<DataFileValue,DataFileValue> newValues =
TabletGroupWatcher.computeNewDfv(original);
+
+ assertEquals(10, newValues.getFirst().getSize());
+ assertEquals(5, newValues.getFirst().getNumEntries());
+ assertEquals(original.getTime(), newValues.getFirst().getTime());
+ assertEquals(10, newValues.getSecond().getSize());
+ assertEquals(5, newValues.getSecond().getNumEntries());
+ assertEquals(original.getTime(), newValues.getSecond().getTime());
+ }
+
+ @Test
+ public void testComputeNewDfvOdd() {
+ DataFileValue original = new DataFileValue(21, 11, 100);
+ Pair<DataFileValue,DataFileValue> newValues =
TabletGroupWatcher.computeNewDfv(original);
+
+ assertEquals(10, newValues.getFirst().getSize());
+ assertEquals(5, newValues.getFirst().getNumEntries());
+ assertEquals(original.getTime(), newValues.getFirst().getTime());
+ assertEquals(11, newValues.getSecond().getSize());
+ assertEquals(6, newValues.getSecond().getNumEntries());
+ assertEquals(original.getTime(), newValues.getSecond().getTime());
+ }
+
+}
