This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new bf61fac87a Fixed bug in TabletsMetadata.fetch (#5342)
bf61fac87a is described below
commit bf61fac87a4b29e1e3aaa408bfea0d2711a9a812
Author: Dave Marion <[email protected]>
AuthorDate: Thu Feb 20 11:56:46 2025 -0500
Fixed bug in TabletsMetadata.fetch (#5342)
A bug was introduced in #5335 to TabletsMetadata.fetch where the
columns being passed as arguments to the function where not being
used. This has the side effect of fetching all columns in the
TabletMetadata which was causing some ITs to time out. The ITs
in these cases were fetching and counting specific columns in
the TabletMetadata.
This bug also highlighted another bug in CloneIT where an invalid
directory was being inserted into the TabletMetadata which would
then subsequently fail when fetched. The directory column name
was being validated on most, but not all, places where it was being
inserted into the TabletMetadata. I added validation to the
ServerColumnFamily.DIRECTORY_COLUMN.put method and fixed CloneIT.
---
.../core/metadata/schema/MetadataSchema.java | 8 +++-
.../core/metadata/schema/TabletMetadata.java | 1 -
.../core/metadata/schema/TabletMutatorBase.java | 1 -
.../core/metadata/schema/TabletsMetadata.java | 3 +-
.../constraints/MetadataConstraintsTest.java | 3 +-
.../java/org/apache/accumulo/test/CloneIT.java | 48 +++++++++++-----------
6 files changed, 34 insertions(+), 30 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
index 885fd32623..6a003c22fd 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
@@ -203,7 +203,13 @@ public class MetadataSchema {
* Holds the location of the tablet in the DFS file system
*/
public static final String DIRECTORY_QUAL = "dir";
- public static final ColumnFQ DIRECTORY_COLUMN = new ColumnFQ(NAME, new
Text(DIRECTORY_QUAL));
+ public static final ColumnFQ DIRECTORY_COLUMN = new ColumnFQ(NAME, new
Text(DIRECTORY_QUAL)) {
+ @Override
+ public void put(Mutation m, Value v) {
+ validateDirCol(v.toString());
+ super.put(m, v);
+ }
+ };
/**
* Initial tablet directory name for the default tablet in all tables
*/
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index c737b0d66b..a5a41a2684 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -73,7 +73,6 @@ import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.SuspendingTServer;
import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CompactedColumnFamily;
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
index 4f39eda7b7..537caf917e 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
@@ -90,7 +90,6 @@ public abstract class TabletMutatorBase<T extends
Ample.TabletUpdates<T>>
@Override
public T putDirName(String dirName) {
- ServerColumnFamily.validateDirCol(dirName);
Preconditions.checkState(updatesEnabled, "Cannot make updates after
calling mutate.");
ServerColumnFamily.DIRECTORY_COLUMN.put(mutation, new Value(dirName));
return getThis();
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index 26d157eb19..8f448c82ec 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@ -21,7 +21,6 @@ package org.apache.accumulo.core.metadata.schema;
import static com.google.common.base.Preconditions.checkState;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
import java.io.IOException;
import java.io.UncheckedIOException;
@@ -305,7 +304,7 @@ public class TabletsMetadata implements
Iterable<TabletMetadata>, AutoCloseable
@Override
public Options fetch(ColumnType... colsToFetch) {
Preconditions.checkArgument(colsToFetch.length > 0);
- for (var col : fetchedCols) {
+ for (var col : colsToFetch) {
fetchedCols.add(col);
var qualifier = ColumnType.resolveQualifier(col);
if (qualifier != null) {
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
index 40d347fcd5..ecbe69e480 100644
---
a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
@@ -634,7 +634,8 @@ public class MetadataConstraintsTest {
assertTrue(violations.isEmpty());
m = new Mutation(new Text("0;foo"));
- ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value("/invalid"));
+ m.put(ServerColumnFamily.DIRECTORY_COLUMN.getColumnFamily(),
+ ServerColumnFamily.DIRECTORY_COLUMN.getColumnQualifier(), new
Value("/invalid"));
violations = mc.check(createEnv(), m);
assertFalse(violations.isEmpty());
assertEquals(1, violations.size());
diff --git a/test/src/main/java/org/apache/accumulo/test/CloneIT.java
b/test/src/main/java/org/apache/accumulo/test/CloneIT.java
index 2d7caa2e94..5b84f3685a 100644
--- a/test/src/main/java/org/apache/accumulo/test/CloneIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/CloneIT.java
@@ -66,7 +66,7 @@ public class CloneIT extends AccumuloClusterHarness {
Mutation mut = TabletColumnFamily.createPrevRowMutation(ke);
ServerColumnFamily.TIME_COLUMN.put(mut, new Value("M0"));
- ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new
Value("/default_tablet"));
+ ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new
Value("default_tablet"));
try (BatchWriter bw1 = client.createBatchWriter(tableName)) {
bw1.addMutation(mut);
@@ -95,7 +95,7 @@ public class CloneIT extends AccumuloClusterHarness {
Mutation mut = TabletColumnFamily.createPrevRowMutation(ke);
ServerColumnFamily.TIME_COLUMN.put(mut, new Value("M0"));
- ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new
Value("/default_tablet"));
+ ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new
Value("default_tablet"));
mut.put(DataFileColumnFamily.NAME.toString(),
getMetadata(filePrefix + "/default_tablet/0_0.rf", range1),
new DataFileValue(1, 200).encodeAsString());
@@ -155,17 +155,17 @@ public class CloneIT extends AccumuloClusterHarness {
try (BatchWriter bw1 = client.createBatchWriter(tableName);
BatchWriter bw2 = client.createBatchWriter(tableName)) {
- bw1.addMutation(createTablet("0", null, null, "/default_tablet",
+ bw1.addMutation(createTablet("0", null, null, "default_tablet",
filePrefix + "/default_tablet/0_0.rf", range));
bw1.flush();
MetadataTableUtil.initializeClone(tableName, TableId.of("0"),
TableId.of("1"), client, bw2);
- bw1.addMutation(createTablet("0", "m", null, "/default_tablet",
+ bw1.addMutation(createTablet("0", "m", null, "default_tablet",
filePrefix + "/default_tablet/0_0.rf", range));
bw1.addMutation(
- createTablet("0", null, "m", "/t-1", filePrefix +
"/default_tablet/0_0.rf", range));
+ createTablet("0", null, "m", "t-1", filePrefix +
"/default_tablet/0_0.rf", range));
bw1.flush();
@@ -203,17 +203,17 @@ public class CloneIT extends AccumuloClusterHarness {
try (BatchWriter bw1 = client.createBatchWriter(tableName);
BatchWriter bw2 = client.createBatchWriter(tableName)) {
- bw1.addMutation(createTablet("0", null, null, "/default_tablet",
+ bw1.addMutation(createTablet("0", null, null, "default_tablet",
filePrefix + "/default_tablet/0_0.rf", range));
bw1.flush();
MetadataTableUtil.initializeClone(tableName, TableId.of("0"),
TableId.of("1"), client, bw2);
- bw1.addMutation(createTablet("0", "m", null, "/default_tablet",
+ bw1.addMutation(createTablet("0", "m", null, "default_tablet",
filePrefix + "/default_tablet/1_0.rf", range));
Mutation mut3 =
- createTablet("0", null, "m", "/t-1", filePrefix +
"/default_tablet/1_0.rf", range);
+ createTablet("0", null, "m", "t-1", filePrefix +
"/default_tablet/1_0.rf", range);
mut3.putDelete(DataFileColumnFamily.NAME.toString(),
getMetadata(filePrefix + "/default_tablet/0_0.rf", range));
bw1.addMutation(mut3);
@@ -285,17 +285,17 @@ public class CloneIT extends AccumuloClusterHarness {
try (BatchWriter bw1 = client.createBatchWriter(tableName);
BatchWriter bw2 = client.createBatchWriter(tableName)) {
- bw1.addMutation(createTablet("0", "m", null, "/d1", filePrefix +
"/d1/file1.rf", range1));
- bw1.addMutation(createTablet("0", null, "m", "/d2", filePrefix +
"/d2/file2.rf", range2));
+ bw1.addMutation(createTablet("0", "m", null, "d1", filePrefix +
"/d1/file1.rf", range1));
+ bw1.addMutation(createTablet("0", null, "m", "d2", filePrefix +
"/d2/file2.rf", range2));
bw1.flush();
MetadataTableUtil.initializeClone(tableName, TableId.of("0"),
TableId.of("1"), client, bw2);
- bw1.addMutation(createTablet("0", "f", null, "/d1", filePrefix +
"/d1/file3.rf", range3));
- bw1.addMutation(createTablet("0", "m", "f", "/d3", filePrefix +
"/d1/file1.rf", range1));
- bw1.addMutation(createTablet("0", "s", "m", "/d2", filePrefix +
"/d2/file2.rf", range2));
- bw1.addMutation(createTablet("0", null, "s", "/d4", filePrefix +
"/d2/file2.rf", range2));
+ bw1.addMutation(createTablet("0", "f", null, "d1", filePrefix +
"/d1/file3.rf", range3));
+ bw1.addMutation(createTablet("0", "m", "f", "d3", filePrefix +
"/d1/file1.rf", range1));
+ bw1.addMutation(createTablet("0", "s", "m", "d2", filePrefix +
"/d2/file2.rf", range2));
+ bw1.addMutation(createTablet("0", null, "s", "d4", filePrefix +
"/d2/file2.rf", range2));
bw1.flush();
@@ -335,8 +335,8 @@ public class CloneIT extends AccumuloClusterHarness {
try (BatchWriter bw1 = client.createBatchWriter(tableName);
BatchWriter bw2 = client.createBatchWriter(tableName)) {
- bw1.addMutation(createTablet("0", "m", null, "/d1", filePrefix +
"/d1/file1.rf", range1));
- bw1.addMutation(createTablet("0", null, "m", "/d2", filePrefix +
"/d2/file2.rf", range2));
+ bw1.addMutation(createTablet("0", "m", null, "d1", filePrefix +
"/d1/file1.rf", range1));
+ bw1.addMutation(createTablet("0", null, "m", "d2", filePrefix +
"/d2/file2.rf", range2));
bw1.flush();
@@ -347,10 +347,10 @@ public class CloneIT extends AccumuloClusterHarness {
bw1.flush();
- bw1.addMutation(createTablet("0", "f", null, "/d1", filePrefix +
"/d1/file3.rf", range3));
- bw1.addMutation(createTablet("0", "m", "f", "/d3", filePrefix +
"/d1/file1.rf", range1));
- bw1.addMutation(createTablet("0", "s", "m", "/d2", filePrefix +
"/d2/file3.rf", range3));
- bw1.addMutation(createTablet("0", null, "s", "/d4", filePrefix +
"/d4/file3.rf", range3));
+ bw1.addMutation(createTablet("0", "f", null, "d1", filePrefix +
"/d1/file3.rf", range3));
+ bw1.addMutation(createTablet("0", "m", "f", "d3", filePrefix +
"/d1/file1.rf", range1));
+ bw1.addMutation(createTablet("0", "s", "m", "d2", filePrefix +
"/d2/file3.rf", range3));
+ bw1.addMutation(createTablet("0", null, "s", "d4", filePrefix +
"/d4/file3.rf", range3));
bw1.flush();
@@ -363,7 +363,7 @@ public class CloneIT extends AccumuloClusterHarness {
bw1.flush();
- bw1.addMutation(createTablet("0", "m", "f", "/d3", filePrefix +
"/d1/file3.rf", range3));
+ bw1.addMutation(createTablet("0", "m", "f", "d3", filePrefix +
"/d1/file3.rf", range3));
bw1.flush();
@@ -405,15 +405,15 @@ public class CloneIT extends AccumuloClusterHarness {
try (BatchWriter bw1 = client.createBatchWriter(tableName);
BatchWriter bw2 = client.createBatchWriter(tableName)) {
- bw1.addMutation(createTablet("0", "m", null, "/d1", filePrefix +
"/d1/file1.rf", range1));
- bw1.addMutation(createTablet("0", null, "m", "/d2", filePrefix +
"/d2/file2.rf", range2));
+ bw1.addMutation(createTablet("0", "m", null, "d1", filePrefix +
"/d1/file1.rf", range1));
+ bw1.addMutation(createTablet("0", null, "m", "d2", filePrefix +
"/d2/file2.rf", range2));
bw1.flush();
MetadataTableUtil.initializeClone(tableName, TableId.of("0"),
TableId.of("1"), client, bw2);
bw1.addMutation(deleteTablet("0", "m", null, filePrefix +
"/d1/file1.rf", range1));
- Mutation mut = createTablet("0", null, null, "/d2", filePrefix +
"/d2/file2.rf", range2);
+ Mutation mut = createTablet("0", null, null, "d2", filePrefix +
"/d2/file2.rf", range2);
mut.put(DataFileColumnFamily.NAME.toString(),
getMetadata(filePrefix + "/d1/file1.rf", range1),
new DataFileValue(10, 200).encodeAsString());
