This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit d594e585a781797082fcd3cb5202cc67cf84e45a
Merge: 8375719384 b8089dd528
Author: Christopher L. Shannon (cshannon) <christopher.l.shan...@gmail.com>
AuthorDate: Sat Nov 18 15:09:55 2023 -0500
Merge branch 'main' into elasticity
.../core/metadata/schema/MetadataSchema.java | 13 ++
.../core/metadata/schema/TabletMetadata.java | 11 ++
.../core/metadata/schema/TabletsMetadata.java | 4 +
.../core/metadata/schema/TabletMetadataTest.java | 29 ++++
.../server/constraints/MetadataConstraints.java | 6 +-
test/pom.xml | 4 +
.../java/org/apache/accumulo/test/CloneIT.java | 149 +++++++++++++--------
.../org/apache/accumulo/test/ImportExportIT.java | 74 +++++++---
.../java/org/apache/accumulo/test/MetaSplitIT.java | 9 ++
.../apache/accumulo/test/functional/MergeIT.java | 22 ++-
.../accumulo/test/util/FileMetadataUtil.java | 9 ++
11 files changed, 248 insertions(+), 82 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
index fefe9b6911,2bc0a6e18e..d7f6452670
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
@@@ -35,6 -35,6 +35,7 @@@ import org.apache.accumulo.core.schema.
import org.apache.accumulo.core.util.ColumnFQ;
import org.apache.accumulo.core.util.Pair;
import org.apache.hadoop.io.Text;
++import org.checkerframework.common.returnsreceiver.qual.This;
import com.google.common.base.Preconditions;
@@@ -397,23 -326,17 +398,35 @@@ public class MetadataSchema
public static final Text NAME = new Text(STR_NAME);
}
+ /**
+ * Column family for indicating that the files in a tablet contain fenced
files that have been
+ * merged from other tablets during a merge operation. This is used to
support resuming a failed
+ * merge operation.
+ */
+ public static class MergedColumnFamily {
+ public static final String STR_NAME = "merged";
+ public static final Text NAME = new Text(STR_NAME);
+ public static final ColumnFQ MERGED_COLUMN = new ColumnFQ(NAME, new
Text(STR_NAME));
+ public static final Value MERGED_VALUE = new Value("merged");
+ }
++
+ /**
+ * This family is used to track which tablets were compacted by a user
compaction. The column
+ * qualifier is expected to contain the fate transaction id that is
executing the compaction.
+ */
+ public static class CompactedColumnFamily {
+ public static final String STR_NAME = "compacted";
+ public static final Text NAME = new Text(STR_NAME);
+ }
+
+ public static class HostingColumnFamily {
+ public static final String STR_NAME = "hosting";
+ public static final Text NAME = new Text(STR_NAME);
+ public static final String GOAL_QUAL = "goal";
+ public static final ColumnFQ GOAL_COLUMN = new ColumnFQ(NAME, new
Text(GOAL_QUAL));
+ public static final String REQUESTED_QUAL = "requested";
+ public static final ColumnFQ REQUESTED_COLUMN = new ColumnFQ(NAME, new
Text(REQUESTED_QUAL));
+ }
}
/**
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index 2ae7f394b7,af31605d51..391f81ffed
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@@ -68,9 -60,9 +68,10 @@@ import org.apache.accumulo.core.metadat
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.FutureLocationColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LastLocationColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+ import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
@@@ -118,15 -105,7 +119,16 @@@ public class TabletMetadata
private OptionalLong compact = OptionalLong.empty();
private Double splitRatio = null;
private Map<ExternalCompactionId,ExternalCompactionMetadata> extCompactions;
+ private boolean merged;
+ private TabletHostingGoal goal = TabletHostingGoal.ONDEMAND;
+ private boolean onDemandHostingRequested = false;
+ private TabletOperationId operationId;
+ private boolean futureAndCurrentLocationSet = false;
+ private Set<Long> compacted;
+
+ public static TabletMetadataBuilder builder(KeyExtent extent) {
+ return new TabletMetadataBuilder(extent);
+ }
public enum LocationType {
CURRENT, FUTURE, LAST
@@@ -149,11 -128,7 +151,12 @@@
SPLIT_RATIO,
SUSPEND,
ECOMP,
- MERGED
++ MERGED,
+ HOSTING_GOAL,
+ HOSTING_REQUESTED,
+ OPID,
+ SELECTED,
+ COMPACTED
}
public static class Location {
@@@ -379,37 -348,11 +382,42 @@@
return splitRatio;
}
+ public boolean hasMerged() {
+ ensureFetched(ColumnType.MERGED);
+ return merged;
+ }
+
+ public TabletHostingGoal getHostingGoal() {
+ if (RootTable.ID.equals(getTableId()) ||
MetadataTable.ID.equals(getTableId())) {
+ // Override the goal for the system tables
+ return TabletHostingGoal.ALWAYS;
+ }
+ ensureFetched(ColumnType.HOSTING_GOAL);
+ return goal;
+ }
+
+ public boolean getHostingRequested() {
+ ensureFetched(ColumnType.HOSTING_REQUESTED);
+ return onDemandHostingRequested;
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this,
ToStringStyle.SHORT_PREFIX_STYLE).append("tableId", tableId)
+ .append("prevEndRow", prevEndRow).append("sawPrevEndRow",
sawPrevEndRow)
+ .append("oldPrevEndRow", oldPrevEndRow).append("sawOldPrevEndRow",
sawOldPrevEndRow)
+ .append("endRow", endRow).append("location",
location).append("files", files)
+ .append("scans", scans).append("loadedFiles", loadedFiles)
+ .append("fetchedCols", fetchedCols).append("extent",
extent).append("last", last)
+ .append("suspend", suspend).append("dirName", dirName).append("time",
time)
+ .append("cloned", cloned).append("flush", flush).append("logs", logs)
+ .append("compact", compact).append("splitRatio", splitRatio)
+ .append("extCompactions", extCompactions).append("goal", goal)
+ .append("onDemandHostingRequested", onDemandHostingRequested)
+ .append("operationId", operationId).append("selectedFiles",
selectedFiles)
+ .append("futureAndCurrentLocationSet",
futureAndCurrentLocationSet).toString();
+ }
+
public SortedMap<Key,Value> getKeyValues() {
Preconditions.checkState(keyValues != null, "Requested key values when it
was not saved");
return keyValues;
@@@ -549,21 -487,11 +557,24 @@@
extCompBuilder.put(ExternalCompactionId.of(qual),
ExternalCompactionMetadata.fromJson(val));
break;
+ case MergedColumnFamily.STR_NAME:
+ te.merged = true;
+ break;
- default:
- throw new IllegalStateException("Unexpected family " + fam);
+ case CompactedColumnFamily.STR_NAME:
+ compactedBuilder.add(FateTxId.fromString(qual));
+ break;
+ case HostingColumnFamily.STR_NAME:
+ switch (qual) {
+ case GOAL_QUAL:
+ te.goal = TabletHostingGoalUtil.fromValue(kv.getValue());
+ break;
+ case REQUESTED_QUAL:
+ te.onDemandHostingRequested = true;
+ break;
+ default:
+ throw new IllegalStateException("Unexpected family " + fam);
+ }
+ break;
}
}
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index 3e50d5d17d,515e6a08ef..997d082084
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@@ -77,9 -74,9 +77,10 @@@ import org.apache.accumulo.core.metadat
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.FutureLocationColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LastLocationColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+ import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
@@@ -346,17 -337,12 +347,20 @@@ public class TabletsMetadata implement
case ECOMP:
families.add(ExternalCompactionColumnFamily.NAME);
break;
+ case MERGED:
+ families.add(MergedColumnFamily.NAME);
+ break;
+ case OPID:
+ qualifiers.add(OPID_COLUMN);
+ break;
+ case SELECTED:
+ qualifiers.add(SELECTED_COLUMN);
+ break;
+ case COMPACTED:
+ families.add(CompactedColumnFamily.NAME);
+ break;
default:
throw new IllegalArgumentException("Unknown col type " +
colToFetch);
-
}
}
diff --cc
core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index b890a54299,d547b8596d..fd8c8a80b8
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@@ -19,8 -19,9 +19,10 @@@
package org.apache.accumulo.core.metadata.schema;
import static java.util.stream.Collectors.toSet;
+import static org.apache.accumulo.core.fate.FateTxId.formatTid;
import static org.apache.accumulo.core.metadata.StoredTabletFile.serialize;
+ import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily.MERGED_COLUMN;
+ import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily.MERGED_VALUE;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
@@@ -274,6 -263,30 +279,30 @@@ public class TabletMetadataTest
assertFalse(tm.hasCurrent());
}
+ @Test
+ public void testMergedColumn() {
+ KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+ // Test merged column set
+ Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ MERGED_COLUMN.put(mutation, MERGED_VALUE);
+ TabletMetadata tm =
TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
- EnumSet.of(ColumnType.MERGED), true);
++ EnumSet.of(ColumnType.MERGED), true, false);
+ assertTrue(tm.hasMerged());
+
+ // Column not set
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
- EnumSet.of(ColumnType.MERGED), true);
++ EnumSet.of(ColumnType.MERGED), true, false);
+ assertFalse(tm.hasMerged());
+
+ // MERGED Column not fetched
+ mutation = TabletColumnFamily.createPrevRowMutation(extent);
+ tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
- EnumSet.of(ColumnType.PREV_ROW), true);
++ EnumSet.of(ColumnType.PREV_ROW), true, false);
+ assertThrows(IllegalStateException.class, tm::hasMerged);
+ }
+
private SortedMap<Key,Value> toRowMap(Mutation mutation) {
SortedMap<Key,Value> rowMap = new TreeMap<>();
mutation.getUpdates().forEach(cu -> {
diff --cc
server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index 10e1aa9fc1,ca45a05e8a..3507130f4c
---
a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@@ -46,9 -45,9 +46,10 @@@ import org.apache.accumulo.core.metadat
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.FutureLocationColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LastLocationColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+ import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
@@@ -103,8 -99,8 +104,9 @@@ public class MetadataConstraints implem
FutureLocationColumnFamily.NAME,
ClonedColumnFamily.NAME,
ExternalCompactionColumnFamily.NAME,
- CompactedColumnFamily.NAME,
- UpgraderDeprecatedConstants.ChoppedColumnFamily.NAME
++ CompactedColumnFamily.NAME,
+ UpgraderDeprecatedConstants.ChoppedColumnFamily.NAME,
+ MergedColumnFamily.NAME
);
// @formatter:on
diff --cc test/pom.xml
index c5eab3f771,7fbc514f9b..7231e1bada
--- a/test/pom.xml
+++ b/test/pom.xml
@@@ -198,10 -202,10 +198,14 @@@
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.opentest4j</groupId>
+ <artifactId>opentest4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
diff --cc test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
index a538d60bba,4d265a1120..cf37fb3369
--- a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
@@@ -20,9 -20,9 +20,10 @@@ package org.apache.accumulo.test.functi
import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.accumulo.test.util.FileMetadataUtil.printAndVerifyFileMetadata;
+ import static
org.apache.accumulo.test.util.FileMetadataUtil.verifyMergedMarkerCleared;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
@@@ -174,23 -98,9 +175,26 @@@ public class MergeIT extends AccumuloCl
c.tableOperations().flush(tableName, null, null, true);
c.tableOperations().merge(tableName, new Text("c1"), new Text("f1"));
assertEquals(8, c.tableOperations().listSplits(tableName).size());
+ // Verify that the MERGED marker was cleared
+ verifyMergedMarkerCleared(getServerContext(),
+ TableId.of(c.tableOperations().tableIdMap().get(tableName)));
+ try (Scanner s = c.createScanner(MetadataTable.NAME)) {
+ String tid = c.tableOperations().tableIdMap().get(tableName);
+ s.setRange(new Range(tid + ";g"));
+ TabletColumnFamily.PREV_ROW_COLUMN.fetch(s);
+ HostingColumnFamily.GOAL_COLUMN.fetch(s);
+ assertEquals(2, Iterables.size(s));
+ for (Entry<Key,Value> rows : s) {
+ if (TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(rows.getKey())) {
+ assertEquals("c",
TabletColumnFamily.decodePrevEndRow(rows.getValue()).toString());
+ } else if
(HostingColumnFamily.GOAL_COLUMN.hasColumns(rows.getKey())) {
+ assertEquals(TabletHostingGoal.ALWAYS,
+ TabletHostingGoalUtil.fromValue(rows.getValue()));
+ } else {
+ fail("Unknown column");
+ }
+ }
+ }
}
}
