This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new f1c33128bd Remove ZK compaction and compaction cancellation ids (#3963)
f1c33128bd is described below
commit f1c33128bdd18fd682f70daeb8c3f0fd8831be7f
Author: Dave Marion <dlmar...@apache.org>
AuthorDate: Fri Dec 1 09:38:30 2023 -0500
Remove ZK compaction and compaction cancellation ids (#3963)
Removed compact columns during upgrade and from code
Fixes #3584
---
.../java/org/apache/accumulo/core/Constants.java | 5 -
.../accumulo/core/metadata/schema/Ample.java | 2 -
.../core/metadata/schema/MetadataSchema.java | 3 -
.../core/metadata/schema/TabletMetadata.java | 16 +--
.../metadata/schema/TabletMetadataBuilder.java | 8 --
.../core/metadata/schema/TabletMutatorBase.java | 7 --
.../core/metadata/schema/TabletsMetadata.java | 4 -
.../core/metadata/schema/TabletMetadataTest.java | 10 +-
.../server/constraints/MetadataConstraints.java | 1 -
.../metadata/ConditionalTabletMutatorImpl.java | 12 +--
.../accumulo/server/tables/TableManager.java | 2 -
.../accumulo/server/util/ManagerMetadataUtil.java | 6 +-
.../server/manager/state/TabletManagementTest.java | 2 -
.../accumulo/manager/upgrade/Upgrader12to13.java | 120 ++++++++++++++++++++-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 2 +-
.../test/functional/AmpleConditionalWriterIT.java | 17 +--
.../accumulo/test/functional/SplitRecoveryIT.java | 2 +-
17 files changed, 135 insertions(+), 84 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java
b/core/src/main/java/org/apache/accumulo/core/Constants.java
index 84dece9030..9c8ecf7e5a 100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@ -44,11 +44,6 @@ public class Constants {
public static final String ZTABLE_STATE = "/state";
public static final String ZTABLE_FLUSH_ID = "/flush-id";
- // ELASTICITY_TODO delete from code and remove from ZK in upgrade
- public static final String ZTABLE_COMPACT_ID = "/compact-id";
-
- // ELASTICITY_TODO delete from code and remove from ZK in upgrade
- public static final String ZTABLE_COMPACT_CANCEL_ID = "/compact-cancel-id";
public static final String ZTABLE_NAMESPACE = "/namespace";
public static final String ZNAMESPACES = "/namespaces";
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index 5c68eba658..d8bb5d62e9 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -357,8 +357,6 @@ public interface Ample {
T deleteScan(StoredTabletFile path);
- T putCompactionId(long compactionId);
-
T putFlushId(long flushId);
T putLocation(Location location);
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
index 4ff3c2a170..6a7b75a679 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
@@ -242,9 +242,6 @@ public class MetadataSchema {
public static final String FLUSH_QUAL = "flush";
public static final ColumnFQ FLUSH_COLUMN = new ColumnFQ(NAME, new
Text(FLUSH_QUAL));
- // ELASTICITY_TODO remove this from code and remove it from metadata in
upgrade
- public static final String COMPACT_QUAL = "compact";
- public static final ColumnFQ COMPACT_COLUMN = new ColumnFQ(NAME, new
Text(COMPACT_QUAL));
/**
* Holds lock IDs to enable a sanity check to ensure that the TServer
writing to the metadata
* tablet is not dead
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index febc43ee45..8576d4c459 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -20,7 +20,6 @@ package org.apache.accumulo.core.metadata.schema;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily.GOAL_QUAL;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily.REQUESTED_QUAL;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_QUAL;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_QUAL;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_QUAL;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.OPID_QUAL;
@@ -116,7 +115,6 @@ public class TabletMetadata {
private SortedMap<Key,Value> keyValues;
private OptionalLong flush = OptionalLong.empty();
private List<LogEntry> logs;
- private OptionalLong compact = OptionalLong.empty();
private Double splitRatio = null;
private Map<ExternalCompactionId,ExternalCompactionMetadata> extCompactions;
private boolean merged;
@@ -147,7 +145,6 @@ public class TabletMetadata {
CLONED,
FLUSH_ID,
LOGS,
- COMPACT_ID,
SPLIT_RATIO,
SUSPEND,
ECOMP,
@@ -372,11 +369,6 @@ public class TabletMetadata {
return flush;
}
- public OptionalLong getCompactId() {
- ensureFetched(ColumnType.COMPACT_ID);
- return compact;
- }
-
public Double getSplitRatio() {
ensureFetched(ColumnType.SPLIT_RATIO);
return splitRatio;
@@ -411,9 +403,8 @@ public class TabletMetadata {
.append("fetchedCols", fetchedCols).append("extent",
extent).append("last", last)
.append("suspend", suspend).append("dirName", dirName).append("time",
time)
.append("cloned", cloned).append("flush", flush).append("logs", logs)
- .append("compact", compact).append("splitRatio", splitRatio)
- .append("extCompactions", extCompactions).append("goal", goal)
- .append("onDemandHostingRequested", onDemandHostingRequested)
+ .append("splitRatio", splitRatio).append("extCompactions",
extCompactions)
+ .append("goal", goal).append("onDemandHostingRequested",
onDemandHostingRequested)
.append("operationId", operationId).append("selectedFiles",
selectedFiles)
.append("futureAndCurrentLocationSet",
futureAndCurrentLocationSet).toString();
}
@@ -514,9 +505,6 @@ public class TabletMetadata {
case FLUSH_QUAL:
te.flush = OptionalLong.of(Long.parseLong(val));
break;
- case COMPACT_QUAL:
- te.compact = OptionalLong.of(Long.parseLong(val));
- break;
case OPID_QUAL:
te.setOperationIdOnce(val, suppressLocationError);
break;
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java
index b9c2d9ae55..f2bd30f15e 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java
@@ -19,7 +19,6 @@
package org.apache.accumulo.core.metadata.schema;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.COMPACTED;
-import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.COMPACT_ID;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.DIR;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
@@ -115,13 +114,6 @@ public class TabletMetadataBuilder implements
Ample.TabletUpdates<TabletMetadata
throw new UnsupportedOperationException();
}
- @Override
- public TabletMetadataBuilder putCompactionId(long compactionId) {
- fetched.add(COMPACT_ID);
- internalBuilder.putCompactionId(compactionId);
- return this;
- }
-
@Override
public TabletMetadataBuilder putFlushId(long flushId) {
fetched.add(FLUSH_ID);
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
index 86d0569fec..ea13271f6f 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java
@@ -128,13 +128,6 @@ public abstract class TabletMutatorBase<T extends
Ample.TabletUpdates<T>>
return getThis();
}
- @Override
- public T putCompactionId(long compactionId) {
- Preconditions.checkState(updatesEnabled, "Cannot make updates after
calling mutate.");
- ServerColumnFamily.COMPACT_COLUMN.put(mutation, new
Value(Long.toString(compactionId)));
- return getThis();
- }
-
@Override
public T putFlushId(long flushId) {
Preconditions.checkState(updatesEnabled, "Cannot make updates after
calling mutate.");
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index 997d082084..4e28e46e9d 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@ -22,7 +22,6 @@ import static com.google.common.base.Preconditions.checkState;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.OPID_COLUMN;
@@ -301,9 +300,6 @@ public class TabletsMetadata implements
Iterable<TabletMetadata>, AutoCloseable
case CLONED:
families.add(ClonedColumnFamily.NAME);
break;
- case COMPACT_ID:
- qualifiers.add(COMPACT_COLUMN);
- break;
case DIR:
qualifiers.add(DIRECTORY_COLUMN);
break;
diff --git
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index b8d73c3e8f..7c4e50266e 100644
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@ -23,7 +23,6 @@ import static
org.apache.accumulo.core.fate.FateTxId.formatTid;
import static org.apache.accumulo.core.metadata.StoredTabletFile.serialize;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily.MERGED_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily.MERGED_VALUE;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
@@ -91,7 +90,6 @@ public class TabletMetadataTest {
Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
- COMPACT_COLUMN.put(mutation, new Value("5"));
DIRECTORY_COLUMN.put(mutation, new Value("t-0001757"));
FLUSH_COLUMN.put(mutation, new Value("6"));
TIME_COLUMN.put(mutation, new Value("M123456789"));
@@ -134,7 +132,6 @@ public class TabletMetadataTest {
EnumSet.allOf(ColumnType.class), true, false);
assertEquals("OK", tm.getCloned());
- assertEquals(5L, tm.getCompactId().getAsLong());
assertEquals("t-0001757", tm.getDirName());
assertEquals(extent.endRow(), tm.getEndRow());
assertEquals(extent, tm.getExtent());
@@ -351,14 +348,12 @@ public class TabletMetadataTest {
TabletMetadata tm =
TabletMetadata.builder(extent).putHostingGoal(TabletHostingGoal.NEVER)
.putLocation(Location.future(ser1)).putFile(sf1, dfv1).putFile(sf2,
dfv2)
- .putCompactionId(23).putBulkFile(rf1, 25).putBulkFile(rf2,
35).putFlushId(27)
-
.putDirName("dir1").putScan(sf3).putScan(sf4).putCompacted(17).putCompacted(23)
- .build(ECOMP, HOSTING_REQUESTED, MERGED);
+ .putBulkFile(rf1, 25).putBulkFile(rf2,
35).putFlushId(27).putDirName("dir1").putScan(sf3)
+ .putScan(sf4).putCompacted(17).putCompacted(23).build(ECOMP,
HOSTING_REQUESTED, MERGED);
assertEquals(extent, tm.getExtent());
assertEquals(TabletHostingGoal.NEVER, tm.getHostingGoal());
assertEquals(Location.future(ser1), tm.getLocation());
- assertEquals(23L, tm.getCompactId().orElse(-1));
assertEquals(27L, tm.getFlushId().orElse(-1));
assertEquals(Map.of(sf1, dfv1, sf2, dfv2), tm.getFilesMap());
assertEquals(Map.of(rf1.insert(), 25L, rf2.insert(), 35L), tm.getLoaded());
@@ -380,7 +375,6 @@ public class TabletMetadataTest {
assertNull(tm2.getLocation());
assertThrows(IllegalStateException.class, tm2::getFiles);
assertThrows(IllegalStateException.class, tm2::getHostingGoal);
- assertThrows(IllegalStateException.class, tm2::getCompactId);
assertThrows(IllegalStateException.class, tm2::getFlushId);
assertThrows(IllegalStateException.class, tm2::getFiles);
assertThrows(IllegalStateException.class, tm2::getLogs);
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index d751cda3e9..190f7e0e3b 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@ -89,7 +89,6 @@ public class MetadataConstraints implements Constraint {
ServerColumnFamily.TIME_COLUMN,
ServerColumnFamily.LOCK_COLUMN,
ServerColumnFamily.FLUSH_COLUMN,
- ServerColumnFamily.COMPACT_COLUMN,
ServerColumnFamily.OPID_COLUMN,
HostingColumnFamily.GOAL_COLUMN,
HostingColumnFamily.REQUESTED_COLUMN,
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
index 83a988ca38..03482ce4fc 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
@@ -20,9 +20,7 @@
package org.apache.accumulo.server.metadata;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily.GOAL_COLUMN;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.OPID_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.SELECTED_COLUMN;
@@ -49,6 +47,7 @@ import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Bu
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CompactedColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
@@ -163,15 +162,6 @@ public class ConditionalTabletMutatorImpl extends
TabletMutatorBase<Ample.Condit
switch (type) {
case PREV_ROW:
throw new IllegalStateException("PREV_ROW already set from Extent");
- case COMPACT_ID: {
- Condition c =
- new Condition(COMPACT_COLUMN.getColumnFamily(),
COMPACT_COLUMN.getColumnQualifier());
- if (tabletMetadata.getCompactId().isPresent()) {
- c =
c.setValue(Long.toString(tabletMetadata.getCompactId().getAsLong()));
- }
- mutation.addCondition(c);
- }
- break;
case LOGS: {
Condition c = SetEqualityIterator.createCondition(new
HashSet<>(tabletMetadata.getLogs()),
logEntry ->
logEntry.getColumnQualifier().toString().getBytes(UTF_8),
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
index 767fc515d6..11c95c9dcc 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
@@ -104,8 +104,6 @@ public class TableManager {
zoo.putPersistentData(zTablePath + Constants.ZTABLE_NAME,
tableName.getBytes(UTF_8),
existsPolicy);
zoo.putPersistentData(zTablePath + Constants.ZTABLE_FLUSH_ID, ZERO_BYTE,
existsPolicy);
- zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_ID, ZERO_BYTE,
existsPolicy);
- zoo.putPersistentData(zTablePath + Constants.ZTABLE_COMPACT_CANCEL_ID,
ZERO_BYTE, existsPolicy);
zoo.putPersistentData(zTablePath + Constants.ZTABLE_STATE,
state.name().getBytes(UTF_8),
existsPolicy);
var propKey = TablePropKey.of(instanceId, tableId);
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java
index e252512843..10ff4e9e55 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java
@@ -61,7 +61,7 @@ public class ManagerMetadataUtil {
public static void addNewTablet(ServerContext context, KeyExtent extent,
String dirName,
TServerInstance tServerInstance, Map<StoredTabletFile,DataFileValue>
datafileSizes,
Map<Long,? extends Collection<ReferencedTabletFile>> bulkLoadedFiles,
MetadataTime time,
- long lastFlushID, long lastCompactID, ServiceLock zooLock) {
+ long lastFlushID, ServiceLock zooLock) {
// ELASTICITY_TODO intentionally not using conditional mutations for this
code because its only
// called when tablets split. Tablet splitting will drastically change, so
there is no need to
@@ -77,10 +77,6 @@ public class ManagerMetadataUtil {
tablet.putFlushId(lastFlushID);
}
- if (lastCompactID > 0) {
- tablet.putCompactionId(lastCompactID);
- }
-
if (tServerInstance != null) {
tablet.putLocation(Location.current(tServerInstance));
tablet.deleteLocation(Location.future(tServerInstance));
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java
b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java
index 2fd6df2978..f1223ee71b 100644
---
a/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletManagementTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.accumulo.server.manager.state;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.COMPACT_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
@@ -73,7 +72,6 @@ public class TabletManagementTest {
Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
- COMPACT_COLUMN.put(mutation, new Value("5"));
DIRECTORY_COLUMN.put(mutation, new Value("t-0001757"));
FLUSH_COLUMN.put(mutation, new Value("6"));
TIME_COLUMN.put(mutation, new Value("M123456789"));
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
index be140d845e..9c657d27d1 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
@@ -18,29 +18,41 @@
*/
package org.apache.accumulo.manager.upgrade;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.RESERVED_PREFIX;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.admin.TabletHostingGoal;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.Ample.TabletsMutator;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ExternalCompactionColumnFamily;
+import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.schema.Section;
+import org.apache.accumulo.core.util.ColumnFQ;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.store.TablePropKey;
import org.apache.accumulo.server.util.PropUtil;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,26 +62,128 @@ public class Upgrader12to13 implements Upgrader {
private static final Logger LOG =
LoggerFactory.getLogger(Upgrader12to13.class);
+ private static final ColumnFQ COMPACT_COL =
+ new ColumnFQ(MetadataSchema.TabletsSection.ServerColumnFamily.NAME, new
Text("compact"));
+
@Override
public void upgradeZookeeper(ServerContext context) {
- LOG.info("setting root table stored hosting goal");
+ LOG.info("Setting root table stored hosting goal");
addHostingGoalToRootTable(context);
+ LOG.info("Removing compact-id paths from ZooKeeper");
+ removeZKCompactIdPaths(context);
+ LOG.info("Removing compact columns from root tablet");
+ removeCompactColumnsFromRootTabletMetadata(context);
}
@Override
public void upgradeRoot(ServerContext context) {
- LOG.info("setting metadata table hosting goal");
+ LOG.info("Setting metadata table hosting goal");
addHostingGoalToMetadataTable(context);
+ LOG.info("Removing MetadataBulkLoadFilter iterator from root table");
removeMetaDataBulkLoadFilter(context, RootTable.ID);
+ LOG.info("Removing compact columns from metadata tablets");
+ removeCompactColumnsFromTable(context, RootTable.NAME);
}
@Override
public void upgradeMetadata(ServerContext context) {
- LOG.info("setting hosting goal on user tables");
+ LOG.info("Setting hosting goal on user tables");
addHostingGoalToUserTables(context);
+ LOG.info("Deleting external compaction final states from user tables");
deleteExternalCompactionFinalStates(context);
+ LOG.info("Deleting external compaction from user tables");
deleteExternalCompactions(context);
+ LOG.info("Removing MetadataBulkLoadFilter iterator from metadata table");
removeMetaDataBulkLoadFilter(context, MetadataTable.ID);
+ LOG.info("Removing compact columns from user tables");
+ removeCompactColumnsFromTable(context, MetadataTable.NAME);
+ }
+
+ private void removeCompactColumnsFromRootTabletMetadata(ServerContext
context) {
+ var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
+
+ try {
+ var zrw = context.getZooReaderWriter();
+ Stat stat = new Stat();
+ byte[] rootData = zrw.getData(rootBase, stat);
+
+ String json = new String(rootData, UTF_8);
+
+ var rtm = new RootTabletMetadata(json);
+
+ ArrayList<Mutation> mutations = new ArrayList<>();
+ for (Map.Entry<Key,Value> entry : rtm.toKeyValues().entrySet()) {
+ var key = entry.getKey();
+
+ if (COMPACT_COL.hasColumns(key)) {
+ var row = key.getRow();
+ Preconditions.checkState(key.getColumnVisibilityData().length() == 0,
+ "Expected empty visibility, saw %s ",
key.getColumnVisibilityData());
+ Mutation m = new Mutation(row);
+ COMPACT_COL.putDelete(m);
+ mutations.add(m);
+ }
+ }
+
+ Preconditions.checkState(mutations.size() <= 1);
+
+ if (!mutations.isEmpty()) {
+ LOG.info("Root metadata in ZooKeeper before upgrade: {}", json);
+ rtm.update(mutations.get(0));
+ zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8),
stat.getVersion());
+ LOG.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
+ }
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Could not read root metadata from ZooKeeper due to interrupt", ex);
+ } catch (KeeperException ex) {
+ throw new IllegalStateException(
+ "Could not read or write root metadata in ZooKeeper because of
ZooKeeper exception", ex);
+ }
+
+ }
+
+ private void removeCompactColumnsFromTable(ServerContext context, String
tableName) {
+
+ try (var scanner = context.createScanner(tableName);
+ var writer = context.createBatchWriter(tableName)) {
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
+ COMPACT_COL.fetch(scanner);
+
+ for (Map.Entry<Key,Value> entry : scanner) {
+ var key = entry.getKey();
+ if (COMPACT_COL.hasColumns(key)) {
+ var row = key.getRow();
+ Preconditions.checkState(key.getColumnVisibilityData().length() == 0,
+ "Expected empty visibility, saw %s ",
key.getColumnVisibilityData());
+ Mutation m = new Mutation(row);
+ COMPACT_COL.putDelete(m);
+ writer.addMutation(m);
+ }
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private void removeZKCompactIdPaths(ServerContext context) {
+ final String ZTABLE_COMPACT_ID = "/compact-id";
+ final String ZTABLE_COMPACT_CANCEL_ID = "/compact-cancel-id";
+
+ for (Entry<String,String> e :
context.tableOperations().tableIdMap().entrySet()) {
+ final String tName = e.getKey();
+ final String tId = e.getValue();
+ final String zTablePath = Constants.ZROOT + "/" +
context.getInstanceID().canonical()
+ + Constants.ZTABLES + "/" + tId;
+ try {
+ context.getZooReaderWriter().delete(zTablePath + ZTABLE_COMPACT_ID);
+ context.getZooReaderWriter().delete(zTablePath +
ZTABLE_COMPACT_CANCEL_ID);
+ } catch (KeeperException | InterruptedException e1) {
+ throw new IllegalStateException(
+ "Error removing compaction ids from ZooKeeper for table: " +
tName);
+ }
+ }
}
private void removeMetaDataBulkLoadFilter(ServerContext context, TableId
tableId) {
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 8eadceb9b8..4a9358ca26 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -941,7 +941,7 @@ public class Tablet extends TabletBase {
try {
var tabletMeta = context.getAmple().readTablet(extent, ColumnType.FILES,
ColumnType.LOGS,
- ColumnType.ECOMP, ColumnType.PREV_ROW, ColumnType.FLUSH_ID,
ColumnType.COMPACT_ID);
+ ColumnType.ECOMP, ColumnType.PREV_ROW, ColumnType.FLUSH_ID);
if (tabletMeta == null) {
String msg = "Closed tablet " + extent + " not found in metadata";
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
index 325970c03f..0633b934f8 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
@@ -791,30 +791,33 @@ public class AmpleConditionalWriterIT extends
AccumuloClusterHarness {
var loc = rootMeta.getLocation();
assertEquals(LocationType.CURRENT, loc.getType());
- assertFalse(rootMeta.getCompactId().isPresent());
+ assertNull(rootMeta.getOperationId());
+
+ TabletOperationId opid =
TabletOperationId.from(TabletOperationType.MERGING, 7);
var ctmi = new ConditionalTabletsMutatorImpl(context);
ctmi.mutateTablet(RootTable.EXTENT).requireAbsentOperation().requireAbsentLocation()
- .putCompactionId(7).submit(tm -> false);
+ .putOperation(opid).submit(tm -> false);
var results = ctmi.process();
assertEquals(Status.REJECTED, results.get(RootTable.EXTENT).getStatus());
-
assertFalse(context.getAmple().readTablet(RootTable.EXTENT).getCompactId().isPresent());
+
assertNull(context.getAmple().readTablet(RootTable.EXTENT).getOperationId());
ctmi = new ConditionalTabletsMutatorImpl(context);
ctmi.mutateTablet(RootTable.EXTENT).requireAbsentOperation()
-
.requireLocation(Location.future(loc.getServerInstance())).putCompactionId(7)
+
.requireLocation(Location.future(loc.getServerInstance())).putOperation(opid)
.submit(tm -> false);
results = ctmi.process();
assertEquals(Status.REJECTED, results.get(RootTable.EXTENT).getStatus());
-
assertFalse(context.getAmple().readTablet(RootTable.EXTENT).getCompactId().isPresent());
+
assertNull(context.getAmple().readTablet(RootTable.EXTENT).getOperationId());
ctmi = new ConditionalTabletsMutatorImpl(context);
ctmi.mutateTablet(RootTable.EXTENT).requireAbsentOperation()
-
.requireLocation(Location.current(loc.getServerInstance())).putCompactionId(7)
+
.requireLocation(Location.current(loc.getServerInstance())).putOperation(opid)
.submit(tm -> false);
results = ctmi.process();
assertEquals(Status.ACCEPTED, results.get(RootTable.EXTENT).getStatus());
- assertEquals(7L,
context.getAmple().readTablet(RootTable.EXTENT).getCompactId().getAsLong());
+ assertEquals(opid.canonical(),
+
context.getAmple().readTablet(RootTable.EXTENT).getOperationId().canonical());
}
@Test
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
index 7816b156d5..b457f93a0c 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
@@ -224,7 +224,7 @@ public class SplitRecoveryIT extends ConfigurableMacBase {
Map<Long,List<ReferencedTabletFile>> bulkFiles =
getBulkFilesLoaded(context, high);
ManagerMetadataUtil.addNewTablet(context, low, "lowDir", instance,
lowDatafileSizes,
- bulkFiles, new MetadataTime(0, TimeType.LOGICAL), -1L, -1L, zl);
+ bulkFiles, new MetadataTime(0, TimeType.LOGICAL), -1L, zl);
}
if (steps >= 2) {
MetadataTableUtil.finishSplit(high, highDatafileSizes,
highDatafilesToRemove, context, zl);
