This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 3aaafa4320 Handle Fate table creation on upgrade (#4760)
3aaafa4320 is described below
commit 3aaafa4320ce735363ab1f853daae2078d9c2329
Author: Daniel Roberts <[email protected]>
AuthorDate: Thu Jul 25 17:07:36 2024 -0400
Handle Fate table creation on upgrade (#4760)
Upgrader now calls the same code for fate table creation as the init code
---
.../server/init/FileSystemInitializer.java | 42 +++++++---------------
.../accumulo/manager/upgrade/Upgrader12to13.java | 32 ++++++++++++-----
2 files changed, 37 insertions(+), 37 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java
b/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java
index 7012fa93b7..066a48161c 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java
@@ -18,21 +18,16 @@
*/
package org.apache.accumulo.server.init;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.AVAILABILITY_COLUMN;
-import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN;
-
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
+import java.util.stream.Collectors;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.admin.TabletAvailability;
import org.apache.accumulo.core.client.admin.TimeType;
-import org.apache.accumulo.core.clientImpl.TabletAvailabilityUtil;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.crypto.CryptoFactoryLoader;
@@ -40,6 +35,7 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
import org.apache.accumulo.core.metadata.AccumuloTable;
@@ -48,9 +44,9 @@ import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.MetadataTime;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.spi.crypto.CryptoService;
import org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment;
-import org.apache.accumulo.core.util.ColumnFQ;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.store.TablePropKey;
import org.apache.accumulo.server.fs.VolumeChooserEnvironmentImpl;
@@ -91,28 +87,16 @@ public class FileSystemInitializer {
this.extent = new
Text(MetadataSchema.TabletsSection.encodeRow(this.tableId, this.endRow));
}
- private TreeMap<Key,Value> createEntries() {
- TreeMap<Key,Value> sorted = new TreeMap<>();
- Value EMPTY_SIZE = new DataFileValue(0, 0).encodeAsValue();
- sorted.put(new Key(this.extent, DIRECTORY_COLUMN.getColumnFamily(),
- DIRECTORY_COLUMN.getColumnQualifier(), 0), new Value(this.dirName));
- sorted.put(
- new Key(this.extent, TIME_COLUMN.getColumnFamily(),
TIME_COLUMN.getColumnQualifier(), 0),
- new Value(new MetadataTime(0, TimeType.LOGICAL).encode()));
- sorted.put(
- new Key(this.extent, PREV_ROW_COLUMN.getColumnFamily(),
- PREV_ROW_COLUMN.getColumnQualifier(), 0),
-
MetadataSchema.TabletsSection.TabletColumnFamily.encodePrevEndRow(this.prevEndRow));
- sorted.put(
- new Key(this.extent, AVAILABILITY_COLUMN.getColumnFamily(),
- AVAILABILITY_COLUMN.getColumnQualifier(), 0),
- TabletAvailabilityUtil.toValue(TabletAvailability.HOSTED));
- for (String file : this.files) {
- var col =
- new
ColumnFQ(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME, new
Text(file));
- sorted.put(new Key(extent, col.getColumnFamily(),
col.getColumnQualifier(), 0), EMPTY_SIZE);
+ private Map<Key,Value> createEntries() {
+ KeyExtent keyExtent = new KeyExtent(tableId, endRow, prevEndRow);
+ var builder = TabletMetadata.builder(keyExtent).putDirName(dirName)
+ .putTime(new MetadataTime(0, TimeType.LOGICAL))
+
.putTabletAvailability(TabletAvailability.HOSTED).putPrevEndRow(prevEndRow);
+ for (String file : files) {
+ builder.putFile(new ReferencedTabletFile(new Path(file)).insert(), new
DataFileValue(0, 0));
}
- return sorted;
+ return builder.build().getKeyValues().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
public Mutation createMutation() {
@@ -175,7 +159,7 @@ public class FileSystemInitializer {
// populate the root tablet with info about the metadata table's two
initial tablets
InitialTablet tablesTablet =
new InitialTablet(AccumuloTable.METADATA.tableId(),
TABLE_TABLETS_TABLET_DIR, null,
- SPLIT_POINT, StoredTabletFile.of(new
Path(metadataFileName)).getMetadata());
+ SPLIT_POINT, StoredTabletFile.of(new
Path(metadataFileName)).getMetadataPath());
InitialTablet defaultTablet = new
InitialTablet(AccumuloTable.METADATA.tableId(),
defaultMetadataTabletDirName, SPLIT_POINT, null);
createMetadataFile(fs, rootTabletFileUri, tablesTablet, defaultTablet);
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
index 6d89ebc8ca..825a5ebd99 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
@@ -23,21 +23,23 @@ import static
org.apache.accumulo.core.metadata.RootTable.ZROOT_TABLET;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.RESERVED_PREFIX;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Upgrade12to13.COMPACT_COL;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.TabletAvailability;
-import org.apache.accumulo.core.clientImpl.Namespace;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
-import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.Ample.TabletsMutator;
@@ -51,7 +53,9 @@ import org.apache.accumulo.core.schema.Section;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.store.TablePropKey;
-import org.apache.accumulo.server.tables.TableManager;
+import org.apache.accumulo.server.init.FileSystemInitializer;
+import org.apache.accumulo.server.init.InitialConfiguration;
+import org.apache.accumulo.server.init.ZooKeeperInitializer;
import org.apache.accumulo.server.util.PropUtil;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
@@ -120,13 +124,25 @@ public class Upgrader12to13 implements Upgrader {
}
private void createFateTable(ServerContext context) {
+ ZooKeeperInitializer zkInit = new ZooKeeperInitializer();
+ zkInit.initFateTableState(context);
+
try {
- TableManager.prepareNewTableState(context, AccumuloTable.FATE.tableId(),
- Namespace.ACCUMULO.id(), AccumuloTable.FATE.tableName(),
TableState.ONLINE,
- ZooUtil.NodeExistsPolicy.SKIP);
- } catch (KeeperException | InterruptedException e) {
- throw new IllegalStateException("Error creating table: " +
AccumuloTable.FATE.tableName(), e);
+ FileSystemInitializer initializer = new FileSystemInitializer(
+ new InitialConfiguration(context.getHadoopConf(),
context.getSiteConfiguration()));
+ FileSystemInitializer.InitialTablet fateTableTableTablet =
+ initializer.createFateRefTablet(context);
+ // Add references to the Metadata Table
+ try (BatchWriter writer =
context.createBatchWriter(AccumuloTable.METADATA.tableName())) {
+ writer.addMutation(fateTableTableTablet.createMutation());
+ } catch (MutationsRejectedException | TableNotFoundException e) {
+ LOG.error("Failed to write tablet refs to metadata table");
+ throw new RuntimeException(e);
+ }
+ } catch (IOException e) {
+ LOG.error("Problem attempting to create Fate table", e);
}
+ LOG.info("Created Fate table");
}
private void removeCompactColumnsFromRootTabletMetadata(ServerContext
context) {
