This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 51004f656a9b4eaa4f6a8238ca3b0b6e34545bc1
Merge: 2f894ae50b dc0d4f3132
Author: Dave Marion <dlmar...@apache.org>
AuthorDate: Tue Nov 28 21:32:24 2023 +0000
Merge branch 'main' into elasticity
.../core/metadata/schema/RootTabletMetadata.java | 67 ++-----
.../metadata/schema/RootTabletMetadataTest.java | 119 -------------
.../accumulo/manager/upgrade/Upgrader11to12.java | 55 ++++--
.../manager/upgrade/Upgrader11to12Test.java | 195 ++++++++++-----------
4 files changed, 140 insertions(+), 296 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
index c5144bb9c4,19285bebdb..f96d0f77c2
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
@@@ -40,9 -39,8 +40,6 @@@ import org.apache.accumulo.core.metadat
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.FutureLocationColumnFamily;
import org.apache.hadoop.io.Text;
--import org.slf4j.Logger;
--import org.slf4j.LoggerFactory;
/**
* This class is used to serialize and deserialize root tablet metadata using
GSon. The only data
@@@ -52,7 -50,7 +49,6 @@@
*/
public class RootTabletMetadata {
-- private static final Logger log =
LoggerFactory.getLogger(RootTabletMetadata.class);
private static final CharsetDecoder UTF8_error_detecting_decoder =
UTF_8.newDecoder();
private static final Predicate<Entry<String,TreeMap<String,String>>>
isLocationCF = e -> {
String fam = e.getKey();
@@@ -195,31 -155,24 +153,30 @@@
}
}
- public Stream<SimpleImmutableEntry<Key,Value>> toKeyValues() {
++ public Stream<SimpleImmutableEntry<Key,Value>> getKeyValues() {
+ String row = RootTable.EXTENT.toMetaRow().toString();
+ return data.columnValues.entrySet().stream()
+ .flatMap(famToQualVal -> famToQualVal.getValue().entrySet().stream()
+ .map(qualVal -> new SimpleImmutableEntry<>(
+ new Key(row, famToQualVal.getKey(), qualVal.getKey(), 1),
+ new Value(qualVal.getValue()))));
+ }
+
+ public SortedMap<Key,Value> toKeyValues() {
+ TreeMap<Key,Value> metamap = new TreeMap<>();
+ getKeyValues().forEach(e -> metamap.put(e.getKey(), e.getValue()));
+ return metamap;
+ }
+
/**
* Convert this class to a {@link TabletMetadata}
*/
public TabletMetadata toTabletMetadata() {
- Stream<SimpleImmutableEntry<Key,Value>> entries = getKeyValues();
- return TabletMetadata.convertRow(entries.iterator(),
+ // use a stream so we don't have to re-sort in a new TreeMap<Key,Value>
structure
- return TabletMetadata.convertRow(toKeyValues().iterator(),
- EnumSet.allOf(TabletMetadata.ColumnType.class), false);
++ return TabletMetadata.convertRow(getKeyValues().iterator(),
+ EnumSet.allOf(TabletMetadata.ColumnType.class), false, false);
}
- private Stream<SimpleImmutableEntry<Key,Value>> getKeyValues() {
- String row = RootTable.EXTENT.toMetaRow().toString();
- Stream<SimpleImmutableEntry<Key,Value>> entries =
data.columnValues.entrySet().stream()
- .flatMap(famToQualVal -> famToQualVal.getValue().entrySet().stream()
- .map(qualVal -> new SimpleImmutableEntry<>(
- new Key(row, famToQualVal.getKey(), qualVal.getKey(), 1),
- new Value(qualVal.getValue()))));
- return entries;
- }
-
public static boolean needsUpgrade(final String json) {
return Data.needsUpgrade(json);
}
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java
index 72127a4d4c,90a6eb4b62..6673c0dfdd
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java
@@@ -69,9 -77,21 +77,21 @@@ public class Upgrader11to12 implements
byte[] rootData = zrw.getData(rootBase, stat);
String json = new String(rootData, UTF_8);
- if (RootTabletMetadata.needsUpgrade(json)) {
+
+ var rtm = new RootTabletMetadata(json);
+
+ TreeMap<Key,Value> entries = new TreeMap<>();
- rtm.toKeyValues().filter(e ->
UPGRADE_FAMILIES.contains(e.getKey().getColumnFamily()))
++ rtm.getKeyValues().filter(e ->
UPGRADE_FAMILIES.contains(e.getKey().getColumnFamily()))
+ .forEach(entry -> entries.put(entry.getKey(), entry.getValue()));
+ ArrayList<Mutation> mutations = new ArrayList<>();
+
+ processReferences(mutations::add, entries.entrySet(),
"root_table_metadata");
+
+ Preconditions.checkState(mutations.size() <= 1);
+
+ if (!mutations.isEmpty()) {
log.info("Root metadata in ZooKeeper before upgrade: {}", json);
- RootTabletMetadata rtm = RootTabletMetadata.upgrade(json);
+ rtm.update(mutations.get(0));
zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8),
stat.getVersion());
log.info("Root metadata in ZooKeeper after upgrade: {}",
rtm.toJson());
}
diff --cc
server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java
index ee45534dab,9fedfe5813..01863bcd3c
---
a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java
+++
b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java
@@@ -458,4 -390,59 +390,59 @@@ public class Upgrader11to12Test
assertFalse(upgrader.fileNeedsConversion(s31));
assertFalse(upgrader.fileNeedsConversion(s31_untrimmed));
}
+
+ @Test
+ public void convertRoot1File() {
+ String root21ZkData =
+
"{\"version\":1,\"columnValues\":{\"file\":{\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/A000000v.rf\":\"1368,61\"},\"last\":{\"100025091780006\":\"localhost:9997\"},\"loc\":{\"100025091780006\":\"localhost:9997\"},\"srv\":{\"dir\":\"root_tablet\",\"flush\":\"3\",\"lock\":\"tservers/localhost:9997/zlock#9db8961a-4ee9-400e-8e80-3353148baadd#0000000000$100025091780006\",\"time\":\"L53\"},\"~tab\":{\"~pr\":\"\\u0000\"}}}";
+
+ RootTabletMetadata rtm = new RootTabletMetadata(root21ZkData);
+ ArrayList<Mutation> mutations = new ArrayList<>();
+ Upgrader11to12 upgrader = new Upgrader11to12();
+ upgrader.processReferences(mutations::add,
- rtm.toKeyValues().filter(e ->
UPGRADE_FAMILIES.contains(e.getKey().getColumnFamily()))
++ rtm.getKeyValues().filter(e ->
UPGRADE_FAMILIES.contains(e.getKey().getColumnFamily()))
+ .collect(Collectors.toList()),
+ "accumulo.metadata");
+ assertEquals(1, mutations.size());
+ var mutation = mutations.get(0);
+ rtm.update(mutation);
+
+ LOG.debug("converted column values: {}",
rtm.toTabletMetadata().getFiles());
+
+ var files = rtm.toTabletMetadata().getFiles();
+ LOG.info("FILES: {}", rtm.toTabletMetadata().getFilesMap());
+
+ assertEquals(1, files.size());
+ assertTrue(files.contains(StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/+r/root_tablet/A000000v.rf"))));
+ }
+
+ @Test
+ public void convertRoot2Files() {
+ String root212ZkData2Files =
+
"{\"version\":1,\"columnValues\":{\"file\":{\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/00000_00000.rf\":\"0,0\",\"hdfs://localhost:8020/accumulo/tables/+r/root_tablet/F000000c.rf\":\"926,18\"},\"last\":{\"10001a84d7d0005\":\"localhost:9997\"},\"loc\":{\"10001a84d7d0005\":\"localhost:9997\"},\"srv\":{\"dir\":\"root_tablet\",\"flush\":\"2\",\"lock\":\"tservers/localhost:9997/zlock#d21adaa4-0f97-4004-9ff8-cce9dbb6687f#0000000000$10001a84d7d0005\",\"time\":\"L6\"},\"~tab
[...]
+
+ RootTabletMetadata rtm = new RootTabletMetadata(root212ZkData2Files);
+ ArrayList<Mutation> mutations = new ArrayList<>();
+ Upgrader11to12 upgrader = new Upgrader11to12();
+ upgrader.processReferences(mutations::add,
- rtm.toKeyValues().filter(e ->
UPGRADE_FAMILIES.contains(e.getKey().getColumnFamily()))
++ rtm.getKeyValues().filter(e ->
UPGRADE_FAMILIES.contains(e.getKey().getColumnFamily()))
+ .collect(Collectors.toList()),
+ "accumulo.metadata");
+ assertEquals(1, mutations.size());
+ var mutation = mutations.get(0);
+ rtm.update(mutation);
+
+ LOG.debug("converted column values: {}", rtm.toTabletMetadata());
+
+ var files = rtm.toTabletMetadata().getFiles();
+ LOG.info("FILES: {}", rtm.toTabletMetadata().getFilesMap());
+
+ assertEquals(2, files.size());
+ assertTrue(files.contains(StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/+r/root_tablet/00000_00000.rf"))));
+ assertTrue(files.contains(StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/+r/root_tablet/F000000c.rf"))));
+ }
+
}
