This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new e4f1f3c79f Update AmpleConditionalWriteIT to use new file metadata
format (#3833)
e4f1f3c79f is described below
commit e4f1f3c79ff621727bb2f9e8d342e904e709f664
Author: Christopher L. Shannon <[email protected]>
AuthorDate: Wed Oct 11 16:15:49 2023 -0400
Update AmpleConditionalWriteIT to use new file metadata format (#3833)
After the changes with no-chop merge the StoredTabletFile metadata is
now a json string that includes a range along with the path. This change
updates the test to serialize the path into the new format with an
infinite range.
Co-authored-by: EdColeman <[email protected]>
---
.../test/functional/AmpleConditionalWriterIT.java | 57 +++++++++++-----------
1 file changed, 29 insertions(+), 28 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
index 5eb9078192..81883b0e2b 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
@@ -28,6 +28,7 @@ import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.TIME;
+import static org.apache.accumulo.core.util.LazySingletons.GSON;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
@@ -74,6 +75,7 @@ import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.server.metadata.ConditionalTabletsMutatorImpl;
import org.apache.accumulo.server.zookeeper.TransactionWatcher;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -182,14 +184,14 @@ public class AmpleConditionalWriterIT extends
AccumuloClusterHarness {
var context = cluster.getServerContext();
- var stf1 = new StoredTabletFile(
-
"hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000070.rf");
- var stf2 = new StoredTabletFile(
-
"hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000071.rf");
- var stf3 = new StoredTabletFile(
-
"hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000072.rf");
- var stf4 = new StoredTabletFile(
-
"hdfs://localhost:8020/accumulo/tables/2a/default_tablet/C0000073.rf");
+ var stf1 = StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000070.rf"));
+ var stf2 = StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000071.rf"));
+ var stf3 = StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000072.rf"));
+ var stf4 = StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/C0000073.rf"));
var dfv = new DataFileValue(100, 100);
System.out.println(context.getAmple().readTablet(e1).getLocation());
@@ -279,8 +281,8 @@ public class AmpleConditionalWriterIT extends
AccumuloClusterHarness {
TransactionWatcher.ZooArbitrator.start(context,
Constants.BULK_ARBITRATOR_TYPE, 9L);
// simulate a bulk import
- var stf5 =
- new
StoredTabletFile("hdfs://localhost:8020/accumulo/tables/2a/b-0000009/I0000074.rf");
+ var stf5 = StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/2a/b-0000009/I0000074.rf"));
ctmi = new ConditionalTabletsMutatorImpl(context);
var tm6 = TabletMetadata.builder(e1).build(LOADED);
ctmi.mutateTablet(e1).requireAbsentOperation().requireSame(tm6, LOADED)
@@ -292,8 +294,8 @@ public class AmpleConditionalWriterIT extends
AccumuloClusterHarness {
assertEquals(Set.of(stf4, stf5),
context.getAmple().readTablet(e1).getFiles());
// simulate a compaction
- var stf6 = new StoredTabletFile(
-
"hdfs://localhost:8020/accumulo/tables/2a/default_tablet/A0000075.rf");
+ var stf6 = StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/A0000075.rf"));
ctmi = new ConditionalTabletsMutatorImpl(context);
var tm7 = TabletMetadata.builder(e1).putFile(stf4, dfv).putFile(stf5,
dfv).build();
ctmi.mutateTablet(e1).requireAbsentOperation().requireSame(tm7, FILES)
@@ -320,14 +322,14 @@ public class AmpleConditionalWriterIT extends
AccumuloClusterHarness {
public void testSelectedFiles() throws Exception {
var context = cluster.getServerContext();
- var stf1 =
- new
StoredTabletFile("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000070.rf");
- var stf2 =
- new
StoredTabletFile("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000071.rf");
- var stf3 =
- new
StoredTabletFile("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000072.rf");
- var stf4 =
- new
StoredTabletFile("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/C0000073.rf");
+ var stf1 = StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000070.rf"));
+ var stf2 = StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000071.rf"));
+ var stf3 = StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000072.rf"));
+ var stf4 = StoredTabletFile
+ .of(new
Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/C0000073.rf"));
var dfv = new DataFileValue(100, 100);
System.out.println(context.getAmple().readTablet(e1).getLocation());
@@ -410,9 +412,9 @@ public class AmpleConditionalWriterIT extends
AccumuloClusterHarness {
var context = cluster.getServerContext();
String pathPrefix =
"hdfs://localhost:8020/accumulo/tables/2a/default_tablet/";
- StoredTabletFile stf1 = new StoredTabletFile(pathPrefix + "F0000070.rf");
- StoredTabletFile stf2 = new StoredTabletFile(pathPrefix + "F0000071.rf");
- StoredTabletFile stf3 = new StoredTabletFile(pathPrefix + "F0000072.rf");
+ StoredTabletFile stf1 = StoredTabletFile.of(new Path(pathPrefix +
"F0000070.rf"));
+ StoredTabletFile stf2 = StoredTabletFile.of(new Path(pathPrefix +
"F0000071.rf"));
+ StoredTabletFile stf3 = StoredTabletFile.of(new Path(pathPrefix +
"F0000072.rf"));
final Set<StoredTabletFile> storedTabletFiles = Set.of(stf1, stf2, stf3);
final boolean initiallySelectedAll = true;
@@ -484,7 +486,7 @@ public class AmpleConditionalWriterIT extends
AccumuloClusterHarness {
TabletMetadata tm1 =
TabletMetadata.builder(e1).putSelectedFiles(selectedFiles).build(SELECTED);
ctmi = new ConditionalTabletsMutatorImpl(context);
- StoredTabletFile stf4 = new StoredTabletFile(pathPrefix + "F0000073.rf");
+ StoredTabletFile stf4 = StoredTabletFile.of(new Path(pathPrefix +
"F0000073.rf"));
// submit a mutation with the condition that the selected files match
what was originally
// written
DataFileValue dfv = new DataFileValue(100, 100);
@@ -516,11 +518,10 @@ public class AmpleConditionalWriterIT extends
AccumuloClusterHarness {
*/
public static String createSelectedFilesJson(Long txid, boolean selAll,
Collection<String> paths) {
- String filesJsonArray =
- paths.stream().map(path -> "'" + path +
"'").collect(Collectors.joining(","));
+ String filesJsonArray = GSON.get().toJson(paths);
String formattedTxid =
FateTxId.formatTid(Long.parseLong(Long.toString(txid), 16));
- return ("{'txid':'" + formattedTxid + "','selAll':" + selAll +
",'files':[" + filesJsonArray
- + "]}").replace('\'', '\"');
+ return ("{'txid':'" + formattedTxid + "','selAll':" + selAll + ",'files':"
+ filesJsonArray
+ + "}").replace('\'', '\"');
}
@Test
