This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 3.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 5e218f1411a8e65f55b22359eb314efd4d9a5724
Merge: 8bac964fba 0f53ae04db
Author: Dave Marion <dlmar...@apache.org>
AuthorDate: Thu Feb 13 19:53:16 2025 +0000
Merge branch '2.1' into 3.1
.../org/apache/accumulo/test/ImportExportIT.java | 243 +++++++++++----------
1 file changed, 130 insertions(+), 113 deletions(-)
diff --cc test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
index efb3bbbf31,995474dcc1..ac1e532d49
--- a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
@@@ -94,145 -82,147 +94,162 @@@ public class ImportExportIT extends Acc
return Duration.ofMinutes(1);
}
- @ParameterizedTest
- @ValueSource(booleans = {true, false})
- public void testExportImportThenScan(boolean fenced) throws Exception {
- @Test
- public void testExportImportThenScan() throws Exception {
- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- String[] tableNames = getUniqueNames(2);
- doExportImportThenScan(client, tableNames[0], tableNames[1]);
- }
- }
-
- @Test
- public void testExportImportSameTableNameThenScan() throws Exception {
-- try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- String ns1 = "namespace1";
- client.namespaceOperations().create(ns1);
- String ns2 = "namespace2";
- client.namespaceOperations().create(ns2);
- String tableName = getUniqueNames(1)[0];
- doExportImportThenScan(client, ns1 + "." + tableName, ns2 + "." +
tableName);
- }
- }
-
- public void doExportImportThenScan(AccumuloClient client, String srcTable,
String destTable)
- throws Exception {
++ private void doExportImportThenScan(boolean fenced, AccumuloClient client,
String srcTable,
++ String destTable) throws Exception {
- String[] tableNames = getUniqueNames(2);
- String srcTable = tableNames[0], destTable = tableNames[1];
- client.tableOperations().create(srcTable);
+ client.tableOperations().create(srcTable);
- try (BatchWriter bw = client.createBatchWriter(srcTable)) {
- for (int row = 0; row < 1000; row++) {
- Mutation m = new Mutation("row_" + String.format("%010d", row));
- for (int col = 0; col < 100; col++) {
- m.put(Integer.toString(col), "", Integer.toString(col * 2));
- }
- bw.addMutation(m);
+ try (BatchWriter bw = client.createBatchWriter(srcTable)) {
+ for (int row = 0; row < 1000; row++) {
- Mutation m = new Mutation(Integer.toString(row));
++ Mutation m = new Mutation("row_" + String.format("%010d", row));
+ for (int col = 0; col < 100; col++) {
+ m.put(Integer.toString(col), "", Integer.toString(col * 2));
}
+ bw.addMutation(m);
}
+ }
- client.tableOperations().compact(srcTable, null, null, true, true);
+ client.tableOperations().compact(srcTable, null, null, true, true);
- int expected = 100000;
- // Test that files with ranges and are fenced work with export/import
- if (fenced) {
- // Split file into 3 ranges of 10000, 20000, and 5000 for a total of
35000
- FileMetadataUtil.splitFilesIntoRanges(getServerContext(), srcTable,
createRanges());
- expected = 35000;
- }
++ int expected = 100000;
++ // Test that files with ranges and are fenced work with export/import
++ if (fenced) {
++ // Split file into 3 ranges of 10000, 20000, and 5000 for a total of
35000
++ FileMetadataUtil.splitFilesIntoRanges(getServerContext(), srcTable,
createRanges());
++ expected = 35000;
++ }
+
- // Make a directory we can use to throw the export and import
directories
- // Must exist on the filesystem the cluster is running.
- FileSystem fs = cluster.getFileSystem();
- log.info("Using FileSystem: " + fs);
- Path baseDir = new Path(cluster.getTemporaryPath(),
getClass().getName());
- fs.deleteOnExit(baseDir);
- if (fs.exists(baseDir)) {
- log.info("{} exists on filesystem, deleting", baseDir);
- assertTrue(fs.delete(baseDir, true), "Failed to deleted " + baseDir);
- }
- log.info("Creating {}", baseDir);
- assertTrue(fs.mkdirs(baseDir), "Failed to create " + baseDir);
- Path exportDir = new Path(baseDir, "export");
- fs.deleteOnExit(exportDir);
- Path importDirA = new Path(baseDir, "import-a");
- Path importDirB = new Path(baseDir, "import-b");
- fs.deleteOnExit(importDirA);
- fs.deleteOnExit(importDirB);
- for (Path p : new Path[] {exportDir, importDirA, importDirB}) {
- assertTrue(fs.mkdirs(p), "Failed to create " + p);
- }
+ // Make a directory we can use to throw the export and import directories
+ // Must exist on the filesystem the cluster is running.
+ FileSystem fs = cluster.getFileSystem();
+ log.info("Using FileSystem: " + fs);
+ Path baseDir = new Path(cluster.getTemporaryPath(), getClass().getName());
+ fs.deleteOnExit(baseDir);
+ if (fs.exists(baseDir)) {
+ log.info("{} exists on filesystem, deleting", baseDir);
+ assertTrue(fs.delete(baseDir, true), "Failed to deleted " + baseDir);
+ }
+ log.info("Creating {}", baseDir);
+ assertTrue(fs.mkdirs(baseDir), "Failed to create " + baseDir);
+ Path exportDir = new Path(baseDir, "export");
+ fs.deleteOnExit(exportDir);
+ Path importDirA = new Path(baseDir, "import-a");
+ Path importDirB = new Path(baseDir, "import-b");
+ fs.deleteOnExit(importDirA);
+ fs.deleteOnExit(importDirB);
+ for (Path p : new Path[] {exportDir, importDirA, importDirB}) {
+ assertTrue(fs.mkdirs(p), "Failed to create " + p);
+ }
- Set<String> importDirs = Set.of(importDirA.toString(),
importDirB.toString());
+ Set<String> importDirs = Set.of(importDirA.toString(),
importDirB.toString());
- Path[] importDirAry = new Path[] {importDirA, importDirB};
+ Path[] importDirAry = new Path[] {importDirA, importDirB};
- log.info("Exporting table to {}", exportDir);
- log.info("Importing table from {}", importDirs);
+ log.info("Exporting table to {}", exportDir);
+ log.info("Importing table from {}", importDirs);
- // test fast fail offline check
- assertThrows(IllegalStateException.class,
- () -> client.tableOperations().exportTable(srcTable,
exportDir.toString()));
+ // test fast fail offline check
+ assertThrows(IllegalStateException.class,
+ () -> client.tableOperations().exportTable(srcTable,
exportDir.toString()));
- // Offline the table
- client.tableOperations().offline(srcTable, true);
- // Then export it
- client.tableOperations().exportTable(srcTable, exportDir.toString());
+ // Offline the table
+ client.tableOperations().offline(srcTable, true);
+ // Then export it
+ client.tableOperations().exportTable(srcTable, exportDir.toString());
- // Make sure the distcp.txt file that exporttable creates is available
- Path distcp = new Path(exportDir, "distcp.txt");
- fs.deleteOnExit(distcp);
- assertTrue(fs.exists(distcp), "Distcp file doesn't exist");
- FSDataInputStream is = fs.open(distcp);
- BufferedReader reader = new BufferedReader(new InputStreamReader(is));
+ // Make sure the distcp.txt file that exporttable creates is available
+ Path distcp = new Path(exportDir, "distcp.txt");
+ fs.deleteOnExit(distcp);
+ assertTrue(fs.exists(distcp), "Distcp file doesn't exist");
+ FSDataInputStream is = fs.open(distcp);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(is));
- // Copy each file that was exported to one of the imports directory
- String line;
+ // Copy each file that was exported to one of the imports directory
+ String line;
- while ((line = reader.readLine()) != null) {
- Path p = new Path(line.substring(5));
- assertTrue(fs.exists(p), "File doesn't exist: " + p);
- Path importDir =
importDirAry[RANDOM.get().nextInt(importDirAry.length)];
- Path dest = new Path(importDir, p.getName());
- assertFalse(fs.exists(dest), "Did not expect " + dest + " to exist");
- FileUtil.copy(fs, p, fs, dest, false, fs.getConf());
- }
+ while ((line = reader.readLine()) != null) {
+ Path p = new Path(line.substring(5));
+ assertTrue(fs.exists(p), "File doesn't exist: " + p);
- Path importDir = importDirAry[random.nextInt(importDirAry.length)];
++ Path importDir =
importDirAry[RANDOM.get().nextInt(importDirAry.length)];
+ Path dest = new Path(importDir, p.getName());
+ assertFalse(fs.exists(dest), "Did not expect " + dest + " to exist");
+ FileUtil.copy(fs, p, fs, dest, false, fs.getConf());
+ }
- reader.close();
-
- log.info("Import dir A: {}",
Arrays.toString(fs.listStatus(importDirA)));
- log.info("Import dir B: {}",
Arrays.toString(fs.listStatus(importDirB)));
-
- // Import the exported data into a new table
- client.tableOperations().importTable(destTable, importDirs,
ImportConfiguration.empty());
-
- // Get the table ID for the table that the importtable command created
- final String tableId =
client.tableOperations().tableIdMap().get(destTable);
- assertNotNull(tableId);
-
- // Get all `file` colfams from the metadata table for the new table
- log.info("Imported into table with ID: {}", tableId);
-
- try (Scanner s =
- client.createScanner(AccumuloTable.METADATA.tableName(),
Authorizations.EMPTY)) {
- s.setRange(TabletsSection.getRange(TableId.of(tableId)));
- s.fetchColumnFamily(DataFileColumnFamily.NAME);
- ServerColumnFamily.DIRECTORY_COLUMN.fetch(s);
-
- // Should find a single entry
- for (Entry<Key,Value> fileEntry : s) {
- Key k = fileEntry.getKey();
- String value = fileEntry.getValue().toString();
- if (k.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
- // The file should be an absolute URI (file:///...), not a
relative path
- // (/b-000.../I000001.rf)
- var tabFile = StoredTabletFile.of(k.getColumnQualifier());
- // Verify that the range is set correctly on the StoredTabletFile
- assertEquals(fenced, !tabFile.getRange().isInfiniteStartKey()
- || !tabFile.getRange().isInfiniteStopKey());
- assertFalse(looksLikeRelativePath(tabFile.getMetadataPath()),
- "Imported files should have absolute URIs, not relative: " +
tabFile);
- } else if (k.getColumnFamily().equals(ServerColumnFamily.NAME)) {
- assertFalse(looksLikeRelativePath(value),
- "Server directory should have absolute URI, not relative: " +
value);
- } else {
- fail("Got expected pair: " + k + "=" + fileEntry.getValue());
- }
+ reader.close();
+
+ log.info("Import dir A: {}", Arrays.toString(fs.listStatus(importDirA)));
+ log.info("Import dir B: {}", Arrays.toString(fs.listStatus(importDirB)));
+
+ // Import the exported data into a new table
+ client.tableOperations().importTable(destTable, importDirs,
ImportConfiguration.empty());
+
+ // Get the table ID for the table that the importtable command created
+ final String tableId =
client.tableOperations().tableIdMap().get(destTable);
+ assertNotNull(tableId);
+
+ // Get all `file` colfams from the metadata table for the new table
+ log.info("Imported into table with ID: {}", tableId);
+
- try (Scanner s = client.createScanner(MetadataTable.NAME,
Authorizations.EMPTY)) {
++ try (Scanner s =
++ client.createScanner(AccumuloTable.METADATA.tableName(),
Authorizations.EMPTY)) {
+ s.setRange(TabletsSection.getRange(TableId.of(tableId)));
+ s.fetchColumnFamily(DataFileColumnFamily.NAME);
+ ServerColumnFamily.DIRECTORY_COLUMN.fetch(s);
+
+ // Should find a single entry
+ for (Entry<Key,Value> fileEntry : s) {
+ Key k = fileEntry.getKey();
+ String value = fileEntry.getValue().toString();
+ if (k.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ // The file should be an absolute URI (file:///...), not a relative
path
+ // (/b-000.../I000001.rf)
- String fileUri = k.getColumnQualifier().toString();
- assertFalse(looksLikeRelativePath(fileUri),
- "Imported files should have absolute URIs, not relative: " +
fileUri);
++ var tabFile = StoredTabletFile.of(k.getColumnQualifier());
++ // Verify that the range is set correctly on the StoredTabletFile
++ assertEquals(fenced,
++ !tabFile.getRange().isInfiniteStartKey() ||
!tabFile.getRange().isInfiniteStopKey());
++ assertFalse(looksLikeRelativePath(tabFile.getMetadataPath()),
++ "Imported files should have absolute URIs, not relative: " +
tabFile);
+ } else if (k.getColumnFamily().equals(ServerColumnFamily.NAME)) {
+ assertFalse(looksLikeRelativePath(value),
+ "Server directory should have absolute URI, not relative: " +
value);
+ } else {
+ fail("Got expected pair: " + k + "=" + fileEntry.getValue());
}
-
}
- // Online the original table before we verify equivalence
- client.tableOperations().online(srcTable, true);
- verifyTableEquality(client, srcTable, destTable, expected);
+ }
+ // Online the original table before we verify equivalence
+ client.tableOperations().online(srcTable, true);
+
- verifyTableEquality(client, srcTable, destTable);
++ verifyTableEquality(client, srcTable, destTable, expected);
+ }
+
- @Test
- public void testExportImportOffline() throws Exception {
++ @ParameterizedTest
++ @ValueSource(booleans = {true, false})
++ public void testExportImportThenScan(boolean fenced) throws Exception {
++ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
++ String[] tableNames = getUniqueNames(2);
++ doExportImportThenScan(fenced, client, tableNames[0], tableNames[1]);
++ }
++ }
++
++ @ParameterizedTest
++ @ValueSource(booleans = {true, false})
++ public void testExportImportSameTableNameThenScan(boolean fenced) throws
Exception {
++ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
++ String ns1 = "namespace1";
++ client.namespaceOperations().create(ns1);
++ String ns2 = "namespace2";
++ client.namespaceOperations().create(ns2);
++ String tableName = getUniqueNames(1)[0];
++ doExportImportThenScan(fenced, client, ns1 + "." + tableName, ns2 + "."
+ tableName);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testExportImportOffline(boolean fenced) throws Exception {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
String[] tableNames = getUniqueNames(2);
