This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1db892b00d Update export version to handle imports of files without
fenced ranges (#4122)
1db892b00d is described below
commit 1db892b00dfca083882ef8f90d3baa43dbb59181
Author: EdColeman <d...@etcoleman.com>
AuthorDate: Wed Jan 31 17:58:54 2024 -0500
Update export version to handle imports of files without fenced ranges
(#4122)
Update export version to handle imports of files without fenced ranges
- adds specific export version
- adds import test with 2.1.3 create files (no fenced ranges)
Co-authored-by: Christopher Tubbs <ctubb...@apache.org>
---
core/pom.xml | 2 -
.../accumulo/core/metadata/StoredTabletFile.java | 15 ++++
.../core/metadata/StoredTabletFileTest.java | 40 +++++++++++
pom.xml | 2 +
.../manager/tableOps/tableExport/ExportTable.java | 10 ++-
.../tableOps/tableExport/WriteExportFiles.java | 2 +-
.../manager/tableOps/tableImport/ImportTable.java | 8 +--
.../tableOps/tableImport/ImportedTableInfo.java | 1 +
.../tableImport/PopulateMetadataTable.java | 16 ++++-
.../accumulo/manager/upgrade/Upgrader11to12.java | 20 ++----
.../manager/upgrade/Upgrader11to12Test.java | 25 ++-----
test/pom.xml | 17 +++++
.../org/apache/accumulo/test/ImportExportIT.java | 53 ++++++++++++++
test/src/main/resources/v2_import_test/README.md | 78 +++++++++++++++++++++
.../main/resources/v2_import_test/data/A0000008.rf | Bin 0 -> 229 bytes
.../main/resources/v2_import_test/data/A0000009.rf | Bin 0 -> 229 bytes
.../main/resources/v2_import_test/data/A000000a.rf | Bin 0 -> 228 bytes
.../main/resources/v2_import_test/data/A000000b.rf | Bin 0 -> 212 bytes
.../main/resources/v2_import_test/data/distcp.txt | 5 ++
.../v2_import_test/data/exportMetadata.zip | Bin 0 -> 934 bytes
20 files changed, 246 insertions(+), 48 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 3b1ec42ea6..05d9a02c69 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -193,7 +193,6 @@
<exclude>src/main/java/org/apache/accumulo/core/bloomfilter/*.java</exclude>
<exclude>src/main/java/org/apache/accumulo/core/util/HostAndPort.java</exclude>
<exclude>src/test/resources/*.jceks</exclude>
-
<exclude>src/test/resources/org/apache/accumulo/core/file/rfile/*.rf</exclude>
</excludes>
</licenseSet>
</licenseSets>
@@ -206,7 +205,6 @@
<excludes>
<exclude>src/main/java/org/apache/accumulo/core/bloomfilter/*.java</exclude>
<exclude>src/test/resources/*.jceks</exclude>
-
<exclude>src/test/resources/org/apache/accumulo/core/file/rfile/*.rf</exclude>
</excludes>
</configuration>
</plugin>
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
index daae673e75..16bb0458a9 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
@@ -33,7 +33,9 @@ import
org.apache.accumulo.core.util.json.ByteArrayToBase64TypeAdapter;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.Text;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
/**
@@ -263,6 +265,19 @@ public class StoredTabletFile extends
AbstractTabletFile<StoredTabletFile> {
}
}
+ /**
+ * Quick validation to see if value has been converted by checking if the
candidate looks like
+ * json by checking the candidate starts with "{" and ends with "}".
+ *
+ * @param candidate a possible file: reference.
+ * @return false if a likely a json object, true if not a likely json object
+ */
+ @VisibleForTesting
+ public static boolean fileNeedsConversion(@NonNull final String candidate) {
+ String trimmed = candidate.trim();
+ return !trimmed.startsWith("{") || !trimmed.endsWith("}");
+ }
+
private static class TabletFileCq {
public final Path path;
public final Range range;
diff --git
a/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java
b/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java
new file mode 100644
index 0000000000..998c9eab11
--- /dev/null
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+public class StoredTabletFileTest {
+
+ @Test
+ public void fileConversionTest() {
+ String s21 =
"hdfs://localhost:8020/accumulo/tables/1/t-0000000/A000003v.rf";
+ String s31 =
+
"{\"path\":\"hdfs://localhost:8020/accumulo/tables/1/t-0000000/A000003v.rf\",\"startRow\":\"\",\"endRow\":\"\"}";
+ String s31_untrimmed =
+ " {
\"path\":\"hdfs://localhost:8020/accumulo/tables/1/t-0000000/A000003v.rf\",\"startRow\":\"\",\"endRow\":\"\"
} ";
+
+ assertTrue(StoredTabletFile.fileNeedsConversion(s21));
+ assertFalse(StoredTabletFile.fileNeedsConversion(s31));
+ assertFalse(StoredTabletFile.fileNeedsConversion(s31_untrimmed));
+ }
+}
diff --git a/pom.xml b/pom.xml
index 6883e839a0..aa11e333aa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -661,6 +661,7 @@
<exclude>**/LICENSE</exclude>
<exclude>**/NOTICE</exclude>
<exclude>**/target/**</exclude>
+ <exclude>**/*.rf</exclude>
</excludes>
</licenseSet>
</licenseSets>
@@ -866,6 +867,7 @@
<exclude>.vscode/**</exclude>
<exclude>.factorypath</exclude>
<exclude>.github/**</exclude>
+ <exclude>**/*.rf</exclude>
</excludes>
</configuration>
</plugin>
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableExport/ExportTable.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableExport/ExportTable.java
index 7e3a2a35e0..8662607c18 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableExport/ExportTable.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableExport/ExportTable.java
@@ -54,7 +54,15 @@ public class ExportTable extends ManagerRepo {
Utils.unreserveHdfsDirectory(env, new
Path(tableInfo.exportDir).toString(), tid);
}
- public static final int VERSION = 1;
+ /**
+ * Defines export / version.
+ * <ul>
+ * <li>version 1 exported by Accumulo < 3.1</li>
+ * <li>version 2 exported by Accumulo => 3.1 - uses file references with
ranges.</li>
+ * </ul>
+ */
+ public static final int VERSION_2 = 2;
+ public static final int CURR_VERSION = VERSION_2;
public static final String DATA_VERSION_PROP = "srcDataVersion";
public static final String EXPORT_VERSION_PROP = "exportVersion";
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableExport/WriteExportFiles.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableExport/WriteExportFiles.java
index 5029ea7363..034f4d61a5 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableExport/WriteExportFiles.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableExport/WriteExportFiles.java
@@ -168,7 +168,7 @@ class WriteExportFiles extends ManagerRepo {
try (OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8)) {
zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_INFO_FILE));
- osw.append(ExportTable.EXPORT_VERSION_PROP + ":" + ExportTable.VERSION +
"\n");
+ osw.append(ExportTable.EXPORT_VERSION_PROP + ":" +
ExportTable.CURR_VERSION + "\n");
osw.append("srcInstanceName:" + context.getInstanceName() + "\n");
osw.append("srcInstanceID:" + context.getInstanceID() + "\n");
osw.append("srcZookeepers:" + context.getZooKeepers() + "\n");
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportTable.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportTable.java
index 484db2b963..58993783b7 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportTable.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportTable.java
@@ -110,7 +110,7 @@ public class ImportTable extends ManagerRepo {
log.debug("Searching for export file in {}", exportDirs);
- Integer exportVersion = null;
+ tableInfo.exportedVersion = null;
Integer dataVersion = null;
try {
@@ -127,7 +127,7 @@ public class ImportTable extends ManagerRepo {
while ((line = in.readLine()) != null) {
String[] sa = line.split(":", 2);
if (sa[0].equals(ExportTable.EXPORT_VERSION_PROP)) {
- exportVersion = Integer.parseInt(sa[1]);
+ tableInfo.exportedVersion = Integer.parseInt(sa[1]);
} else if (sa[0].equals(ExportTable.DATA_VERSION_PROP)) {
dataVersion = Integer.parseInt(sa[1]);
}
@@ -142,10 +142,10 @@ public class ImportTable extends ManagerRepo {
"Failed to read export metadata " + e.getMessage());
}
- if (exportVersion == null || exportVersion > ExportTable.VERSION) {
+ if (tableInfo.exportedVersion == null || tableInfo.exportedVersion >
ExportTable.CURR_VERSION) {
throw new AcceptableThriftTableOperationException(null,
tableInfo.tableName,
TableOperation.IMPORT, TableOperationExceptionType.OTHER,
- "Incompatible export version " + exportVersion);
+ "Incompatible export version " + tableInfo.exportedVersion);
}
if (dataVersion == null || dataVersion > AccumuloDataVersion.get()) {
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportedTableInfo.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportedTableInfo.java
index 306c7a38e5..824786d329 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportedTableInfo.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportedTableInfo.java
@@ -36,6 +36,7 @@ class ImportedTableInfo implements Serializable {
public String exportFile;
public boolean keepMappings;
public boolean keepOffline;
+ public Integer exportedVersion = null;
static class DirectoryMapping implements Serializable {
private static final long serialVersionUID = 1L;
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/PopulateMetadataTable.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/PopulateMetadataTable.java
index 4a0e0f245a..2cdaa64138 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/PopulateMetadataTable.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/PopulateMetadataTable.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.manager.tableOps.tableImport;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.core.Constants.IMPORT_MAPPINGS_FILE;
+import static
org.apache.accumulo.manager.tableOps.tableExport.ExportTable.VERSION_2;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
@@ -131,8 +132,17 @@ class PopulateMetadataTable extends ManagerRepo {
Text cq;
if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
- final StoredTabletFile oldTabletFile =
StoredTabletFile.of(key.getColumnQualifier());
- String oldName = oldTabletFile.getFileName();
+ StoredTabletFile exportedRef;
+ var dataFileCQ = key.getColumnQualifier().toString();
+ if (tableInfo.exportedVersion == null ||
tableInfo.exportedVersion < VERSION_2) {
+ // written without fenced range information (accumulo < 3.1),
use default
+ // (null,null)
+ exportedRef = StoredTabletFile.of(new Path(dataFileCQ));
+ } else {
+ exportedRef = StoredTabletFile.of(key.getColumnQualifier());
+ }
+
+ String oldName = exportedRef.getFileName();
String newName = fileNameMappings.get(oldName);
if (newName == null) {
@@ -142,7 +152,7 @@ class PopulateMetadataTable extends ManagerRepo {
}
// Copy over the range for the new file
- cq = StoredTabletFile.of(URI.create(newName),
oldTabletFile.getRange())
+ cq = StoredTabletFile.of(URI.create(newName),
exportedRef.getRange())
.getMetadataText();
} else {
cq = key.getColumnQualifier();
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java
index c638d45315..0c5e086549 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java
@@ -197,28 +197,16 @@ public class Upgrader11to12 implements Upgrader {
}
@VisibleForTesting
- void upgradeDataFileCF(final Key key, final Value value, final Mutation m) {
+ static void upgradeDataFileCF(final Key key, final Value value, final
Mutation m) {
String file = key.getColumnQualifier().toString();
// filter out references if they are in the correct format already.
- if (fileNeedsConversion(file)) {
+ boolean needsConversion = StoredTabletFile.fileNeedsConversion(file);
+ log.trace("file: {} needs conversion: {}", file, needsConversion);
+ if (needsConversion) {
var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
m.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
m.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
}
}
- /**
- * Quick validation to see if value has been converted by checking if the
candidate looks like
- * json by checking the candidate starts with "{" and ends with "}".
- *
- * @param candidate a possible file: reference.
- * @return false if a likely a json object, true if not a likely json object
- */
- @VisibleForTesting
- boolean fileNeedsConversion(@NonNull final String candidate) {
- String trimmed = candidate.trim();
- boolean needsConversion = !trimmed.startsWith("{") ||
!trimmed.endsWith("}");
- log.trace("file: {} needs conversion: {}", candidate, needsConversion);
- return needsConversion;
- }
}
diff --git
a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java
b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java
index 7818164e75..14d19de465 100644
---
a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java
+++
b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java
@@ -71,15 +71,13 @@ public class Upgrader11to12Test {
@Test
void upgradeDataFileCF2Test() {
- Upgrader11to12 upgrader = new Upgrader11to12();
-
String fileName =
"hdfs://localhost:8020/accumulo/tables/12/default_tablet/A000000v.rf";
Key k = Key.builder().row(new
Text("12;")).family(DataFileColumnFamily.NAME)
.qualifier(new Text(fileName)).build();
Value v = new Value("1234,5678");
Mutation upgrade = new Mutation(k.getRow());
- upgrader.upgradeDataFileCF(k, v, upgrade);
+ Upgrader11to12.upgradeDataFileCF(k, v, upgrade);
var pending = upgrade.getUpdates();
assertEquals(2, pending.size());
@@ -105,7 +103,7 @@ public class Upgrader11to12Test {
}
@Test
- public void processReferencesTest() throws Exception {
+ public void processReferencesTest() {
// create sample data "served" by the mocked scanner
TreeMap<Key,Value> scanData = new TreeMap<>();
@@ -161,7 +159,7 @@ public class Upgrader11to12Test {
}
@Test
- public void skipConvertedFileTest() throws Exception {
+ public void skipConvertedFileTest() {
// create sample data "served" by the mocked scanner
TreeMap<Key,Value> scanData = new TreeMap<>();
Text row1 = new Text("123");
@@ -288,7 +286,7 @@ public class Upgrader11to12Test {
* called for those rows
*/
@Test
- public void verifyEmptyMutation() throws Exception {
+ public void verifyEmptyMutation() {
// create sample data "served" by the mocked scanner
TreeMap<Key,Value> scanData = new TreeMap<>();
@@ -377,21 +375,6 @@ public class Upgrader11to12Test {
verify(context, zrw);
}
- @Test
- public void fileConversionTest() {
- String s21 =
"hdfs://localhost:8020/accumulo/tables/1/t-0000000/A000003v.rf";
- String s31 =
-
"{\"path\":\"hdfs://localhost:8020/accumulo/tables/1/t-0000000/A000003v.rf\",\"startRow\":\"\",\"endRow\":\"\"}";
- String s31_untrimmed =
- " {
\"path\":\"hdfs://localhost:8020/accumulo/tables/1/t-0000000/A000003v.rf\",\"startRow\":\"\",\"endRow\":\"\"
} ";
-
- Upgrader11to12 upgrader = new Upgrader11to12();
-
- assertTrue(upgrader.fileNeedsConversion(s21));
- assertFalse(upgrader.fileNeedsConversion(s31));
- assertFalse(upgrader.fileNeedsConversion(s31_untrimmed));
- }
-
@Test
public void convertRoot1File() {
String root21ZkData =
diff --git a/test/pom.xml b/test/pom.xml
index 7fbc514f9b..7db2364742 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -236,6 +236,23 @@
<mapping>
<shellit>SCRIPT_STYLE</shellit>
</mapping>
+ <licenseSets>
+ <licenseSet>
+ <header>${rootlocation}/src/build/license-header.txt</header>
+ <excludes>
+
<exclude>src/main/resources/v2_import_test/data/distcp.txt</exclude>
+ </excludes>
+ </licenseSet>
+ </licenseSets>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+
<exclude>src/main/resources/v2_import_test/data/distcp.txt</exclude>
+ </excludes>
</configuration>
</plugin>
<plugin>
diff --git a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
index e5d20997f4..4fc909c4d5 100644
--- a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
@@ -28,11 +28,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import java.io.BufferedReader;
+import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.file.Paths;
import java.time.Duration;
import java.util.Arrays;
import java.util.Iterator;
+import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
@@ -59,11 +62,14 @@ import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl;
import org.apache.accumulo.test.util.FileMetadataUtil;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.slf4j.Logger;
@@ -380,6 +386,53 @@ public class ImportExportIT extends AccumuloClusterHarness
{
return false;
}
+ /**
+ * Validate that files exported with Accumulo 2.x without fence ranges can
be imported into
+ * version that require the fenced ranges (3.1 and later)
+ */
+ @Test
+ public void importV2data() throws Exception {
+ final String dataRoot = "./target/classes/v2_import_test";
+ final String dataSrc = dataRoot + "/data";
+ final String importDir = dataRoot + "/import";
+
+ // copy files each run will "move the files" on import, allows multiple
runs in IDE without
+ // rebuild
+ java.nio.file.Path importDirPath = Paths.get(importDir);
+ java.nio.file.Files.createDirectories(importDirPath);
+ FileUtils.copyDirectory(new File(dataSrc), new File(importDir));
+
+ String table = getUniqueNames(1)[0];
+
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ log.debug("importing from: {} into table: {}", importDir, table);
+ client.tableOperations().importTable(table, importDir);
+
+ int rowCount = 0;
+ try (Scanner s = client.createScanner(table, Authorizations.EMPTY)) {
+ for (Entry<Key,Value> entry : s) {
+ log.trace("data:{}", entry);
+ rowCount++;
+ }
+ }
+ assertEquals(7, rowCount);
+ int metaFileCount = 0;
+ try (Scanner s =
+ client.createScanner(AccumuloTable.METADATA.tableName(),
Authorizations.EMPTY)) {
+ TableId tid =
TableId.of(client.tableOperations().tableIdMap().get(table));
+ s.setRange(TabletsSection.getRange(tid));
+ s.fetchColumnFamily(DataFileColumnFamily.NAME);
+ for (Entry<Key,Value> entry : s) {
+ log.trace("metadata file:{}", entry);
+ metaFileCount++;
+ }
+ }
+ final List<Text> expectedSplits = List.of(new Text("2"), new Text("4"),
new Text("6"));
+ assertEquals(expectedSplits, client.tableOperations().listSplits(table));
+ assertEquals(4, metaFileCount);
+ }
+ }
+
private void verifyTableEquality(AccumuloClient client, String srcTable,
String destTable,
int expected) throws Exception {
Iterator<Entry<Key,Value>> src =
diff --git a/test/src/main/resources/v2_import_test/README.md
b/test/src/main/resources/v2_import_test/README.md
new file mode 100644
index 0000000000..7568d434a3
--- /dev/null
+++ b/test/src/main/resources/v2_import_test/README.md
@@ -0,0 +1,78 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+# Import test data created with Accumulo version 2.1.3
+
+This data was created using the Accumulo shell to
+ - create a table
+ - add some splits so there is more than one file (optional)
+ - insert some data
+ - compact the table so the data is written to the tablet's files
+ - offline and export the table to an hdfs directory in the /accumulo namespace
+
+The data can be recreated using the following commands:
+
+Using hdfs create a directory in the accumulo namespace
+`hadoop fs -mkdir /accumulo/export_test`
+
+Using the Accumulo shell with any version prior to 3.1
+
+```
+> createtable tableA
+> addsplits -t tableA 2 4 6
+> insert -t tableA 1 1
+> insert -t tableA 1 cf cq 1
+> insert -t tableA 2 cf cq 2
+> insert -t tableA 3 cf cq 3
+> insert -t tableA 4 cf cq 4
+> insert -t tableA 5 cf cq 5
+> insert -t tableA 6 cf cq 6
+> insert -t tableA 7 cf cq 7
+
+> compact -w -t tableA
+
+to see the current tablet files:
+
+> scan -t accumulo.metadata -c file -np
+> offline -t tableA
+
+> exporttable -t tableA /accumulo/export_test
+
+```
+
+The export command will create two files:
+ - /accumulo/export_test/distcp.txt
+ - /accumulo/export_test/exportMetadata.zip
+
+The distcp files lists the tablet files:
+
+```
+> hadoop fs -cat /accumulo/export_test/distcp.txt
+
+hdfs://localhost:8020/accumulo/tables/1/default_tablet/A000000b.rf
+hdfs://localhost:8020/accumulo/tables/1/t-0000002/A000000a.rf
+hdfs://localhost:8020/accumulo/tables/1/t-0000001/A0000009.rf
+hdfs://localhost:8020/accumulo/tables/1/t-0000003/A0000008.rf
+hdfs://localhost:8020/accumulo/export_test/exportMetadata.zip
+```
+
+The files distcp.txt, exportMetadata.zip and the files listed in distcp where
copied from
+hdfs to make this data set.
\ No newline at end of file
diff --git a/test/src/main/resources/v2_import_test/data/A0000008.rf
b/test/src/main/resources/v2_import_test/data/A0000008.rf
new file mode 100644
index 0000000000..4c333f644a
Binary files /dev/null and
b/test/src/main/resources/v2_import_test/data/A0000008.rf differ
diff --git a/test/src/main/resources/v2_import_test/data/A0000009.rf
b/test/src/main/resources/v2_import_test/data/A0000009.rf
new file mode 100644
index 0000000000..fa4df66f73
Binary files /dev/null and
b/test/src/main/resources/v2_import_test/data/A0000009.rf differ
diff --git a/test/src/main/resources/v2_import_test/data/A000000a.rf
b/test/src/main/resources/v2_import_test/data/A000000a.rf
new file mode 100644
index 0000000000..8a7642c15f
Binary files /dev/null and
b/test/src/main/resources/v2_import_test/data/A000000a.rf differ
diff --git a/test/src/main/resources/v2_import_test/data/A000000b.rf
b/test/src/main/resources/v2_import_test/data/A000000b.rf
new file mode 100644
index 0000000000..c0503dca33
Binary files /dev/null and
b/test/src/main/resources/v2_import_test/data/A000000b.rf differ
diff --git a/test/src/main/resources/v2_import_test/data/distcp.txt
b/test/src/main/resources/v2_import_test/data/distcp.txt
new file mode 100644
index 0000000000..672c11fba8
--- /dev/null
+++ b/test/src/main/resources/v2_import_test/data/distcp.txt
@@ -0,0 +1,5 @@
+hdfs://localhost:8020/accumulo/tables/1/default_tablet/A000000b.rf
+hdfs://localhost:8020/accumulo/tables/1/t-0000002/A000000a.rf
+hdfs://localhost:8020/accumulo/tables/1/t-0000001/A0000009.rf
+hdfs://localhost:8020/accumulo/tables/1/t-0000003/A0000008.rf
+hdfs://localhost:8020/accumulo/export_test/exportMetadata.zip
diff --git a/test/src/main/resources/v2_import_test/data/exportMetadata.zip
b/test/src/main/resources/v2_import_test/data/exportMetadata.zip
new file mode 100644
index 0000000000..04494de5f7
Binary files /dev/null and
b/test/src/main/resources/v2_import_test/data/exportMetadata.zip differ
