Repository: accumulo Updated Branches: refs/heads/1.6 c85bf4f7d -> fa978b0c3 refs/heads/master cc48e37dd -> b5d7b9485
ACCUMULO-3312 Fix incorrect path generation during clone table. Expanded some ITs to actually cover this in the future. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fa978b0c Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fa978b0c Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fa978b0c Branch: refs/heads/1.6 Commit: fa978b0c3097e7d94fe1c8e9a49db24175711221 Parents: c85bf4f Author: Josh Elser <[email protected]> Authored: Thu Nov 6 23:38:37 2014 -0500 Committer: Josh Elser <[email protected]> Committed: Thu Nov 6 23:38:37 2014 -0500 ---------------------------------------------------------------------- .../accumulo/server/util/MetadataTableUtil.java | 2 +- .../accumulo/test/functional/CloneTestIT.java | 90 +++++++++++++++++++- 2 files changed, 90 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/fa978b0c/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java index 9cd02eb..6efd68c 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java @@ -882,7 +882,7 @@ public class MetadataTableUtil { Key k = entry.getKey(); Mutation m = new Mutation(k.getRow()); m.putDelete(k.getColumnFamily(), k.getColumnQualifier()); - String dir = volumeManager.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId + String dir = volumeManager.choose(ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR + Path.SEPARATOR + tableId + Path.SEPARATOR + new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, Constants.CLONE_PREFIX_BYTES)); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes(UTF_8))); http://git-wip-us.apache.org/repos/asf/accumulo/blob/fa978b0c/test/src/test/java/org/apache/accumulo/test/functional/CloneTestIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/CloneTestIT.java b/test/src/test/java/org/apache/accumulo/test/functional/CloneTestIT.java index fc69831..505dd5a 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/CloneTestIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/CloneTestIT.java @@ -19,6 +19,8 @@ package org.apache.accumulo.test.functional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -26,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.TreeSet; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; @@ -37,17 +40,22 @@ import org.apache.accumulo.core.client.admin.DiskUsage; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.server.ServerConstants; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; import org.junit.Assert; import org.junit.Test; /** - * + * */ public class CloneTestIT extends SimpleMacIT { @@ -88,6 +96,8 @@ public class CloneTestIT extends SimpleMacIT { checkData(table2, c); + checkMetadata(table2, c); + HashMap<String,String> tableProps = new HashMap<String,String>(); for (Entry<String,String> prop : c.tableOperations().getProperties(table2)) { tableProps.put(prop.getKey(), prop.getValue()); @@ -119,6 +129,48 @@ public class CloneTestIT extends SimpleMacIT { Assert.assertEquals(expected, actual); } + private void checkMetadata(String table, Connector conn) throws Exception { + Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY); + + s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME); + MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(s); + String tableId = conn.tableOperations().tableIdMap().get(table); + + Assert.assertNotNull("Could not get table id for " + table, tableId); + + s.setRange(Range.prefix(tableId)); + + Key k; + Text cf = new Text(), cq = new Text(); + Configuration conf = new Configuration(); + int itemsInspected = 0; + for (Entry<Key,Value> entry : s) { + itemsInspected++; + k = entry.getKey(); + k.getColumnFamily(cf); + k.getColumnQualifier(cq); + + if (cf.equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME)) { + Path p = new Path(cq.toString()); + // Will this actually work against HDFS? + FileSystem fs = p.getFileSystem(conf); + Assert.assertTrue("File does not exist: " + p, fs.exists(p)); + } else if (cf.equals(MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnFamily())) { + Assert.assertEquals("Saw unexpected cq", MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.getColumnQualifier(), cq); + Path tabletDir = new Path(entry.getValue().toString()); + Path tableDir = tabletDir.getParent(); + Path tablesDir = tableDir.getParent(); + + Assert.assertEquals(ServerConstants.TABLE_DIR, tablesDir.getName()); + } else { + Assert.fail("Got unexpected key-value: " + entry); + throw new RuntimeException(); + } + } + + Assert.assertTrue("Expected to find metadata entries", itemsInspected > 0); + } + private BatchWriter writeData(String table1, Connector c) throws TableNotFoundException, MutationsRejectedException { BatchWriter bw = c.createBatchWriter(table1, new BatchWriterConfig()); @@ -199,4 +251,40 @@ public class CloneTestIT extends SimpleMacIT { } + @Test + public void testCloneWithSplits() throws Exception { + Connector conn = getConnector(); + + List<Mutation> mutations = new ArrayList<Mutation>(); + TreeSet<Text> splits = new TreeSet<Text>(); + for (int i = 0; i < 10; i++) { + splits.add(new Text(Integer.toString(i))); + Mutation m = new Mutation(Integer.toString(i)); + m.put("", "", ""); + mutations.add(m); + } + + String[] tables = getUniqueNames(2); + + conn.tableOperations().create(tables[0]); + + conn.tableOperations().addSplits(tables[0], splits); + + BatchWriter bw = conn.createBatchWriter(tables[0], new BatchWriterConfig()); + bw.addMutations(mutations); + bw.close(); + + conn.tableOperations().clone(tables[0], tables[1], true, null, null); + + conn.tableOperations().deleteRows(tables[1], new Text("4"), new Text("8")); + + List<String> rows = Arrays.asList("0", "1", "2", "3", "4", "9"); + List<String> actualRows = new ArrayList<String>(); + for (Entry<Key,Value> entry : conn.createScanner(tables[1], Authorizations.EMPTY)) { + actualRows.add(entry.getKey().getRow().toString()); + } + + Assert.assertEquals(rows, actualRows); + } + }
