Repository: accumulo Updated Branches: refs/heads/1.8 e67317cb2 -> 24465d675
ACCUMULO-4165 Made RFileWriter start default LG automatically Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/24465d67 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/24465d67 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/24465d67 Branch: refs/heads/1.8 Commit: 24465d675501440685aa026a032211afa837db64 Parents: e67317c Author: Keith Turner <ke...@deenlo.com> Authored: Thu Jun 2 16:57:48 2016 -0400 Committer: Keith Turner <ke...@deenlo.com> Committed: Thu Jun 2 16:57:48 2016 -0400 ---------------------------------------------------------------------- .../accumulo/core/client/rfile/RFileWriter.java | 29 +++++++++--------- .../accumulo/core/client/rfile/RFileTest.java | 31 ++++---------------- 2 files changed, 20 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/24465d67/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java index aad4908..13da017 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java @@ -42,10 +42,9 @@ import com.google.common.base.Preconditions; * {@link TableOperations#importDirectory(String, String, String, boolean)} * * <p> - * A RFileWriter has the following constraints. Violating these contraints will result in runtime exceptions. + * A RFileWriter has the following constraints. Violating these constraints will result in runtime exceptions. * * <ul> - * <li>Before appending any keys, a locality group must be started by calling one of the startNewLocalityGroup functions or startDefaultLocalityGroup.</li> * <li>Keys must be appended in sorted order within a locality group.</li> * <li>Locality groups must have a mutually exclusive set of column families.</li> * <li>The default locality group must be started last.</li> @@ -128,14 +127,17 @@ public class RFileWriter implements AutoCloseable { } /** - * See have doc for {@link #startNewLocalityGroup(String, List)} + * See javadoc for {@link #startNewLocalityGroup(String, List)} + * + * @throws IllegalStateException + * When default locality group already started. */ public void startNewLocalityGroup(String name, byte[]... families) throws IOException { startNewLocalityGroup(name, Arrays.asList(families)); } /** - * See have doc for {@link #startNewLocalityGroup(String, List)}. + * See javadoc for {@link #startNewLocalityGroup(String, List)}. * * @param families * will be encoded using UTF-8 @@ -152,7 +154,7 @@ public class RFileWriter implements AutoCloseable { } /** - * See have doc for {@link #startNewLocalityGroup(String, List)}. + * See javadoc for {@link #startNewLocalityGroup(String, List)}. * * @param families * will be encoded using UTF-8 @@ -170,7 +172,7 @@ public class RFileWriter implements AutoCloseable { /** * A locality group in which the column families do not need to specified. The locality group must be started after all other locality groups. Can not append - * column families that were in a previous locality group. + * column families that were in a previous locality group. If no locality groups were started, then the first append will start the default locality group. * * @throws IllegalStateException * When default locality group already started. @@ -184,7 +186,8 @@ public class RFileWriter implements AutoCloseable { } /** - * Append the key and value to the last locality group that was started. + * Append the key and value to the last locality group that was started. If no locality group was started, then the default group will automatically be + * started. * * @param key * This key must be greater than or equal to the last key appended. For non-default locality groups, the keys column family must be one of the column @@ -195,11 +198,11 @@ public class RFileWriter implements AutoCloseable { * @throws IllegalArgumentException * This is thrown when data is appended out of order OR when the key contains a invalid visibility OR when a column family is not valid for a * locality group. - * @throws IllegalStateException - * Thrown when no locality group was started. */ public void append(Key key, Value val) throws IOException { - Preconditions.checkState(startedLG, "No locality group was started"); + if (!startedLG) { + startDefaultLocalityGroup(); + } Boolean wasChecked = (Boolean) validVisibilities.get(key.getColumnVisibilityData()); if (wasChecked == null) { byte[] cv = key.getColumnVisibilityData().toArray(); @@ -214,16 +217,14 @@ public class RFileWriter implements AutoCloseable { * * @param keyValues * The keys must be in sorted order. The first key returned by the iterable must be greater than or equal to the last key appended. For non-default - * locality groups, the keys column family must be one of the column families specified when calling startNewLocalityGroup(). Must be non-null. + * locality groups, the keys column family must be one of the column families specified when calling startNewLocalityGroup(). Must be non-null. If no + * locality group was started, then the default group will automatically be started. * * @throws IllegalArgumentException * This is thrown when data is appended out of order OR when the key contains a invalid visibility OR when a column family is not valid for a * locality group. - * @throws IllegalStateException - * When no locality group was started. */ public void append(Iterable<Entry<Key,Value>> keyValues) throws IOException { - Preconditions.checkState(startedLG, "No locality group was started"); for (Entry<Key,Value> entry : keyValues) { append(entry.getKey(), entry.getValue()); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/24465d67/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java index 3029592..4993810 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java @@ -104,7 +104,6 @@ public class RFileTest { String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(FileSystem.getLocal(new Configuration())).build()) { - writer.startDefaultLocalityGroup(); writer.append(testData.entrySet()); // TODO ensure compressors are returned } @@ -198,7 +197,6 @@ public class RFileTest { RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).withTableProperties(props).build(); SortedMap<Key,Value> testData1 = createTestData(10, 10, 10); - writer.startDefaultLocalityGroup(); writer.append(testData1.entrySet()); writer.close(); @@ -307,7 +305,6 @@ public class RFileTest { Value v2 = new Value("c".getBytes()); Value v3 = new Value("t".getBytes()); - writer.startDefaultLocalityGroup(); writer.append(k1, v1); writer.append(k2, v2); writer.append(k3, v3); @@ -345,7 +342,6 @@ public class RFileTest { Value v1 = new Value("p".getBytes()); Value v2 = new Value("".getBytes()); - writer.startDefaultLocalityGroup(); writer.append(k2, v2); writer.append(k1, v1); writer.close(); @@ -409,7 +405,6 @@ public class RFileTest { Value v1 = new Value("p".getBytes()); Value v2 = new Value("q".getBytes()); - writer.startDefaultLocalityGroup(); writer.append(k2, v2); writer.append(k1, v1); writer.close(); @@ -435,7 +430,6 @@ public class RFileTest { SamplerConfiguration sc = new SamplerConfiguration(RowSampler.class).setOptions(ImmutableMap.of("hasher", "murmur3_32", "modulus", "19")); RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).withSampler(sc).build(); - writer.startDefaultLocalityGroup(); writer.append(testData1.entrySet()); writer.close(); @@ -473,7 +467,6 @@ public class RFileTest { String testFile2 = createTmpTestFile(); RFileWriter writer = RFile.newWriter().to(testFile2).build(); - writer.startDefaultLocalityGroup(); writer.append(scanner); writer.close(); scanner.close(); @@ -517,7 +510,6 @@ public class RFileTest { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { - writer.startDefaultLocalityGroup(); writer.append(k2, v2); writer.append(k1, v1); } @@ -539,7 +531,6 @@ public class RFileTest { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { - writer.startDefaultLocalityGroup(); writer.append(data); } } @@ -570,32 +561,21 @@ public class RFileTest { } @Test(expected = IllegalStateException.class) - public void testNoLocalityGroupStarted() throws Exception { - LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); - String testFile = createTmpTestFile(); - try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { - Key k1 = new Key("r1", "f1", "q1"); - writer.append(k1, new Value("".getBytes())); - } - } - - @Test(expected = IllegalStateException.class) - public void testNoLocalityGroupStartedIterable() throws Exception { + public void testDoubleStart() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { - Key k1 = new Key("r1", "f1", "q1"); - Entry<Key,Value> entry = new AbstractMap.SimpleEntry<>(k1, new Value("".getBytes())); - writer.append(Collections.singletonList(entry)); + writer.startDefaultLocalityGroup(); + writer.startDefaultLocalityGroup(); } } @Test(expected = IllegalStateException.class) - public void testDoubleStart() throws Exception { + public void testAppendStartDefault() throws Exception { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { - writer.startDefaultLocalityGroup(); + writer.append(new Key("r1", "f1", "q1"), new Value("1".getBytes())); writer.startDefaultLocalityGroup(); } } @@ -605,7 +585,6 @@ public class RFileTest { LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); String testFile = createTmpTestFile(); try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build()) { - writer.startDefaultLocalityGroup(); Key k1 = new Key("r1", "f1", "q1"); writer.append(k1, new Value("".getBytes())); writer.startNewLocalityGroup("lg1", "fam1");
