KYLIN-1894 fix test and cleanup tmp files Signed-off-by: shaofengshi <shaofeng...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28a2b9f7 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28a2b9f7 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28a2b9f7 Branch: refs/heads/1.5.x-CDH5.7 Commit: 28a2b9f7eadc05fdf4e0a9e268b0adb142bdfccf Parents: f0db336 Author: sunyerui <sunye...@gmail.com> Authored: Mon Jul 18 17:50:12 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Mon Jul 18 18:37:46 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/dict/AppendTrieDictionary.java | 2 +- .../kylin/dict/AppendTrieDictionaryTest.java | 16 +++++++-- .../apache/kylin/dict/CachedTreeMapTest.java | 34 +++++++++++++------- 3 files changed, 36 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/28a2b9f7/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java index 4cce586..14980bf 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java @@ -950,8 +950,8 @@ public class AppendTrieDictionary<T> extends Dictionary<T> { if (dict == null) { dict = new AppendTrieDictionary<T>(); } - dict.flushIndex((CachedTreeMap) mutableDictSliceMap); dict.update(baseDir, baseId, maxId, maxValueLength, nValues, bytesConverter, (CachedTreeMap)mutableDictSliceMap); + dict.flushIndex((CachedTreeMap) mutableDictSliceMap); return dict; } http://git-wip-us.apache.org/repos/asf/kylin/blob/28a2b9f7/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java index b81a439..4266f2a 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java @@ -26,6 +26,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; +import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -49,6 +50,8 @@ import org.junit.Test; */ public class AppendTrieDictionaryTest { + public static final String BASE_DIR = "/tmp/kylin_append_dict"; + @BeforeClass public static void setUp() { KylinConfig.destroyInstance(); @@ -56,7 +59,7 @@ public class AppendTrieDictionaryTest { KylinConfig config = KylinConfig.getInstanceFromEnv(); config.setAppendDictEntrySize(50000); config.setAppendDictCacheSize(3); - config.setProperty("kylin.hdfs.working.dir", "/tmp/kylin_append_dict"); + config.setProperty("kylin.hdfs.working.dir", BASE_DIR); } @AfterClass @@ -66,6 +69,13 @@ public class AppendTrieDictionaryTest { FileSystem.get(new Path(workingDir).toUri(), new Configuration()).delete(new Path(workingDir), true); } catch (IOException e) { } + File tmpLocalDir = new File(BASE_DIR); + if (tmpLocalDir.exists()) { + for (File f : tmpLocalDir.listFiles()) { + f.delete(); + } + tmpLocalDir.delete(); + } } public static final String[] words = new String[] { "paint", "par", "part", "parts", "partition", "partitions", "party", "partie", "parties", "patient", "taste", "tar", "trie", "try", "tries", "åå ¸", "åå ¸æ ", "åæ¯", // non-ascii characters @@ -122,7 +132,7 @@ public class AppendTrieDictionaryTest { @Test public void testHugeKeySet() throws IOException { BytesConverter converter = new StringBytesConverter(); - AppendTrieDictionary.Builder<String> b = AppendTrieDictionary.Builder.create("/tmp/kylin_append_dict"); + AppendTrieDictionary.Builder<String> b = AppendTrieDictionary.Builder.create(BASE_DIR); AppendTrieDictionary<String> dict = null; InputStream is = new FileInputStream("src/test/resources/dict/huge_key"); @@ -152,7 +162,7 @@ public class AppendTrieDictionaryTest { } BytesConverter converter = new StringBytesConverter(); - AppendTrieDictionary.Builder<String> b = AppendTrieDictionary.Builder.create("/tmp/kylin_append_dict"); + AppendTrieDictionary.Builder<String> b = AppendTrieDictionary.Builder.create(BASE_DIR); AppendTrieDictionary<String> dict = null; TreeMap<Integer, String> checkMap = new TreeMap<>(); int firstAppend = rnd.nextInt(strList.size() / 2); http://git-wip-us.apache.org/repos/asf/kylin/blob/28a2b9f7/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java index d2af621..3366bf6 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java @@ -3,6 +3,7 @@ package org.apache.kylin.dict; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; @@ -81,8 +82,7 @@ public class CachedTreeMapTest { public static final String backupDir = "/tmp/kylin_cachedtreemap_test.bak/"; public static final String tmpDir = "/tmp/kylin_cachedtreemap_test.tmp/"; - @After - public void afterTest() { + private static void cleanup() { File dir = new File(baseDir); if (dir.exists()) { for (File f : dir.listFiles()) { @@ -110,9 +110,19 @@ public class CachedTreeMapTest { VALUE_WRITE_ERROR_TOGGLE = false; } + @After + public void afterTest() { + cleanup(); + } + + @AfterClass + public static void tearDown() { + cleanup(); + } + @Test public void testCachedTreeMap() throws IOException { - CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir) + CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir) .persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build(); map.put(Key.of(1), Value.of("a")); map.put(Key.of(2), Value.of("b")); @@ -133,7 +143,7 @@ public class CachedTreeMapTest { assertEquals(5, dir.listFiles(new CachedFileFilter()).length); DataInputStream in = new DataInputStream(new FileInputStream(baseDir+".index")); - CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir) + CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir) .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build(); map2.readFields(in); assertEquals(5, map2.size()); @@ -152,7 +162,7 @@ public class CachedTreeMapTest { @Test public void testWriteFailed() throws IOException { // normal case - CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir) + CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir) .persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build(); map.put(Key.of(1), Value.of("a")); map.put(Key.of(2), Value.of("b")); @@ -167,14 +177,14 @@ public class CachedTreeMapTest { map.commit(false); DataInputStream in = new DataInputStream(new FileInputStream(baseDir+".index")); - CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir) + CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir) .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build(); map2.readFields(in); assertEquals(3, map2.size()); assertEquals("a", ((Value)map2.get(Key.of(1))).valueStr); // suppose write value failed and didn't commit data - map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir) + map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir) .persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build(); VALUE_WRITE_ERROR_TOGGLE = true; map.put(Key.of(1), Value.of("aa")); @@ -191,7 +201,7 @@ public class CachedTreeMapTest { // read map data should not be modified in = new DataInputStream(new FileInputStream(baseDir+".index")); - map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir) + map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir) .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build(); map2.readFields(in); assertEquals(3, map2.size()); @@ -203,7 +213,7 @@ public class CachedTreeMapTest { @Test public void testCommit() throws IOException { - CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir) + CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir) .persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build(); map.put(Key.of(1), Value.of("a")); map.put(Key.of(2), Value.of("b")); @@ -220,7 +230,7 @@ public class CachedTreeMapTest { assertFalse(new File(backupDir).exists()); DataInputStream in = new DataInputStream(new FileInputStream(baseDir+".index")); - CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir) + CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir) .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build(); map2.readFields(in); assertEquals(4, map2.size()); @@ -242,7 +252,7 @@ public class CachedTreeMapTest { assertEquals(4, new File(baseDir).listFiles(new CachedFileFilter()).length); in = new DataInputStream(new FileInputStream(baseDir+".index")); - map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir) + map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir) .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build(); map2.readFields(in); assertEquals(4, map2.size()); @@ -254,7 +264,7 @@ public class CachedTreeMapTest { assertEquals(6, new File(baseDir).listFiles(new CachedFileFilter()).length); in = new DataInputStream(new FileInputStream(baseDir+".index")); - map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir("file://"+baseDir) + map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir) .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build(); map2.readFields(in); assertEquals(6, map2.size());
