http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryChecker.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryChecker.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryChecker.java index 8af633e..94b6e9d 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryChecker.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryChecker.java @@ -17,8 +17,6 @@ */ package org.apache.kylin.dict.global; -import static org.apache.kylin.dict.global.GlobalDictHDFSStore.BUFFER_SIZE; - import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -30,6 +28,8 @@ import org.apache.hadoop.fs.Path; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.HadoopUtil; +import static org.apache.kylin.dict.global.GlobalDictHDFSStore.BUFFER_SIZE; + /** * Created by sunyerui on 16/11/15. */
http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java index a5ba46c..b30d5b9 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java @@ -18,15 +18,6 @@ package org.apache.kylin.dict.global; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.TreeMap; -import java.util.TreeSet; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -42,6 +33,15 @@ import org.apache.kylin.dict.BytesConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; + public class GlobalDictHDFSStore extends GlobalDictStore { static final Logger logger = LoggerFactory.getLogger(GlobalDictHDFSStore.class); @@ -241,12 +241,9 @@ public class GlobalDictHDFSStore extends GlobalDictStore { @Override public String copyToAnotherMeta(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException { - checkArgument(baseDir.startsWith(srcConfig.getHdfsWorkingDirectory()), - "Please check why current directory {} doesn't belong to source working directory {}", baseDir, - srcConfig.getHdfsWorkingDirectory()); + checkArgument(baseDir.startsWith(srcConfig.getHdfsWorkingDirectory()), "Please check why current directory {} doesn't belong to source working directory {}", baseDir, srcConfig.getHdfsWorkingDirectory()); - final String dstBaseDir = baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), - dstConfig.getHdfsWorkingDirectory()); + final String dstBaseDir = baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()); Long[] versions = listAllVersions(); if (versions.length == 0) { // empty dict, nothing to copy @@ -254,8 +251,7 @@ public class GlobalDictHDFSStore extends GlobalDictStore { } Path srcVersionDir = getVersionDir(versions[versions.length - 1]); - Path dstVersionDir = new Path(srcVersionDir.toString().replaceFirst(srcConfig.getHdfsWorkingDirectory(), - dstConfig.getHdfsWorkingDirectory())); + Path dstVersionDir = new Path(srcVersionDir.toString().replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory())); FileSystem dstFS = dstVersionDir.getFileSystem(conf); if (dstFS.exists(dstVersionDir)) { dstFS.delete(dstVersionDir, true); @@ -405,8 +401,7 @@ public class GlobalDictHDFSStore extends GlobalDictStore { public void sanityCheck(Path dir, GlobalDictMetadata metadata) throws IOException { for (Map.Entry<AppendDictSliceKey, String> entry : metadata.sliceFileMap.entrySet()) { if (!fs.exists(new Path(dir, entry.getValue()))) { - throw new RuntimeException("The slice file " + entry.getValue() + " for the key: " + entry.getKey() - + " must be existed!"); + throw new RuntimeException("The slice file " + entry.getValue() + " for the key: " + entry.getKey() + " must be existed!"); } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java index d35380c..7c89ea2 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java @@ -18,12 +18,11 @@ package org.apache.kylin.dict.global; -import java.util.NavigableMap; -import java.util.TreeMap; - +import com.google.common.base.Preconditions; import org.apache.kylin.dict.BytesConverter; -import com.google.common.base.Preconditions; +import java.util.NavigableMap; +import java.util.TreeMap; /** * Encapsulates the metadata for a particular version of the global dictionary. @@ -37,8 +36,7 @@ public class GlobalDictMetadata { public final BytesConverter bytesConverter; public final TreeMap<AppendDictSliceKey, String> sliceFileMap; // slice key -> slice file name - public GlobalDictMetadata(int baseId, int maxId, int maxValueLength, int nValues, BytesConverter bytesConverter, - NavigableMap<AppendDictSliceKey, String> sliceFileMap) { + public GlobalDictMetadata(int baseId, int maxId, int maxValueLength, int nValues, BytesConverter bytesConverter, NavigableMap<AppendDictSliceKey, String> sliceFileMap) { Preconditions.checkNotNull(bytesConverter, "bytesConverter"); Preconditions.checkNotNull(sliceFileMap, "sliceFileMap"); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java index c370afc..eaf0729 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java @@ -18,12 +18,11 @@ package org.apache.kylin.dict.global; -import java.io.IOException; - +import com.google.common.base.Preconditions; import org.apache.hadoop.fs.Path; import org.apache.kylin.common.KylinConfig; -import com.google.common.base.Preconditions; +import java.io.IOException; public abstract class GlobalDictStore { @@ -74,8 +73,7 @@ public abstract class GlobalDictStore { * @return file name of the new written slice * @throws IOException on I/O error */ - public abstract String writeSlice(String workingDir, AppendDictSliceKey key, AppendDictNode slice) - throws IOException; + public abstract String writeSlice(String workingDir, AppendDictSliceKey key, AppendDictNode slice) throws IOException; /** * Delete a slice with the specified file name. http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java index a22878f..a99ef29 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java @@ -83,8 +83,7 @@ abstract public class LookupTable<T> { Array<T> key = new Array<T>(keyCols); if (data.containsKey(key)) - throw new IllegalStateException("The table: " + tableDesc.getName() + " Dup key found, key=" - + toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + toString(value)); + throw new IllegalStateException("The table: " + tableDesc.getName() + " Dup key found, key=" + toString(keyCols) + ", value1=" + toString(data.get(key)) + ", value2=" + toString(value)); data.put(key, value); } @@ -148,8 +147,7 @@ abstract public class LookupTable<T> { } private boolean between(T beginValue, T v, T endValue, Comparator<T> comp) { - return (beginValue == null || comp.compare(beginValue, v) <= 0) - && (endValue == null || comp.compare(v, endValue) <= 0); + return (beginValue == null || comp.compare(beginValue, v) <= 0) && (endValue == null || comp.compare(v, endValue) <= 0); } abstract protected Comparator<T> getComparator(int colIdx); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java index 07659a8..45fd56b 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java @@ -41,8 +41,7 @@ public class SnapshotCLI { if (tableDesc == null) throw new IllegalArgumentException("Not table found by " + table); - SnapshotTable snapshot = snapshotMgr.rebuildSnapshot(SourceFactory.createReadableTable(tableDesc), tableDesc, - overwriteUUID); + SnapshotTable snapshot = snapshotMgr.rebuildSnapshot(SourceFactory.createReadableTable(tableDesc), tableDesc, overwriteUUID); System.out.println("resource path updated: " + snapshot.getResourcePath()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java index d7c1b68..b997a88 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java @@ -86,8 +86,7 @@ public class SnapshotManager { this.snapshotCache = CacheBuilder.newBuilder().removalListener(new RemovalListener<String, SnapshotTable>() { @Override public void onRemoval(RemovalNotification<String, SnapshotTable> notification) { - SnapshotManager.logger.info("Snapshot with resource path " + notification.getKey() - + " is removed due to " + notification.getCause()); + SnapshotManager.logger.info("Snapshot with resource path " + notification.getKey() + " is removed due to " + notification.getCause()); } }).maximumSize(config.getCachedSnapshotMaxEntrySize())// .expireAfterWrite(1, TimeUnit.DAYS).build(new CacheLoader<String, SnapshotTable>() { @@ -142,8 +141,7 @@ public class SnapshotManager { return trySaveNewSnapshot(snapshot); } - public SnapshotTable rebuildSnapshot(IReadableTable table, TableDesc tableDesc, String overwriteUUID) - throws IOException { + public SnapshotTable rebuildSnapshot(IReadableTable table, TableDesc tableDesc, String overwriteUUID) throws IOException { SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity()); snapshot.setUuid(overwriteUUID); @@ -216,8 +214,7 @@ public class SnapshotManager { logger.info("Loading snapshotTable from " + resourcePath + ", with loadData: " + loadData); ResourceStore store = MetadataManager.getInstance(this.config).getStore(); - SnapshotTable table = store.getResource(resourcePath, SnapshotTable.class, - loadData ? SnapshotTableSerializer.FULL_SERIALIZER : SnapshotTableSerializer.INFO_SERIALIZER); + SnapshotTable table = store.getResource(resourcePath, SnapshotTable.class, loadData ? SnapshotTableSerializer.FULL_SERIALIZER : SnapshotTableSerializer.INFO_SERIALIZER); if (loadData) logger.debug("Loaded snapshot at " + resourcePath); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java index ed0cb7a..1d7e474 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import com.google.common.base.Strings; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.kylin.common.persistence.ResourceStore; @@ -42,7 +43,6 @@ import org.apache.kylin.source.IReadableTable; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Strings; /** * @author yangli9 @@ -83,8 +83,7 @@ public class SnapshotTable extends RootPersistentEntity implements IReadableTabl while (reader.next()) { String[] row = reader.getRow(); if (row.length <= maxIndex) { - throw new IllegalStateException("Bad hive table row, " + tableDesc + " expect " + (maxIndex + 1) - + " columns, but got " + Arrays.toString(row)); + throw new IllegalStateException("Bad hive table row, " + tableDesc + " expect " + (maxIndex + 1) + " columns, but got " + Arrays.toString(row)); } for (ColumnDesc column : tableDesc.getColumns()) { String cell = row[column.getZeroBasedIndex()]; @@ -169,7 +168,7 @@ public class SnapshotTable extends RootPersistentEntity implements IReadableTabl public boolean exists() throws IOException { return true; } - + /** * a naive implementation * http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java index cd86936..4820318 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java @@ -61,14 +61,14 @@ public class DictionaryManagerTest extends LocalFileMetadataTestCase { assertEquals(lookupDate, dictMgr.decideSourceData(innerModel, lookupDate)); assertEquals(formatName, dictMgr.decideSourceData(innerModel, formatName)); } - + { DataModelDesc outerModel = metaMgr.getDataModelDesc("test_kylin_left_join_model_desc"); TblColRef factDate = outerModel.findColumn("TEST_KYLIN_FACT.CAL_DT"); assertEquals(factDate, dictMgr.decideSourceData(outerModel, factDate)); } } - + @Test public void testBuildSaveDictionary() throws IOException { KylinConfig config = KylinConfig.getInstanceFromEnv(); @@ -80,32 +80,25 @@ public class DictionaryManagerTest extends LocalFileMetadataTestCase { // non-exist input returns null; DictionaryInfo nullInfo = dictMgr.buildDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path")); assertEquals(null, nullInfo); - - DictionaryInfo info1 = dictMgr.buildDictionary(model, col, - MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3")); + + DictionaryInfo info1 = dictMgr.buildDictionary(model, col, MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3")); assertEquals(3, info1.getDictionaryObject().getSize()); // same input returns same dict - DictionaryInfo info2 = dictMgr.buildDictionary(model, col, - MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3")); + DictionaryInfo info2 = dictMgr.buildDictionary(model, col, MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3")); assertTrue(info1 == info2); - + // same input values (different path) returns same dict - DictionaryInfo info3 = dictMgr.buildDictionary(model, col, - MockupReadableTable.newSingleColumnTable("/a/different/path", "1", "2", "3")); + DictionaryInfo info3 = dictMgr.buildDictionary(model, col, MockupReadableTable.newSingleColumnTable("/a/different/path", "1", "2", "3")); assertTrue(info1 == info3); - + // save dictionary works in spite of non-exist table - Dictionary<String> dict = DictionaryGenerator.buildDictionary(col.getType(), - new IterableDictionaryValueEnumerator("1", "2", "3")); - DictionaryInfo info4 = dictMgr.saveDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path"), - dict); + Dictionary<String> dict = DictionaryGenerator.buildDictionary(col.getType(), new IterableDictionaryValueEnumerator("1", "2", "3")); + DictionaryInfo info4 = dictMgr.saveDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path"), dict); assertTrue(info1 == info4); - - Dictionary<String> dict2 = DictionaryGenerator.buildDictionary(col.getType(), - new IterableDictionaryValueEnumerator("1", "2", "3", "4")); - DictionaryInfo info5 = dictMgr.saveDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path"), - dict2); + + Dictionary<String> dict2 = DictionaryGenerator.buildDictionary(col.getType(), new IterableDictionaryValueEnumerator("1", "2", "3", "4")); + DictionaryInfo info5 = dictMgr.saveDictionary(model, col, MockupReadableTable.newNonExistTable("/a/path"), dict2); assertTrue(info1 != info5); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java index 794f334..4b386a7 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java @@ -37,7 +37,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -public class DictionaryProviderTest extends LocalFileMetadataTestCase { +public class DictionaryProviderTest extends LocalFileMetadataTestCase{ @Before public void setUp() throws Exception { @@ -52,28 +52,23 @@ public class DictionaryProviderTest extends LocalFileMetadataTestCase { @Test public void testReadWrite() throws Exception { //string dict - Dictionary<String> dict = getDict(DataType.getType("string"), - Arrays.asList(new String[] { "a", "b" }).iterator()); + Dictionary<String> dict = getDict(DataType.getType("string"), Arrays.asList(new String[] { "a", "b" }).iterator()); readWriteTest(dict); //number dict - Dictionary<String> dict2 = getDict(DataType.getType("long"), - Arrays.asList(new String[] { "1", "2" }).iterator()); + Dictionary<String> dict2 = getDict(DataType.getType("long"), Arrays.asList(new String[] { "1", "2" }).iterator()); readWriteTest(dict2); //date dict - Dictionary<String> dict3 = getDict(DataType.getType("datetime"), - Arrays.asList(new String[] { "20161122", "20161123" }).iterator()); + Dictionary<String> dict3 = getDict(DataType.getType("datetime"), Arrays.asList(new String[] { "20161122", "20161123" }).iterator()); readWriteTest(dict3); //date dict - Dictionary<String> dict4 = getDict(DataType.getType("datetime"), - Arrays.asList(new String[] { "2016-11-22", "2016-11-23" }).iterator()); + Dictionary<String> dict4 = getDict(DataType.getType("datetime"), Arrays.asList(new String[] { "2016-11-22", "2016-11-23" }).iterator()); readWriteTest(dict4); //date dict try { - Dictionary<String> dict5 = getDict(DataType.getType("date"), - Arrays.asList(new String[] { "2016-11-22", "20161122" }).iterator()); + Dictionary<String> dict5 = getDict(DataType.getType("date"), Arrays.asList(new String[] { "2016-11-22", "20161122" }).iterator()); readWriteTest(dict5); fail("Date format not correct.Should throw exception"); } catch (IllegalArgumentException e) { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java index a9d2398..53c9476 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MockupReadableTable.java @@ -34,7 +34,7 @@ public class MockupReadableTable implements IReadableTable { } return new MockupReadableTable(content, sig, true); } - + public static IReadableTable newNonExistTable(String path) { TableSignature sig = new TableSignature(path, -1, 0); return new MockupReadableTable(null, sig, false); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java index d25f2f4..2e90bcf 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java @@ -56,52 +56,52 @@ public class MultipleDictionaryValueEnumeratorTest { @Test public void testNormalDicts() throws IOException { List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(2); - dictionaryInfoList.add(createDictInfo(new int[] { 0, 1, 2 })); - dictionaryInfoList.add(createDictInfo(new int[] { 4, 5, 6 })); + dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2})); + dictionaryInfoList.add(createDictInfo(new int[]{4, 5, 6})); Integer[] values = enumerateDictInfoList(dictionaryInfoList); assertEquals(6, values.length); - assertArrayEquals(new Integer[] { 0, 1, 2, 4, 5, 6 }, values); + assertArrayEquals(new Integer[]{0, 1, 2, 4, 5, 6}, values); } @Test public void testFirstEmptyDicts() throws IOException { List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(2); - dictionaryInfoList.add(createDictInfo(new int[] {})); - dictionaryInfoList.add(createDictInfo(new int[] { 4, 5, 6 })); + dictionaryInfoList.add(createDictInfo(new int[]{})); + dictionaryInfoList.add(createDictInfo(new int[]{4, 5, 6})); Integer[] values = enumerateDictInfoList(dictionaryInfoList); assertEquals(3, values.length); - assertArrayEquals(new Integer[] { 4, 5, 6 }, values); + assertArrayEquals(new Integer[]{4, 5, 6}, values); } @Test public void testMiddleEmptyDicts() throws IOException { List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(3); - dictionaryInfoList.add(createDictInfo(new int[] { 0, 1, 2 })); - dictionaryInfoList.add(createDictInfo(new int[] {})); - dictionaryInfoList.add(createDictInfo(new int[] { 7, 8, 9 })); + dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2})); + dictionaryInfoList.add(createDictInfo(new int[]{})); + dictionaryInfoList.add(createDictInfo(new int[]{7, 8, 9})); Integer[] values = enumerateDictInfoList(dictionaryInfoList); assertEquals(6, values.length); - assertArrayEquals(new Integer[] { 0, 1, 2, 7, 8, 9 }, values); + assertArrayEquals(new Integer[]{0, 1, 2, 7, 8, 9}, values); } @Test public void testLastEmptyDicts() throws IOException { List<DictionaryInfo> dictionaryInfoList = new ArrayList<>(3); - dictionaryInfoList.add(createDictInfo(new int[] { 0, 1, 2 })); - dictionaryInfoList.add(createDictInfo(new int[] { 6, 7, 8 })); - dictionaryInfoList.add(createDictInfo(new int[] {})); + dictionaryInfoList.add(createDictInfo(new int[]{0, 1, 2})); + dictionaryInfoList.add(createDictInfo(new int[]{6, 7, 8})); + dictionaryInfoList.add(createDictInfo(new int[]{})); Integer[] values = enumerateDictInfoList(dictionaryInfoList); assertEquals(6, values.length); - assertArrayEquals(new Integer[] { 0, 1, 2, 6, 7, 8 }, values); + assertArrayEquals(new Integer[]{0, 1, 2, 6, 7, 8}, values); } public static class MockDictionary extends Dictionary<String> { private static final long serialVersionUID = 1L; - + public int[] values; @Override @@ -111,7 +111,7 @@ public class MultipleDictionaryValueEnumeratorTest { @Override public int getMaxId() { - return values.length - 1; + return values.length-1; } @Override @@ -134,17 +134,16 @@ public class MultipleDictionaryValueEnumeratorTest { return "" + values[id]; } + @Override - public void dump(PrintStream out) { - } + public void dump(PrintStream out) {} @Override - public void write(DataOutput out) throws IOException { - } + public void write(DataOutput out) throws IOException {} @Override - public void readFields(DataInput in) throws IOException { - } + public void readFields(DataInput in) throws IOException {} + @Override public boolean contains(Dictionary another) { @@ -152,4 +151,4 @@ public class MultipleDictionaryValueEnumeratorTest { } } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java index 9bdb174..8da7208 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java @@ -45,8 +45,7 @@ import com.google.common.collect.Sets; */ public class NumberDictionaryTest extends LocalFileMetadataTestCase { - Number2BytesConverter.NumberBytesCodec codec = new Number2BytesConverter.NumberBytesCodec( - MAX_DIGITS_BEFORE_DECIMAL_POINT); + Number2BytesConverter.NumberBytesCodec codec = new Number2BytesConverter.NumberBytesCodec(MAX_DIGITS_BEFORE_DECIMAL_POINT); Random rand = new Random(); @Before @@ -79,8 +78,7 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase { String[] ints = new String[] { "", "0", "5", "100", "13" }; // check "" is treated as NULL, not a code of dictionary - Dictionary<?> dict = DictionaryGenerator.buildDictionary(DataType.getType("integer"), - new IterableDictionaryValueEnumerator(ints)); + Dictionary<?> dict = DictionaryGenerator.buildDictionary(DataType.getType("integer"), new IterableDictionaryValueEnumerator(ints)); assertEquals(4, dict.getSize()); final int id = ((NumberDictionary<String>) dict).getIdFromValue(""); @@ -97,8 +95,7 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase { //test resolved jira-1800 checkCodec("-0.0045454354354354359999999999877218", "-9999999999999999999.9954545645645645640000000000122781;"); checkCodec("-0.009999999999877218", "-9999999999999999999.990000000000122781;"); - checkCodec("12343434372493274.438403840384023840253554345345345345", - "00012343434372493274.438403840384023840253554345345345345"); + checkCodec("12343434372493274.438403840384023840253554345345345345", "00012343434372493274.438403840384023840253554345345345345"); assertEquals("00000000000000000052.57", encodeNumber("52.5700")); assertEquals("00000000000000000000", encodeNumber("0.00")); assertEquals("00000000000000000000", encodeNumber("0.0")); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java index e1eb119..adc1074 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java @@ -18,16 +18,16 @@ package org.apache.kylin.dict; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Random; -import java.util.UUID; - import org.apache.kylin.common.util.Dictionary; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Random; +import java.util.UUID; + /** * Created by xiefan on 16-12-28. */ @@ -50,8 +50,7 @@ public class TrieDictionaryForestBenchmark { public void before() { int dataSize = 100 * 10000; TrieDictionaryBuilder<String> b1 = new TrieDictionaryBuilder<>(new StringBytesConverter()); - TrieDictionaryForestBuilder<String> b2 = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), 0, - 5); + TrieDictionaryForestBuilder<String> b2 = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), 0, 5); this.rawData = genStringDataSet(dataSize); for (String str : this.rawData) { b1.addValue(str); @@ -153,8 +152,7 @@ public class TrieDictionaryForestBenchmark { return System.currentTimeMillis() - startTime; } - private long runQueryIdByValueBytes(ArrayList<String> rawData, Dictionary<String> dict, int cardnality, - int testTimes) { + private long runQueryIdByValueBytes(ArrayList<String> rawData, Dictionary<String> dict, int cardnality, int testTimes) { long startTime = System.currentTimeMillis(); int step = 1; for (int i = 0; i < testTimes; i++) { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java index a8c9a67..82380b3 100755 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java @@ -186,14 +186,8 @@ public class TrieDictionaryForestTest { str.add("party"); str.add("parties"); str.add("paint"); - String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"; + String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"; System.out.println("The length of the long string is " + longStr.length()); str.add(longStr); @@ -745,8 +739,7 @@ public class TrieDictionaryForestTest { System.out.println("times:" + i); } - System.out.println("compare build time. Old trie : " + oldDictTotalBuildTime / 1000.0 + "s.New trie : " - + newDictTotalBuildTime / 1000.0 + "s"); + System.out.println("compare build time. Old trie : " + oldDictTotalBuildTime / 1000.0 + "s.New trie : " + newDictTotalBuildTime / 1000.0 + "s"); } private void evaluateDataSize(ArrayList<String> list) { @@ -812,8 +805,7 @@ public class TrieDictionaryForestTest { benchmark("Benchmark", dict, set, map, strArray, array); } - private static int benchmark(String msg, TrieDictionaryForest<String> dict, TreeSet<String> set, - HashMap<String, Integer> map, String[] strArray, byte[][] array) { + private static int benchmark(String msg, TrieDictionaryForest<String> dict, TreeSet<String> set, HashMap<String, Integer> map, String[] strArray, byte[][] array) { int n = set.size(); int times = Math.max(10 * 1000 * 1000 / n, 1); // run 10 million lookups int keep = 0; // make sure JIT don't OPT OUT function calls under test @@ -937,16 +929,14 @@ public class TrieDictionaryForestTest { } public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterable<String> strs, int baseId) { - TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), - baseId); + TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId); for (String s : strs) b.addValue(s); return b; } public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterable<String> strs, int baseId, int treeSize) { - TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), - baseId); + TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId); b.setMaxTrieTreeSize(treeSize); for (String s : strs) { b.addValue(s); @@ -955,8 +945,7 @@ public class TrieDictionaryForestTest { } public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterator<String> strs, int baseId, int treeSize) { - TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), - baseId); + TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId); b.setMaxTrieTreeSize(treeSize); while (strs.hasNext()) b.addValue(strs.next()); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java index af4c9cd..13c83ac 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java @@ -111,14 +111,8 @@ public class TrieDictionaryTest { str.add("party"); str.add("parties"); str.add("paint"); - String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"; + String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"; System.out.println("The length of the long string is " + longStr.length()); str.add(longStr); @@ -275,8 +269,7 @@ public class TrieDictionaryTest { benchmark("Benchmark", dict, set, map, strArray, array); } - private static int benchmark(String msg, TrieDictionary<String> dict, TreeSet<String> set, - HashMap<String, Integer> map, String[] strArray, byte[][] array) { + private static int benchmark(String msg, TrieDictionary<String> dict, TreeSet<String> set, HashMap<String, Integer> map, String[] strArray, byte[][] array) { int n = set.size(); int times = Math.max(10 * 1000 * 1000 / n, 1); // run 10 million lookups int keep = 0; // make sure JIT don't OPT OUT function calls under test @@ -412,12 +405,10 @@ public class TrieDictionaryTest { @Test public void testSuperLongStringValue() { - String longPrefix = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" - + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"; + String longPrefix = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"; TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); - String v1 = longPrefix - + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; + String v1 = longPrefix + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; String v2 = longPrefix + "xyz"; b.addValue(v1); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java index 1671748..47011fe 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java @@ -66,8 +66,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { public void beforeTest() { staticCreateTestMetadata(); KylinConfig.getInstanceFromEnv().setProperty("kylin.dictionary.append-entry-size", "50000"); - BASE_DIR = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR - + "/"; + BASE_DIR = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR + "/"; LOCAL_BASE_DIR = getLocalWorkingDirectory() + "/resources/GlobalDict" + RESOURCE_DIR + "/"; } @@ -85,19 +84,11 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { } } - private static final String[] words = new String[] { "paint", "par", "part", "parts", "partition", "partitions", - "party", "partie", "parties", "patient", "taste", "tar", "trie", "try", "tries", "åå ¸", "åå ¸æ ", "åæ¯", // non-ascii characters + private static final String[] words = new String[] { "paint", "par", "part", "parts", "partition", "partitions", "party", "partie", "parties", "patient", "taste", "tar", "trie", "try", "tries", "åå ¸", "åå ¸æ ", "åæ¯", // non-ascii characters "", // empty - "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii", - "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiipaiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii", - "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" - + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk", + "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii", "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiipaiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii", + "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk", "paint", "tar", "try", // some dup }; @@ -177,8 +168,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { dict.dump(System.out); } - private void testStringDictAppend(ArrayList<String> list, ArrayList<String> notfound, boolean shuffleList) - throws IOException { + private void testStringDictAppend(ArrayList<String> list, ArrayList<String> notfound, boolean shuffleList) throws IOException { Random rnd = new Random(System.currentTimeMillis()); ArrayList<String> strList = new ArrayList<String>(); strList.addAll(list); @@ -205,8 +195,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { byte[] bytes = converter.convertToBytes(str); int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0); assertNotEquals(String.format("Value %s not exist", str), -1, id); - assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), - checkMap.containsKey(id) && !str.equals(checkMap.get(id))); + assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id))); checkMap.put(id, str); } @@ -230,8 +219,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id)); } else { // check second append str, should be new id - assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), - checkMap.containsKey(id) && !str.equals(checkMap.get(id))); + assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id))); checkMap.put(id, str); } } @@ -256,8 +244,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id)); } else { // check third append str, should be new id - assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), - checkMap.containsKey(id) && !str.equals(checkMap.get(id))); + assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id))); checkMap.put(id, str); } } @@ -278,8 +265,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { } } - private static AppendTrieDictionary<String> testSerialize(AppendTrieDictionary<String> dict, - BytesConverter converter) { + private static AppendTrieDictionary<String> testSerialize(AppendTrieDictionary<String> dict, BytesConverter converter) { try { ByteArrayOutputStream bout = new ByteArrayOutputStream(); DataOutputStream dataout = new DataOutputStream(bout); @@ -526,8 +512,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase { Path v2IndexFile = new Path(versionPath, V2_INDEX_NAME); fs.delete(v2IndexFile, true); - GlobalDictHDFSStore.IndexFormat indexFormatV1 = new GlobalDictHDFSStore.IndexFormatV1(fs, - HadoopUtil.getCurrentConfiguration()); + GlobalDictHDFSStore.IndexFormat indexFormatV1 = new GlobalDictHDFSStore.IndexFormatV1(fs, HadoopUtil.getCurrentConfiguration()); indexFormatV1.writeIndexFile(versionPath, metadata); //convert v2 fileName format to v1 fileName format http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java index b798b86..5a5c988 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java @@ -72,8 +72,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase { @Test public void testMapRange() throws Exception { - Pair<String, String> results = lookupTable.mapRange("CAL_DT", millis("2012-01-24"), millis("2012-12-30"), - "QTR_BEG_DT"); + Pair<String, String> results = lookupTable.mapRange("CAL_DT", millis("2012-01-24"), millis("2012-12-30"), "QTR_BEG_DT"); Assert.assertTrue(results != null); System.out.println("The first qtr_beg_dt is " + results.getFirst()); @@ -85,8 +84,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase { @Test public void testMapRange2() throws Exception { - Pair<String, String> results = lookupTable.mapRange("WEEK_BEG_DT", millis("2013-05-01"), millis("2013-08-01"), - "CAL_DT"); + Pair<String, String> results = lookupTable.mapRange("WEEK_BEG_DT", millis("2013-05-01"), millis("2013-08-01"), "CAL_DT"); System.out.println(DateFormat.formatToDateStr(Long.parseLong(results.getFirst()))); System.out.println(DateFormat.formatToDateStr(Long.parseLong(results.getSecond()))); @@ -111,7 +109,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase { } @Test - public void testGetClassName() { + public void testGetClassName(){ String name = TrieDictionaryForest.class.getName(); System.out.println(name); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java index 302af14..acaa7da 100644 --- a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java +++ b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java @@ -45,11 +45,11 @@ public class EngineFactory { public static IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc) { return batchEngine(cubeDesc).getJoinedFlatTableDesc(cubeDesc); } - + public static IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment) { return batchEngine(newSegment).getJoinedFlatTableDesc(newSegment); } - + /** Build a new cube segment, typically its time range appends to the end of current cube. */ public static DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter) { return batchEngine(newSegment).createBatchCubingJob(newSegment, submitter); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java b/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java index 3abdbc2..754dbde 100644 --- a/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java +++ b/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java @@ -24,11 +24,11 @@ import org.apache.kylin.job.execution.DefaultChainedExecutable; import org.apache.kylin.metadata.model.IJoinedFlatTableDesc; public interface IBatchCubingEngine { - + /** Mark deprecated to indicate for test purpose only */ @Deprecated public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc); - + public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment); /** Build a new cube segment, typically its time range appends to the end of current cube. */ http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/JobInstance.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java index b45e3e0..8dcdff6 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java +++ b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java @@ -70,8 +70,7 @@ public class JobInstance extends RootPersistentEntity implements Comparable<JobI public JobStep getRunningStep() { for (JobStep step : this.getSteps()) { - if (step.getStatus().equals(JobStepStatusEnum.RUNNING) - || step.getStatus().equals(JobStepStatusEnum.WAITING)) { + if (step.getStatus().equals(JobStepStatusEnum.RUNNING) || step.getStatus().equals(JobStepStatusEnum.WAITING)) { return step; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index 3f82f16..4665465 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -87,21 +87,18 @@ public class JoinedFlatTable { if (kylinConfig.isAdvancedFlatTableUsed()) { try { Class advancedFlatTable = Class.forName(kylinConfig.getAdvancedFlatTableClass()); - Method method = advancedFlatTable.getMethod("generateInsertDataStatement", IJoinedFlatTableDesc.class, - JobEngineConfig.class); + Method method = advancedFlatTable.getMethod("generateInsertDataStatement", IJoinedFlatTableDesc.class, JobEngineConfig.class); return (String) method.invoke(null, flatDesc); } catch (Exception e) { throw new RuntimeException(e); } } - return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) - + ";\n"; + return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + ";\n"; } public static String generateInsertPartialDataStatement(IJoinedFlatTableDesc flatDesc, String statement) { - return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) - + statement + ";\n"; + return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + statement + ";\n"; } public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc) { @@ -123,8 +120,7 @@ public class JoinedFlatTable { final StringBuilder sql = new StringBuilder(); final TableRef rootTbl = flatDesc.getDataModel().getRootFactTable(); sql.append("dfs -mkdir -p " + outputDir + ";\n"); - sql.append("INSERT OVERWRITE DIRECTORY '" + outputDir + "' SELECT count(*) FROM " + rootTbl.getTableIdentity() - + " " + rootTbl.getAlias() + "\n"); + sql.append("INSERT OVERWRITE DIRECTORY '" + outputDir + "' SELECT count(*) FROM " + rootTbl.getTableIdentity() + " " + rootTbl.getAlias() + "\n"); appendWhereStatement(flatDesc, sql); return sql.toString(); } @@ -195,8 +191,7 @@ public class JoinedFlatTable { if (!(dateStart == 0 && dateEnd == Long.MAX_VALUE)) { whereBuilder.append(hasCondition ? " AND (" : " ("); - whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, - dateStart, dateEnd)); + whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, dateStart, dateEnd)); whereBuilder.append(")\n"); hasCondition = true; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java b/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java index 3b007f6..8be5d02 100644 --- a/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java +++ b/core-job/src/main/java/org/apache/kylin/job/common/PatternedLogger.java @@ -37,23 +37,21 @@ public class PatternedLogger extends BufferedLogger { private static final Pattern PATTERN_APP_ID = Pattern.compile("Submitted application (.*?) to ResourceManager"); private static final Pattern PATTERN_APP_URL = Pattern.compile("The url to track the job: (.*)"); private static final Pattern PATTERN_JOB_ID = Pattern.compile("Running job: (.*)"); - private static final Pattern PATTERN_HDFS_BYTES_WRITTEN = Pattern - .compile("(?:HD|MAPR)FS: Number of bytes written=(\\d+)"); + private static final Pattern PATTERN_HDFS_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS: Number of bytes written=(\\d+)"); private static final Pattern PATTERN_SOURCE_RECORDS_COUNT = Pattern.compile("Map input records=(\\d+)"); private static final Pattern PATTERN_SOURCE_RECORDS_SIZE = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write"); // hive private static final Pattern PATTERN_HIVE_APP_ID_URL = Pattern.compile("Starting Job = (.*?), Tracking URL = (.*)"); - private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern - .compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS"); + private static final Pattern PATTERN_HIVE_BYTES_WRITTEN = Pattern.compile("(?:HD|MAPR)FS Read: (\\d+) HDFS Write: (\\d+) SUCCESS"); - private static final Pattern PATTERN_HIVE_APP_ID_URL_2 = Pattern - .compile("Executing on YARN cluster with App id (.*?)"); + private static final Pattern PATTERN_HIVE_APP_ID_URL_2 = Pattern.compile("Executing on YARN cluster with App id (.*?)"); // spark private static final Pattern PATTERN_SPARK_APP_ID = Pattern.compile("Submitted application (.*?)"); private static final Pattern PATTERN_SPARK_APP_URL = Pattern.compile("tracking URL: (.*)"); + public PatternedLogger(Logger wrappedLogger) { super(wrappedLogger); } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java index 2b69d8f..9f431b0 100644 --- a/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java +++ b/core-job/src/main/java/org/apache/kylin/job/common/ShellExecutable.java @@ -19,7 +19,6 @@ package org.apache.kylin.job.common; import java.io.IOException; - import org.apache.kylin.common.util.Pair; import org.apache.kylin.job.exception.ExecuteException; import org.apache.kylin.job.execution.AbstractExecutable; @@ -44,11 +43,9 @@ public class ShellExecutable extends AbstractExecutable { try { logger.info("executing:" + getCmd()); final PatternedLogger patternedLogger = new PatternedLogger(logger); - final Pair<Integer, String> result = context.getConfig().getCliCommandExecutor().execute(getCmd(), - patternedLogger); + final Pair<Integer, String> result = context.getConfig().getCliCommandExecutor().execute(getCmd(), patternedLogger); getManager().addJobInfo(getId(), patternedLogger.getInfo()); - return new ExecuteResult(result.getFirst() == 0 ? ExecuteResult.State.SUCCEED : ExecuteResult.State.FAILED, - result.getSecond()); + return new ExecuteResult(result.getFirst() == 0 ? ExecuteResult.State.SUCCEED : ExecuteResult.State.FAILED, result.getSecond()); } catch (IOException e) { logger.error("job:" + getId() + " execute finished with exception", e); return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage()); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java index 6f30d06..d7f6292 100644 --- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java +++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java @@ -57,11 +57,5 @@ public final class ExecutableConstants { public static final String STEP_NAME_GARBAGE_COLLECTION_HBASE = "Garbage Collection on HBase"; public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage Collection on HDFS"; public static final String STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE = "Redistribute Flat Hive Table"; - public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" - + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" - + "<li>Env: ${env_name}</li>" + "<li>Project: ${project_name}</li>" + "<li>Cube Name: ${cube_name}</li>" - + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" - + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" - + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" - + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>"; + public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Env: ${env_name}</li>" + "<li>Project: ${project_name}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>"; } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java b/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java index 8a9693d..4c6ac97 100644 --- a/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java +++ b/core-job/src/main/java/org/apache/kylin/job/constant/JobStatusEnum.java @@ -43,8 +43,7 @@ public enum JobStatusEnum { } public boolean isComplete() { - return code == JobStatusEnum.FINISHED.getCode() || code == JobStatusEnum.ERROR.getCode() - || code == JobStatusEnum.DISCARDED.getCode(); + return code == JobStatusEnum.FINISHED.getCode() || code == JobStatusEnum.ERROR.getCode() || code == JobStatusEnum.DISCARDED.getCode(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java b/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java index d83b322..08cd138 100644 --- a/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java +++ b/core-job/src/main/java/org/apache/kylin/job/constant/JobStepStatusEnum.java @@ -42,8 +42,7 @@ public enum JobStepStatusEnum { } public boolean isComplete() { - return code == JobStepStatusEnum.FINISHED.getCode() || code == JobStepStatusEnum.ERROR.getCode() - || code == JobStepStatusEnum.DISCARDED.getCode(); + return code == JobStepStatusEnum.FINISHED.getCode() || code == JobStepStatusEnum.ERROR.getCode() || code == JobStepStatusEnum.DISCARDED.getCode(); } public boolean isRunable() { http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java index b395ff8..70799d8 100644 --- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java +++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java @@ -42,8 +42,7 @@ import com.google.common.collect.Lists; public class ExecutableDao { private static final Serializer<ExecutablePO> JOB_SERIALIZER = new JsonSerializer<ExecutablePO>(ExecutablePO.class); - private static final Serializer<ExecutableOutputPO> JOB_OUTPUT_SERIALIZER = new JsonSerializer<ExecutableOutputPO>( - ExecutableOutputPO.class); + private static final Serializer<ExecutableOutputPO> JOB_OUTPUT_SERIALIZER = new JsonSerializer<ExecutableOutputPO>(ExecutableOutputPO.class); private static final Logger logger = LoggerFactory.getLogger(ExecutableDao.class); private static final ConcurrentMap<KylinConfig, ExecutableDao> CACHE = new ConcurrentHashMap<KylinConfig, ExecutableDao>(); @@ -107,8 +106,7 @@ public class ExecutableDao { public List<ExecutableOutputPO> getJobOutputs() throws PersistentException { try { - return store.getAllResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT, ExecutableOutputPO.class, - JOB_OUTPUT_SERIALIZER); + return store.getAllResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT, ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER); } catch (IOException e) { logger.error("error get all Jobs:", e); throw new PersistentException(e); @@ -117,8 +115,7 @@ public class ExecutableDao { public List<ExecutableOutputPO> getJobOutputs(long timeStart, long timeEndExclusive) throws PersistentException { try { - return store.getAllResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT, timeStart, timeEndExclusive, - ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER); + return store.getAllResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT, timeStart, timeEndExclusive, ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER); } catch (IOException e) { logger.error("error get all Jobs:", e); throw new PersistentException(e); @@ -136,8 +133,7 @@ public class ExecutableDao { public List<ExecutablePO> getJobs(long timeStart, long timeEndExclusive) throws PersistentException { try { - return store.getAllResources(ResourceStore.EXECUTE_RESOURCE_ROOT, timeStart, timeEndExclusive, - ExecutablePO.class, JOB_SERIALIZER); + return store.getAllResources(ResourceStore.EXECUTE_RESOURCE_ROOT, timeStart, timeEndExclusive, ExecutablePO.class, JOB_SERIALIZER); } catch (IOException e) { logger.error("error get all Jobs:", e); throw new PersistentException(e); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java index 43789d6..c9ac583 100644 --- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java +++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java @@ -59,8 +59,7 @@ public class JobEngineConfig { File jobConfig = getJobConfig(hadoopJobConfFile); if (jobConfig == null || !jobConfig.exists()) { - logger.warn( - "fail to locate " + hadoopJobConfFile + ", trying to locate " + HADOOP_JOB_CONF_FILENAME + ".xml"); + logger.warn("fail to locate " + hadoopJobConfFile + ", trying to locate " + HADOOP_JOB_CONF_FILENAME + ".xml"); jobConfig = getJobConfig(HADOOP_JOB_CONF_FILENAME + ".xml"); if (jobConfig == null || !jobConfig.exists()) { logger.error("fail to locate " + HADOOP_JOB_CONF_FILENAME + ".xml"); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java b/core-job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java index 67ff03f..f19b0ca 100644 --- a/core-job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java +++ b/core-job/src/main/java/org/apache/kylin/job/exception/IllegalStateTranferException.java @@ -39,8 +39,7 @@ public class IllegalStateTranferException extends RuntimeException { super(cause); } - public IllegalStateTranferException(String message, Throwable cause, boolean enableSuppression, - boolean writableStackTrace) { + public IllegalStateTranferException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java index 63a36b5..1f1be41 100644 --- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java +++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java @@ -60,16 +60,16 @@ public abstract class AbstractExecutable implements Executable, Idempotent { public AbstractExecutable() { setId(UUID.randomUUID().toString()); } - + protected void initConfig(KylinConfig config) { Preconditions.checkState(this.config == null || this.config == config); this.config = config; } - + protected KylinConfig getConfig() { return config; } - + protected ExecutableManager getManager() { return ExecutableManager.getInstance(config); } @@ -391,7 +391,6 @@ public abstract class AbstractExecutable implements Executable, Idempotent { @Override public String toString() { - return Objects.toStringHelper(this).add("id", getId()).add("name", getName()).add("state", getStatus()) - .toString(); + return Objects.toStringHelper(this).add("id", getId()).add("name", getName()).add("state", getStatus()).toString(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java index 817565b..8bcaaad 100644 --- a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java +++ b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java @@ -43,7 +43,7 @@ public class DefaultChainedExecutable extends AbstractExecutable implements Chai sub.initConfig(config); } } - + @Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { List<? extends Executable> executables = getTasks(); @@ -58,8 +58,7 @@ public class DefaultChainedExecutable extends AbstractExecutable implements Chai // the job is paused break; } else if (state == ExecutableState.ERROR) { - throw new IllegalStateException( - "invalid subtask state, subtask:" + subTask.getName() + ", state:" + subTask.getStatus()); + throw new IllegalStateException("invalid subtask state, subtask:" + subTask.getName() + ", state:" + subTask.getStatus()); } if (subTask.isRunnable()) { return subTask.execute(context); @@ -89,7 +88,7 @@ public class DefaultChainedExecutable extends AbstractExecutable implements Chai @Override protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) { ExecutableManager mgr = getManager(); - + if (isDiscarded()) { setEndTime(System.currentTimeMillis()); notifyUserStatusChange(executableContext, ExecutableState.DISCARDED); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java index 4dc22b7..2272582 100644 --- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java +++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java @@ -18,10 +18,6 @@ package org.apache.kylin.job.execution; -import static org.apache.kylin.job.constant.ExecutableConstants.MR_JOB_ID; -import static org.apache.kylin.job.constant.ExecutableConstants.YARN_APP_ID; -import static org.apache.kylin.job.constant.ExecutableConstants.YARN_APP_URL; - import java.lang.reflect.Constructor; import java.util.HashMap; import java.util.IllegalFormatException; @@ -46,6 +42,10 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import static org.apache.kylin.job.constant.ExecutableConstants.MR_JOB_ID; +import static org.apache.kylin.job.constant.ExecutableConstants.YARN_APP_ID; +import static org.apache.kylin.job.constant.ExecutableConstants.YARN_APP_URL; + /** */ public class ExecutableManager { @@ -235,8 +235,7 @@ public class ExecutableManager { * @param expectedClass * @return */ - public List<AbstractExecutable> getAllAbstractExecutables(long timeStartInMillis, long timeEndInMillis, - Class<? extends AbstractExecutable> expectedClass) { + public List<AbstractExecutable> getAllAbstractExecutables(long timeStartInMillis, long timeEndInMillis, Class<? extends AbstractExecutable> expectedClass) { try { List<AbstractExecutable> ret = Lists.newArrayList(); for (ExecutablePO po : executableDao.getJobs(timeStartInMillis, timeEndInMillis)) { @@ -383,8 +382,7 @@ public class ExecutableManager { ExecutableState oldStatus = ExecutableState.valueOf(jobOutput.getStatus()); if (newStatus != null && oldStatus != newStatus) { if (!ExecutableState.isValidStateTransfer(oldStatus, newStatus)) { - throw new IllegalStateTranferException("there is no valid state transfer from:" + oldStatus + " to:" - + newStatus + ", job id: " + jobId); + throw new IllegalStateTranferException("there is no valid state transfer from:" + oldStatus + " to:" + newStatus + ", job id: " + jobId); } jobOutput.setStatus(newStatus.toString()); } @@ -484,8 +482,7 @@ public class ExecutableManager { } } - private AbstractExecutable parseToAbstract(ExecutablePO executablePO, - Class<? extends AbstractExecutable> expectedClass) { + private AbstractExecutable parseToAbstract(ExecutablePO executablePO, Class<? extends AbstractExecutable> expectedClass) { if (executablePO == null) { logger.warn("executablePO is null"); return null; http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java index 0d3258e..910bd7e 100644 --- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java +++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableState.java @@ -36,14 +36,12 @@ public enum ExecutableState { private static Multimap<ExecutableState, ExecutableState> VALID_STATE_TRANSFER; static { - VALID_STATE_TRANSFER = Multimaps.newSetMultimap( - Maps.<ExecutableState, Collection<ExecutableState>> newEnumMap(ExecutableState.class), - new Supplier<Set<ExecutableState>>() { - @Override - public Set<ExecutableState> get() { - return new CopyOnWriteArraySet<ExecutableState>(); - } - }); + VALID_STATE_TRANSFER = Multimaps.newSetMultimap(Maps.<ExecutableState, Collection<ExecutableState>> newEnumMap(ExecutableState.class), new Supplier<Set<ExecutableState>>() { + @Override + public Set<ExecutableState> get() { + return new CopyOnWriteArraySet<ExecutableState>(); + } + }); //scheduler VALID_STATE_TRANSFER.put(ExecutableState.READY, ExecutableState.RUNNING); @@ -66,9 +64,11 @@ public enum ExecutableState { VALID_STATE_TRANSFER.put(ExecutableState.ERROR, ExecutableState.DISCARDED); VALID_STATE_TRANSFER.put(ExecutableState.ERROR, ExecutableState.READY); + VALID_STATE_TRANSFER.put(ExecutableState.READY, ExecutableState.STOPPED); VALID_STATE_TRANSFER.put(ExecutableState.RUNNING, ExecutableState.STOPPED); + //rollback VALID_STATE_TRANSFER.put(ExecutableState.SUCCEED, ExecutableState.READY); http://git-wip-us.apache.org/repos/asf/kylin/blob/19585846/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java index 128fa35..8b6b5aa 100644 --- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java +++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java @@ -121,9 +121,7 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti logger.warn(jobDesc + " fail to schedule", ex); } } - logger.info("Job Fetcher: " + nRunning + " should running, " + runningJobs.size() + " actual running, " - + nStopped + " stopped, " + nReady + " ready, " + nSUCCEED + " already succeed, " + nError - + " error, " + nDiscarded + " discarded, " + nOthers + " others"); + logger.info("Job Fetcher: " + nRunning + " should running, " + runningJobs.size() + " actual running, " + nStopped + " stopped, " + nReady + " ready, " + nSUCCEED + " already succeed, " + nError + " error, " + nDiscarded + " discarded, " + nOthers + " others"); } catch (Exception e) { logger.warn("Job Fetcher caught a exception " + e); } @@ -191,7 +189,7 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti @Override public synchronized void init(JobEngineConfig jobEngineConfig, JobLock lock) throws SchedulerException { jobLock = lock; - + String serverMode = jobEngineConfig.getConfig().getServerMode(); if (!("job".equals(serverMode.toLowerCase()) || "all".equals(serverMode.toLowerCase()))) { logger.info("server mode: " + serverMode + ", no need to run job scheduler"); @@ -215,15 +213,13 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti //load all executable, set them to a consistent status fetcherPool = Executors.newScheduledThreadPool(1); int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit(); - jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS, - new SynchronousQueue<Runnable>()); + jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS, new SynchronousQueue<Runnable>()); context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig()); executableManager.resumeAllRunningJobs(); fetcher = new FetcherRunner(); - fetcherPool.scheduleAtFixedRate(fetcher, 10, ExecutableConstants.DEFAULT_SCHEDULER_INTERVAL_SECONDS, - TimeUnit.SECONDS); + fetcherPool.scheduleAtFixedRate(fetcher, 10, ExecutableConstants.DEFAULT_SCHEDULER_INTERVAL_SECONDS, TimeUnit.SECONDS); hasStarted = true; }
