Repository: kylin Updated Branches: refs/heads/fix-sonar-1 [created] c8d95251b
fix-1 Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b32435c2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b32435c2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b32435c2 Branch: refs/heads/fix-sonar-1 Commit: b32435c26fa43a2be3c1448c0a87a1f59a78feaf Parents: 32d7244 Author: Billy Liu <billy...@apache.org> Authored: Fri Dec 30 15:07:01 2016 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Fri Dec 30 15:07:01 2016 +0800 ---------------------------------------------------------------------- .../adapter/enumerable/EnumerableWindow.java | 48 -------------------- .../apache/kylin/common/KylinConfigBase.java | 11 +++-- .../apache/kylin/common/util/ThreadUtil.java | 47 ------------------- .../org/apache/kylin/common/util/TimeUtil.java | 2 +- .../apache/kylin/common/util/ZipFileUtils.java | 5 +- .../kylin/cube/cli/DumpDictionaryCLI.java | 6 ++- .../kylin/cube/model/AggregationGroup.java | 3 +- .../org/apache/kylin/cube/model/CubeDesc.java | 3 +- .../apache/kylin/cube/model/DictionaryDesc.java | 4 +- .../apache/kylin/cube/model/DimensionDesc.java | 3 +- .../kylin/cube/model/HBaseMappingDesc.java | 3 +- .../org/apache/kylin/cube/model/RowKeyDesc.java | 3 +- .../kylin/cube/model/v1_4_0/CubeDesc.java | 3 +- .../kylin/gridtable/GTAggregateScanner.java | 4 +- .../org/apache/kylin/dict/CachedTreeMap.java | 16 +++---- .../org/apache/kylin/dict/TrieDictionary.java | 2 +- .../apache/kylin/dict/TrieDictionaryForest.java | 2 +- .../apache/kylin/dimension/BooleanDimEnc.java | 2 +- .../kylin/measure/MeasureTypeFactory.java | 3 +- .../kylin/metadata/model/FunctionDesc.java | 8 ++-- .../kylin/metadata/model/JoinTableDesc.java | 4 +- .../kylin/metadata/model/MeasureDesc.java | 3 +- .../metadata/model/ModelDimensionDesc.java | 9 ++-- .../kylin/metadata/model/PartitionDesc.java | 4 +- .../apache/kylin/metadata/model/TableDesc.java | 2 +- .../kylin/metadata/model/TableExtDesc.java | 3 +- .../apache/kylin/metadata/model/TableRef.java | 3 +- .../metadata/project/RealizationEntry.java | 4 +- .../org/apache/kylin/metadata/tuple/Tuple.java | 2 +- .../apache/kylin/metadata/tuple/TupleInfo.java | 10 ++-- .../org/apache/kylin/source/ReadableTable.java | 3 +- .../kylin/source/datagen/ColumnGenerator.java | 2 +- .../kylin/engine/mr/DFSFileTableReader.java | 4 +- .../engine/mr/steps/SelfDefineSortableKey.java | 2 +- .../apache/kylin/engine/spark/SparkCubing.java | 4 ++ server-base/pom.xml | 7 --- .../kylin/rest/request/PrepareSqlRequest.java | 3 +- .../apache/kylin/rest/service/CubeService.java | 4 ++ .../kylin/rest/service/DiagnosisService.java | 6 ++- .../cardinality/ColumnCardinalityMapper.java | 3 +- source-kafka/pom.xml | 1 - .../kylin/source/kafka/config/BrokerConfig.java | 4 +- .../common/coprocessor/AggregationCache.java | 4 +- .../hbase/util/ZookeeperDistributedJobLock.java | 2 + .../org/apache/kylin/tool/CubeMetaIngester.java | 3 +- .../apache/kylin/tool/HBaseUsageExtractor.java | 2 +- 46 files changed, 111 insertions(+), 165 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java ---------------------------------------------------------------------- diff --git a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java index 203ce02..547210c 100644 --- a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java +++ b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java @@ -113,54 +113,6 @@ public class EnumerableWindow extends Window implements EnumerableRel { } } - private void sampleOfTheGeneratedWindowedAggregate() { - // Here's overview of the generated code - // For each list of rows that have the same partitioning key, evaluate - // all of the windowed aggregate functions. - - // builder - Iterator<Integer[]> iterator = null; - - // builder3 - Integer[] rows = iterator.next(); - - int prevStart = -1; - int prevEnd = -1; - - for (int i = 0; i < rows.length; i++) { - // builder4 - Integer row = rows[i]; - - int start = 0; - int end = 100; - if (start != prevStart || end != prevEnd) { - // builder5 - int actualStart = 0; - if (start != prevStart || end < prevEnd) { - // builder6 - // recompute - actualStart = start; - // implementReset - } else { // must be start == prevStart && end > prevEnd - actualStart = prevEnd + 1; - } - prevStart = start; - prevEnd = end; - - if (start != -1) { - for (int j = actualStart; j <= end; j++) { - // builder7 - // implementAdd - } - } - // implementResult - // list.add(new Xxx(row.deptno, row.empid, sum, count)); - } - } - // multiMap.clear(); // allows gc - // source = Linq4j.asEnumerable(list); - } - public Result implement(EnumerableRelImplementor implementor, Prefer pref) { final JavaTypeFactory typeFactory = implementor.getTypeFactory(); final EnumerableRel child = (EnumerableRel) getInput(); http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index d73b694..d482c80 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -533,10 +533,13 @@ abstract public class KylinConfigBase implements Serializable { File home = new File(homePath); SortedSet<String> files = Sets.newTreeSet(); if (home.exists() && home.isDirectory()) { - for (File file : home.listFiles()) { - final Matcher matcher = pattern.matcher(file.getName()); - if (matcher.matches()) { - files.add(file.getAbsolutePath()); + File[] listFiles = home.listFiles(); + if (listFiles != null) { + for (File file : listFiles) { + final Matcher matcher = pattern.matcher(file.getName()); + if (matcher.matches()) { + files.add(file.getAbsolutePath()); + } } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java deleted file mode 100644 index aebbc57..0000000 --- a/core-common/src/main/java/org/apache/kylin/common/util/ThreadUtil.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common.util; - -import java.util.concurrent.Future; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -/** - */ -public class ThreadUtil { - @SuppressWarnings("unused") - public static void main(String[] args) { - ThreadPoolExecutor pool = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>());//Threads.newDaemonThreadFactory("htable")); - - for (int i = 0; i < Integer.MAX_VALUE; ++i) { - System.out.println("index: " + i); - Future<?> future = pool.submit(new Runnable() { - @Override - public void run() { - try { - Thread.sleep(10000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java index d3d4283..7a24c10 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/TimeUtil.java @@ -26,7 +26,7 @@ import java.util.TimeZone; public class TimeUtil { private static TimeZone gmt = TimeZone.getTimeZone("GMT"); - private static long ONE_MINUTE_TS = 60 * 1000; + private static long ONE_MINUTE_TS = 60 * 1000L; private static long ONE_HOUR_TS = 60 * ONE_MINUTE_TS; private static long ONE_DAY_TS = 24 * ONE_HOUR_TS; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java b/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java index 7ee770d..a2326ee 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java @@ -77,7 +77,10 @@ public class ZipFileUtils { } private static void compressDirectoryToZipfile(String rootDir, String sourceDir, ZipOutputStream out) throws IOException { - for (File sourceFile : new File(sourceDir).listFiles()) { + File[] files = new File(sourceDir).listFiles(); + if (files == null) + return; + for (File sourceFile : files) { if (sourceFile.isDirectory()) { compressDirectoryToZipfile(rootDir, sourceDir + normDir(sourceFile.getName()), out); } else { http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java index 24daddf..729a6da 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java @@ -38,7 +38,11 @@ public class DumpDictionaryCLI { public static void dump(File f) throws IOException { if (f.isDirectory()) { - for (File c : f.listFiles()) + File[] files = f.listFiles(); + if (files == null) { + return; + } + for (File c : files) dump(c); return; } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java index 9bd082f..1d4d496 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java @@ -18,6 +18,7 @@ package org.apache.kylin.cube.model; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -35,7 +36,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class AggregationGroup { +public class AggregationGroup implements Serializable{ public static class HierarchyMask { public long fullMask; // 00000111 public long[] allMasks; // 00000100,00000110,00000111 http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 4686a15..556cc72 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -825,7 +825,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols); List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray); if (infoList == null) { - hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>()); + infoList = new ArrayList<DeriveInfo>(); + hostToDerivedMap.put(hostColArray, infoList); } // Merged duplicated derived column http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java index f471f9f..fb7053e 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java @@ -26,8 +26,10 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.Serializable; + @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class DictionaryDesc { +public class DictionaryDesc implements Serializable{ @JsonProperty("column") private String column; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java index cd75228..29fd6c3 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java @@ -18,6 +18,7 @@ package org.apache.kylin.cube.model; +import java.io.Serializable; import java.util.Arrays; import org.apache.commons.lang.NotImplementedException; @@ -35,7 +36,7 @@ import com.google.common.base.Objects; /** */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class DimensionDesc { +public class DimensionDesc implements Serializable { @JsonProperty("name") private String name; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java index 2ef1e17..84d87de 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java @@ -18,6 +18,7 @@ package org.apache.kylin.cube.model; +import java.io.Serializable; import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; @@ -33,7 +34,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; /** */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class HBaseMappingDesc { +public class HBaseMappingDesc implements Serializable{ @JsonProperty("column_family") private HBaseColumnFamilyDesc[] columnFamily; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java index f1a403d..1bd06d2 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java @@ -18,6 +18,7 @@ package org.apache.kylin.cube.model; +import java.io.Serializable; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -35,7 +36,7 @@ import com.google.common.base.Objects; /** */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class RowKeyDesc { +public class RowKeyDesc implements Serializable{ @JsonProperty("rowkey_columns") private RowKeyColDesc[] rowkeyColumns; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java index bd73bc8..4019e4f 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java @@ -630,7 +630,8 @@ public class CubeDesc extends RootPersistentEntity { Map<String, TblColRef> cols = columnMap.get(ref.getTable()); if (cols == null) { - columnMap.put(ref.getTable(), cols = new HashMap<String, TblColRef>()); + cols = new HashMap<String, TblColRef>(); + columnMap.put(ref.getTable(), cols); } cols.put(ref.getName(), ref); return ref; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java index 9bbcd10..55c04c6 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java @@ -99,7 +99,7 @@ public class GTAggregateScanner implements IGTScanner { public static long estimateSizeOf(MeasureAggregator[] aggrs) { // size of array, AggregationCacheMemSizeTest reports 4 for [0], 12 for [1], 12 for [2], 20 for [3] etc.. // Memory alignment to 8 bytes - long est = (aggrs.length + 1) / 2 * 8 + 4 + (4 /* extra */); + long est = (aggrs.length + 1) / 2 * 8L + 4 + (4 /* extra */); for (MeasureAggregator aggr : aggrs) { if (aggr != null) est += aggr.getMemBytesEstimate(); @@ -110,7 +110,7 @@ public class GTAggregateScanner implements IGTScanner { public static long estimateSizeOf(byte[] bytes) { // AggregationCacheMemSizeTest reports 20 for byte[10] and 20 again for byte[16] // Memory alignment to 8 bytes - return (bytes.length + 7) / 8 * 8 + 4 + (4 /* extra */); + return (bytes.length + 7) / 8 * 8L + 4 + (4 /* extra */); } public void trackMemoryLevel(MemoryWaterLevel tracker) { http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java index 6acf764..3be4fd7 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java @@ -269,7 +269,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext } public void commit(boolean keepAppend) throws IOException { - assert this.keepAppend & !immutable : "Only support commit method with immutable false and keepAppend true"; + assert this.keepAppend && !immutable : "Only support commit method with immutable false and keepAppend true"; Path newVersionDir = new Path(baseDir, VERSION_PREFIX + System.currentTimeMillis()); if (keepAppend) { @@ -312,7 +312,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext } String fileName = generateFileName(key); Path filePath = new Path(fileName); - try (FSDataOutputStream out = fs.create(filePath, true, BUFFER_SIZE, (short) 5, BUFFER_SIZE * 8)) { + try (FSDataOutputStream out = fs.create(filePath, true, BUFFER_SIZE, (short) 5, BUFFER_SIZE * 8L)) { value.write(out); } catch (Exception e) { logger.error(String.format("write value into %s exception: %s", fileName, e), e); @@ -350,7 +350,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext @Override public V put(K key, V value) { - assert keepAppend & !immutable : "Only support put method with immutable false and keepAppend true"; + assert keepAppend && !immutable : "Only support put method with immutable false and keepAppend true"; super.put(key, null); valueCache.put(key, value); return null; @@ -362,7 +362,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext try { return valueCache.get((K) key); } catch (ExecutionException e) { - logger.error(String.format("get value with key %s exception: ", key, e), e); + logger.error(String.format("get value with key %s exception: %s", key, e), e); return null; } } else { @@ -372,7 +372,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext @Override public V remove(Object key) { - assert keepAppend & !immutable : "Only support remove method with immutable false keepAppend true"; + assert keepAppend && !immutable : "Only support remove method with immutable false keepAppend true"; super.remove(key); valueCache.invalidate(key); return null; @@ -421,21 +421,21 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext try { return (V) valueCache.get(currentKey); } catch (ExecutionException e) { - logger.error(String.format("get value with key %s exception: ", currentKey, e), e); + logger.error(String.format("get value with key %s exception: %s", currentKey, e), e); return null; } } @Override public void remove() { - assert keepAppend & !immutable : "Only support remove method with immutable false and keepAppend true"; + assert keepAppend && !immutable : "Only support remove method with immutable false and keepAppend true"; keyIterator.remove(); valueCache.invalidate(currentKey); } } public FSDataOutputStream openIndexOutput() throws IOException { - assert keepAppend & !immutable : "Only support write method with immutable false and keepAppend true"; + assert keepAppend && !immutable : "Only support write method with immutable false and keepAppend true"; Path indexPath = new Path(getCurrentDir(), ".index"); return fs.create(indexPath, true, 8 * 1024 * 1024, (short) 5, 8 * 1024 * 1024 * 8); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java index c099de0..d5dbf7e 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java @@ -117,7 +117,7 @@ public class TrieDictionary<T> extends Dictionary<T> { this.bytesConvert = ClassUtil.forName(converterName, BytesConverter.class).newInstance(); this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath); - this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case + this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1L); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case this.childOffsetMask = ~((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8)); this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java index e746348..a15399b 100755 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java @@ -90,7 +90,7 @@ public class TrieDictionaryForest<T> extends Dictionary<T> { return 1; int maxOffset = accuOffset.get(accuOffset.size() - 1); TrieDictionary<T> lastTree = trees.get(trees.size() - 1); - int sizeOfId = BytesUtil.sizeForValue(baseId + maxOffset + lastTree.getMaxId() + 1); + int sizeOfId = BytesUtil.sizeForValue(baseId + maxOffset + lastTree.getMaxId() + 1L); return sizeOfId; } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java index c3f4c11..08fbe51 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java @@ -63,7 +63,7 @@ public class BooleanDimEnc extends DimensionEncoding { // ============================================================================ - private static int fixedLen = 1; + private int fixedLen = 1; //no-arg constructor is required for Externalizable public BooleanDimEnc() { http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java index 694459b..8e58858 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java @@ -142,7 +142,8 @@ abstract public class MeasureTypeFactory<T> { registerUDAF(factory); List<MeasureTypeFactory<?>> list = factories.get(funcName); if (list == null) - factories.put(funcName, list = Lists.newArrayListWithCapacity(2)); + list = Lists.newArrayListWithCapacity(2); + factories.put(funcName, list); list.add(factory); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index 4d89e1a..184c290 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -95,7 +95,8 @@ public class FunctionDesc { } } - parameter.setColRefs(colRefs); + if(parameter != null) + parameter.setColRefs(colRefs); } private void reInitMeasureType() { @@ -265,10 +266,7 @@ public class FunctionDesc { if (other.parameter != null) return false; } else { - if (parameter == null) { - if (other.parameter != null) - return false; - } else if (!parameter.equals(other.parameter)) + if (!parameter.equals(other.parameter)) return false; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java index 5d0409a..513217e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinTableDesc.java @@ -25,8 +25,10 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.Serializable; + @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class JoinTableDesc { +public class JoinTableDesc implements Serializable{ @JsonProperty("table") private String table; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java index 253b06b..6ddbbf4 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java @@ -18,6 +18,7 @@ package org.apache.kylin.metadata.model; +import java.io.Serializable; import java.util.Objects; import com.fasterxml.jackson.annotation.JsonAutoDetect; @@ -29,7 +30,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class MeasureDesc { +public class MeasureDesc implements Serializable{ @JsonProperty("name") private String name; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java index 6460f71..bd3dfef 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java @@ -18,6 +18,7 @@ package org.apache.kylin.metadata.model; +import java.io.Serializable; import java.util.List; import org.apache.kylin.common.util.StringUtil; @@ -28,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; /** */ @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) -public class ModelDimensionDesc { +public class ModelDimensionDesc implements Serializable{ @JsonProperty("table") private String table; @JsonProperty("columns") @@ -58,8 +59,10 @@ public class ModelDimensionDesc { if (model != null) { table = model.findTable(table).getAlias(); - for (int i = 0; i < columns.length; i++) { - columns[i] = model.findColumn(table, columns[i]).getName(); + if (columns != null) { + for (int i = 0; i < columns.length; i++) { + columns[i] = model.findColumn(table, columns[i]).getName(); + } } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java index 9925990..bab3ec3 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java @@ -27,10 +27,12 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.Serializable; + /** */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class PartitionDesc { +public class PartitionDesc implements Serializable{ public static enum PartitionType { APPEND, // http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java index e845da1..e4fc4c3 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java @@ -130,7 +130,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { this.name = splits[0]; } } else { - this.name = name; + this.name = null; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java index 6393dbf..8ee484b 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableExtDesc.java @@ -18,6 +18,7 @@ package org.apache.kylin.metadata.model; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -231,7 +232,7 @@ public class TableExtDesc extends RootPersistentEntity { } @JsonIgnoreProperties(ignoreUnknown = true) - public static class ColumnStats implements Comparable<ColumnStats> { + public static class ColumnStats implements Comparable<ColumnStats>, Serializable { @JsonBackReference private TableExtDesc tableExtDesc; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java index 7089eba..9837ff4 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableRef.java @@ -18,13 +18,14 @@ package org.apache.kylin.metadata.model; +import java.io.Serializable; import java.util.Collection; import java.util.Collections; import java.util.Map; import com.google.common.collect.Maps; -public class TableRef { +public class TableRef implements Serializable{ final private DataModelDesc model; final private String alias; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java index 84e5fd5..421f802 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java @@ -23,10 +23,12 @@ import org.apache.kylin.metadata.realization.RealizationType; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.Serializable; + /** */ @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) -public class RealizationEntry { +public class RealizationEntry implements Serializable{ @JsonProperty("type") private RealizationType type; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java index d51ae9e..b50a7e4 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java @@ -213,7 +213,7 @@ public class Tuple implements ITuple { case "float": return Float.valueOf(strValue); case "boolean": - return Boolean.valueOf(strValue) | "1".equals(strValue); // in some extended encodings boolean might be encoded as a number + return Boolean.valueOf(strValue) || "1".equals(strValue); // in some extended encodings boolean might be encoded as a number default: return strValue; } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java index 8970124..c3e88b5 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/TupleInfo.java @@ -93,10 +93,12 @@ public class TupleInfo { else columns.add(index, col); - if (dataTypeNames.size() > index) - dataTypeNames.set(index, col.getColumnDesc().getUpgradedType().getName()); - else - dataTypeNames.add(index, col.getColumnDesc().getUpgradedType().getName()); + if (col != null) { + if (dataTypeNames.size() > index) + dataTypeNames.set(index, col.getColumnDesc().getUpgradedType().getName()); + else + dataTypeNames.add(index, col.getColumnDesc().getUpgradedType().getName()); + } } public List<String> getAllFields() { http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java b/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java index 8ea96de..c62f1fe 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/ReadableTable.java @@ -20,6 +20,7 @@ package org.apache.kylin.source; import java.io.Closeable; import java.io.IOException; +import java.io.Serializable; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; @@ -59,7 +60,7 @@ public interface ReadableTable { // ============================================================================ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) - public class TableSignature { + public class TableSignature implements Serializable{ @JsonProperty("path") private String path; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java index 7c54714..f086ae9 100644 --- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java +++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java @@ -195,7 +195,7 @@ public class ColumnGenerator { private long randomMillis() { int secondsInYear = 3600 * 24 * 365; - long year = randStart + rand.nextInt(randEnd - randStart) - 1970; + long year = randStart + rand.nextInt(randEnd - randStart) - 1970L; long second = year * secondsInYear + rand.nextInt(secondsInYear); return second * 1000; } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java index 67fedbd..847f4bf 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTableReader.java @@ -88,7 +88,7 @@ public class DFSFileTableReader implements TableReader { try { for (FileStatus f : allFiles) { - RowReader rowReader = new SeqRowReader(HadoopUtil.getCurrentConfiguration(), fs, f.getPath().toString()); + RowReader rowReader = new SeqRowReader(HadoopUtil.getCurrentConfiguration(), f.getPath().toString()); this.readerList.add(rowReader); } } catch (IOException e) { @@ -208,7 +208,7 @@ public class DFSFileTableReader implements TableReader { Writable key; Text value; - SeqRowReader(Configuration hconf, FileSystem fs, String path) throws IOException { + SeqRowReader(Configuration hconf, String path) throws IOException { reader = new Reader(hconf, SequenceFile.Reader.file(new Path(path))); key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf); value = new Text(); http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java index b804eef..74525d3 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java @@ -60,7 +60,7 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta byte[] data2 = o.text.getBytes(); String str1 = new String(data1, 1, data1.length - 1); String str2 = new String(data2, 1, data2.length - 1); - if (str1 == null || str1.equals("") || str2 == null || str2.equals("")) { + if (str1.equals("") || str2.equals("")) { //should not achieve here logger.error("none numeric value!"); return 0; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java index 6e894dd..4ec5048 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java @@ -167,6 +167,9 @@ public class SparkCubing extends AbstractApplication { return false; } }); + if (files == null) { + return; + } for (File file : files) { sc.addFile(file.getAbsolutePath()); } @@ -341,6 +344,7 @@ public class SparkCubing extends AbstractApplication { Dictionary<String> dict = cubeSegment.getDictionary(col); if (dict == null) { System.err.println("Dictionary for " + col + " was not found."); + continue; } dictionaryMap.put(col, dict); System.out.println("col:" + col + " dictionary size:" + dict.getSize()); http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/server-base/pom.xml ---------------------------------------------------------------------- diff --git a/server-base/pom.xml b/server-base/pom.xml index e75e683..1147008 100644 --- a/server-base/pom.xml +++ b/server-base/pom.xml @@ -126,13 +126,6 @@ <artifactId>aspectjweaver</artifactId> </dependency> - <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka_2.10</artifactId> - <version>${kafka.version}</version> - <scope>provided</scope> - </dependency> - <!-- Test & Env --> <dependency> <groupId>org.apache.kylin</groupId> http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/server-base/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java index 95bc2f5..e296248 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java +++ b/server-base/src/main/java/org/apache/kylin/rest/request/PrepareSqlRequest.java @@ -18,6 +18,7 @@ package org.apache.kylin.rest.request; +import java.io.Serializable; import java.util.Arrays; /** @@ -40,7 +41,7 @@ public class PrepareSqlRequest extends SQLRequest { this.params = params; } - public static class StateParam { + public static class StateParam implements Serializable{ private String className; private String value; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index 23aa5a4..d28c87c 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -455,6 +455,10 @@ public class CubeService extends BasicService { } } + if(toDelete == null){ + throw new IllegalArgumentException("Cannot find segment '" + segmentName +"'"); + } + if (toDelete.getStatus() != SegmentStatusEnum.READY) { throw new IllegalArgumentException("Cannot delete segment '" + segmentName + "' as its status is not READY. Discard the on-going job for it."); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java b/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java index dee5531..8208596 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java @@ -45,7 +45,11 @@ public class DiagnosisService extends BasicService { } private String getDiagnosisPackageName(File destDir) { - for (File subDir : destDir.listFiles()) { + File[] files = destDir.listFiles(); + if (files == null) { + throw new RuntimeException("Diagnosis package is not available in directory: " + destDir.getAbsolutePath()); + } + for (File subDir : files) { if (subDir.isDirectory()) { for (File file : subDir.listFiles()) { if (file.getName().endsWith(".zip")) { http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java ---------------------------------------------------------------------- diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java index f046f78..145ed31 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java @@ -80,8 +80,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritab System.out.println("Get row " + counter + " column '" + field + "' value: " + fieldValue); } - if (fieldValue != null) - getHllc(m).add(Bytes.toBytes(fieldValue.toString())); + getHllc(m).add(Bytes.toBytes(fieldValue.toString())); } counter++; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/source-kafka/pom.xml ---------------------------------------------------------------------- diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml index d59c257..e43894d 100644 --- a/source-kafka/pom.xml +++ b/source-kafka/pom.xml @@ -51,7 +51,6 @@ <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> - <version>${commons-lang3.version}</version> </dependency> <!-- Env & Test --> <dependency> http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/BrokerConfig.java ---------------------------------------------------------------------- diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/BrokerConfig.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/BrokerConfig.java index add825b..fc3bba0 100644 --- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/BrokerConfig.java +++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/BrokerConfig.java @@ -21,10 +21,12 @@ package org.apache.kylin.source.kafka.config; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.Serializable; + /** */ @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE) -public class BrokerConfig { +public class BrokerConfig implements Serializable{ @JsonProperty("id") private int id; http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java index 06cdbba..2a85894 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/AggregationCache.java @@ -29,7 +29,7 @@ import com.google.common.collect.Maps; */ @SuppressWarnings("rawtypes") public abstract class AggregationCache { - static final long MEMORY_USAGE_CAP = 500 * 1024 * 1024; // 500 MB + static final long MEMORY_USAGE_CAP = 500 * 1024 * 1024L; // 500 MB static final long MEMOERY_MAX_BYTES = Runtime.getRuntime().maxMemory(); protected final Map<AggrKey, MeasureAggregator[]> aggBufMap; transient int rowMemBytes; @@ -75,7 +75,7 @@ public abstract class AggregationCache { } } int size = aggBufMap.size(); - long memUsage = (40 + rowMemBytes) * size; + long memUsage = (40L + rowMemBytes) * size; if (memUsage > MEMORY_USAGE_CAP) { throw new RuntimeException("Kylin coprocessor memory usage goes beyond cap, (40 + " + rowMemBytes + ") * " + size + " > " + MEMORY_USAGE_CAP + ". Abort coprocessor."); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java index 613d783..5a44cc1 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java @@ -141,6 +141,8 @@ public class ZookeeperDistributedJobLock implements DistributedJobLock { } catch (Exception e) { logger.error("fail to get the serverName for the path: " + lockPath, e); } + if(lockServerName == null) + return false; return lockServerName.equalsIgnoreCase(serverName); } http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java index 40cbf32..b4c44c3 100644 --- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java +++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java @@ -20,6 +20,7 @@ package org.apache.kylin.tool; import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.Set; import org.apache.commons.cli.Option; @@ -126,7 +127,7 @@ public class CubeMetaIngester extends AbstractApplication { tempFolder.mkdir(); ZipFileUtils.decompressZipfileToDirectory(srcPath, tempFolder); if (tempFolder.list().length != 1) { - throw new IllegalStateException(tempFolder.list().toString()); + throw new IllegalStateException(Arrays.toString(tempFolder.list())); } injest(tempFolder.listFiles()[0].getAbsoluteFile()); http://git-wip-us.apache.org/repos/asf/kylin/blob/b32435c2/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java index f993155..0d8c08f 100644 --- a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java @@ -51,7 +51,7 @@ import com.google.common.collect.Lists; public class HBaseUsageExtractor extends AbstractInfoExtractor { - private static final Logger logger = LoggerFactory.getLogger(CubeMetaExtractor.class); + private static final Logger logger = LoggerFactory.getLogger(HBaseUsageExtractor.class); @SuppressWarnings("static-access") private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("Specify which cube to extract").create("cube"); @SuppressWarnings("static-access")
