KYLIN-2428 cleanup unnecessary shade dependency in job/coprocessor/spark/jdbc/tool
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f10d9d9b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f10d9d9b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f10d9d9b Branch: refs/heads/KYLIN-2428 Commit: f10d9d9ba95878a97f70975547763083c000bcfe Parents: c93ea9c Author: Billy Liu <billy...@apache.org> Authored: Tue Feb 14 13:29:25 2017 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Tue Feb 14 13:29:25 2017 +0800 ---------------------------------------------------------------------- assembly/pom.xml | 49 +++- atopcalcite/pom.xml | 1 + build/bin/get-properties.sh | 3 +- build/bin/kylin.sh | 8 +- build/conf/kylin-tools-log4j.properties | 9 +- build/script/prepare_libs.sh | 2 +- core-common/pom.xml | 22 +- .../apache/kylin/common/util/SparkEntry.java | 42 --- core-cube/pom.xml | 18 +- .../org/apache/kylin/cube/model/CubeDesc.java | 2 +- .../apache/kylin/gridtable/GTInvertedIndex.java | 223 ---------------- .../gridtable/GTInvertedIndexOfColumn.java | 133 ---------- .../gridtable/SimpleInvertedIndexTest.java | 196 -------------- core-job/pom.xml | 9 + core-metadata/pom.xml | 23 ++ .../apache/kylin/dimension/BooleanDimEnc.java | 5 +- .../org/apache/kylin/dimension/DateDimEnc.java | 4 +- .../kylin/dimension/DictionaryDimEnc.java | 3 +- .../apache/kylin/dimension/FixedLenDimEnc.java | 3 +- .../kylin/dimension/FixedLenHexDimEnc.java | 3 +- .../org/apache/kylin/dimension/IntDimEnc.java | 3 +- .../apache/kylin/dimension/IntegerDimEnc.java | 3 +- .../kylin/dimension/OneMoreByteVLongDimEnc.java | 3 +- .../org/apache/kylin/dimension/TimeDimEnc.java | 4 +- .../kylin/measure/MeasureTypeFactory.java | 2 +- .../measure/hllc/HLLDistinctCountAggFunc.java | 2 +- .../org/apache/kylin/measure/topn/Counter.java | 3 +- .../kylin/metadata/model/DataModelDesc.java | 5 +- .../apache/kylin/metadata/model/JoinsTree.java | 2 +- .../kylin/metadata/model/PartitionDesc.java | 2 +- .../kylin/metadata/model/SegmentStatusEnum.java | 4 +- .../apache/kylin/metadata/model/Segments.java | 3 +- .../realization/RealizationStatusEnum.java | 4 +- core-storage/pom.xml | 20 ++ engine-spark/pom.xml | 47 +--- .../engine/spark/KylinKryoRegistrator.java | 265 ++++++++++++++----- .../spark/SparkBatchCubingJobBuilder2.java | 1 + .../kylin/engine/spark/SparkCubingByLayer.java | 6 +- .../kylin/engine/spark/SparkExecutable.java | 17 +- .../spark/cube/ListBackedCuboidWriter.java | 4 +- .../kylin/engine/spark/util/IteratorUtils.java | 4 +- .../kylin/engine/spark/util/SparkEntry.java | 48 ++++ .../engine/spark/util/KyroMappingGenerator.java | 56 ++++ jdbc/pom.xml | 14 +- pom.xml | 216 ++++++++------- query/pom.xml | 6 + server/pom.xml | 4 + source-kafka/pom.xml | 16 +- storage-hbase/pom.xml | 19 +- tool-assembly/pom.xml | 135 ++++++++++ tool/pom.xml | 39 --- 51 files changed, 791 insertions(+), 924 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/assembly/pom.xml ---------------------------------------------------------------------- diff --git a/assembly/pom.xml b/assembly/pom.xml index 873abff..ba69583 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -19,16 +19,20 @@ <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <artifactId>kylin-assembly</artifactId> + <name>Apache Kylin - Assembly</name> + <parent> <artifactId>kylin</artifactId> <groupId>org.apache.kylin</groupId> <version>2.0.0-SNAPSHOT</version> - </parent> - <modelVersion>4.0.0</modelVersion> - <artifactId>kylin-assembly</artifactId> - <name>Apache Kylin - Assembly</name> + <properties> + <shadeBase>org.apache.kylin.job.shaded</shadeBase> + </properties> <dependencies> <dependency> @@ -180,8 +184,45 @@ <exclude>org.apache.zookeeper:*</exclude> <exclude>net.sf.ehcache:*</exclude> <exclude>org.apache.httpcomponents:*</exclude> + <exclude>org.apache.commons:commons-email:*</exclude> + <exclude>com.sun.mail:javax.mail:*</exclude> + <exclude>javax.activation:activation:*</exclude> </excludes> </artifactSet> + <relocations> + <relocation> + <pattern>com.fasterxml.jackson</pattern> + <shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern> + </relocation> + <relocation> + <pattern>com.tdunning</pattern> + <shadedPattern>${shadeBase}.com.tdunning</shadedPattern> + </relocation> + <relocation> + <pattern>com.fasterxml.jackson</pattern> + <shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.commons.codec</pattern> + <shadedPattern>${shadeBase}.org.apache.commons.codec</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.commons.collections</pattern> + <shadedPattern>${shadeBase}.org.apache.commons.collections</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.commons.lang3</pattern> + <shadedPattern>${shadeBase}.org.apache.commons.lang3</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.commons.logging</pattern> + <shadedPattern>${shadeBase}.org.apache.commons.logging</shadedPattern> + </relocation> + <relocation> + <pattern>org.roaringbitmap</pattern> + <shadedPattern>${shadeBase}.org.roaringbitmap</shadedPattern> + </relocation> + </relocations> <filters> <filter> <artifact>*:*</artifact> http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/atopcalcite/pom.xml ---------------------------------------------------------------------- diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index 1b327fe..c65c0d1 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -43,6 +43,7 @@ </exclusion> </exclusions> </dependency> + <!-- It should be avatica(the shaded one), not avatica-core, since the inconsistency protobuf dependency with Hadoop --> <dependency> <groupId>org.apache.calcite.avatica</groupId> <artifactId>avatica</artifactId> http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/build/bin/get-properties.sh ---------------------------------------------------------------------- diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh index 170442d..57721d8 100755 --- a/build/bin/get-properties.sh +++ b/build/bin/get-properties.sh @@ -25,7 +25,6 @@ then exit -1 fi -job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar) tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar) -result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null` +result=`java -cp $tool_jar -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null` echo "$result" http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/build/bin/kylin.sh ---------------------------------------------------------------------- diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh index cc571e5..15e1aed 100644 --- a/build/bin/kylin.sh +++ b/build/bin/kylin.sh @@ -38,7 +38,7 @@ function retrieveDependency() { source ${dir}/setenv.sh fi - export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX} + export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX} export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency} if [ -n "$KAFKA_HOME" ] then @@ -103,7 +103,7 @@ then # KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh hbase ${KYLIN_EXTRA_START_OPTS} \ -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \ - -Dlog4j.configuration=kylin-server-log4j.properties \ + -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-server-log4j.properties \ -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \ -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \ -Djava.endorsed.dirs=${tomcat_root}/endorsed \ @@ -165,8 +165,10 @@ then then source ${dir}/setenv-tool.sh fi + hbase_original=${HBASE_CLASSPATH} + export HBASE_CLASSPATH=${hbase_original}:${KYLIN_HOME}/tool/* exec hbase ${KYLIN_EXTRA_START_OPTS} -Dkylin.hive.dependency=${hive_dependency} -Dkylin.hbase.dependency=${hbase_dependency} -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties "$@" - + export HBASE_CLASSPATH=${hbase_original} else quit "Usage: 'kylin.sh [-v] start' or 'kylin.sh [-v] stop'" fi http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/build/conf/kylin-tools-log4j.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin-tools-log4j.properties b/build/conf/kylin-tools-log4j.properties index 0ba7d2e..e975d18 100644 --- a/build/conf/kylin-tools-log4j.properties +++ b/build/conf/kylin-tools-log4j.properties @@ -25,11 +25,12 @@ # It's called kylin-tools-log4j.properties so that it won't distract users from the other more important log4j config file: kylin-server-log4j.properties # enable this by -Dlog4j.configuration=kylin-tools-log4j.properties -log4j.rootLogger=INFO,stdout +log4j.rootLogger=INFO,stderr -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: %m%n +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: %m%n #log4j.logger.org.apache.hadoop=ERROR log4j.logger.org.apache.kylin=DEBUG http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/build/script/prepare_libs.sh ---------------------------------------------------------------------- diff --git a/build/script/prepare_libs.sh b/build/script/prepare_libs.sh index aec9923..3dbdf21 100755 --- a/build/script/prepare_libs.sh +++ b/build/script/prepare_libs.sh @@ -33,7 +33,7 @@ mkdir build/lib build/tool cp assembly/target/kylin-assembly-${version}-job.jar build/lib/kylin-job-${version}.jar cp storage-hbase/target/kylin-storage-hbase-${version}-coprocessor.jar build/lib/kylin-coprocessor-${version}.jar cp jdbc/target/kylin-jdbc-${version}.jar build/lib/kylin-jdbc-${version}.jar -cp tool/target/kylin-tool-${version}-assembly.jar build/tool/kylin-tool-${version}.jar +cp tool-assembly/target/kylin-tool-assembly-${version}-assembly.jar build/tool/kylin-tool-${version}.jar # Copied file becomes 000 for some env (e.g. my Cygwin) chmod 644 build/lib/kylin-job-${version}.jar http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-common/pom.xml ---------------------------------------------------------------------- diff --git a/core-common/pom.xml b/core-common/pom.xml index 016d470..95d3c29 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -35,26 +35,10 @@ <dependencies> <!-- Basic Utilities --> <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>commons-cli</groupId> - <artifactId>commons-cli</artifactId> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-email</artifactId> </dependency> @@ -63,9 +47,11 @@ <artifactId>httpclient</artifactId> </dependency> <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> </dependency> + + <!-- Provided --> <dependency> <groupId>com.google.code.findbugs</groupId> <artifactId>jsr305</artifactId> http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java deleted file mode 100644 index fd324a2..0000000 --- a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kylin.common.util; - -import org.apache.commons.lang3.StringUtils; - -import com.google.common.base.Preconditions; - -/** - */ -public final class SparkEntry { - - public static void main(String[] args) throws Exception { - System.out.println("SparkEntry args:" + StringUtils.join(args, " ")); - Preconditions.checkArgument(args.length >= 2, "-className is required"); - Preconditions.checkArgument(args[0].equals("-className"), "-className is required"); - final String className = args[1]; - final Object o = Class.<AbstractApplication> forName(className).newInstance(); - Preconditions.checkArgument(o instanceof AbstractApplication, className + " is not a subClass of AbstractSparkApplication"); - String[] appArgs = new String[args.length - 2]; - for (int i = 2; i < args.length; i++) { - appArgs[i - 2] = args[i]; - } - AbstractApplication abstractApplication = (AbstractApplication) o; - abstractApplication.execute(appArgs); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-cube/pom.xml ---------------------------------------------------------------------- diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 0a30432..fe80783 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -48,9 +48,23 @@ <groupId>commons-collections</groupId> <artifactId>commons-collections</artifactId> </dependency> + + <!-- Provided --> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> <dependency> - <groupId>com.n3twork.druid</groupId> - <artifactId>extendedset</artifactId> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> </dependency> <dependency> <groupId>com.esotericsoftware</groupId> http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 9acb90c..0c1419d 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -97,7 +97,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { } } - public enum DeriveType { + public enum DeriveType implements java.io.Serializable{ LOOKUP, PK_FK, EXTENDED_COLUMN } http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java deleted file mode 100644 index a2c713a..0000000 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.gridtable; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.ImmutableBitSet; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; - -import it.uniroma3.mat.extendedset.intset.ConciseSet; - -/** - * A thread-safe inverted index of row blocks in memory. - * - * Note function not() must return all blocks, because index only know what block contains a value, - * but not sure what block does not contain a value. - * - * @author yangli9 - */ -public class GTInvertedIndex { - - private final GTInfo info; - private final ImmutableBitSet colPreferIndex; - private final ImmutableBitSet colBlocks; - private final GTInvertedIndexOfColumn[] index; // for each column - - private volatile int nIndexedBlocks; - - public GTInvertedIndex(GTInfo info) { - this.info = info; - this.colPreferIndex = info.colPreferIndex; - this.colBlocks = info.selectColumnBlocks(colPreferIndex); - - index = new GTInvertedIndexOfColumn[info.getColumnCount()]; - for (int i = 0; i < colPreferIndex.trueBitCount(); i++) { - int c = colPreferIndex.trueBitAt(i); - index[c] = new GTInvertedIndexOfColumn(info.codeSystem.getComparator()); - } - } - - public void add(GTRowBlock block) { - - @SuppressWarnings("unchecked") - Set<ByteArray>[] distinctValues = new Set[info.getColumnCount()]; - for (int i = 0; i < colPreferIndex.trueBitCount(); i++) { - int c = colPreferIndex.trueBitAt(i); - distinctValues[c] = new HashSet<ByteArray>(); - } - - GTRowBlock.Reader reader = block.getReader(colBlocks); - GTRecord record = new GTRecord(info); - while (reader.hasNext()) { - reader.fetchNext(record); - for (int i = 0; i < colPreferIndex.trueBitCount(); i++) { - int c = colPreferIndex.trueBitAt(i); - distinctValues[c].add(record.get(c)); - } - } - - for (int i = 0; i < colPreferIndex.trueBitCount(); i++) { - int c = colPreferIndex.trueBitAt(i); - index[c].add(distinctValues[c], block.getSequenceId()); - } - - nIndexedBlocks = Math.max(nIndexedBlocks, block.seqId + 1); - } - - public ConciseSet filter(TupleFilter filter) { - return filter(filter, nIndexedBlocks); - } - - public ConciseSet filter(TupleFilter filter, int totalBlocks) { - // number of indexed blocks may increase as we do evaluation - int indexedBlocks = nIndexedBlocks; - - Evaluator evaluator = new Evaluator(indexedBlocks); - ConciseSet r = evaluator.evaluate(filter); - - // add blocks that have not been indexed - for (int i = indexedBlocks; i < totalBlocks; i++) { - r.add(i); - } - - return r; - } - - private class Evaluator { - private int indexedBlocks; - - Evaluator(int indexedBlocks) { - this.indexedBlocks = indexedBlocks; - } - - public ConciseSet evaluate(TupleFilter filter) { - if (filter == null) { - return all(); - } - - if (filter instanceof LogicalTupleFilter) - return evalLogical((LogicalTupleFilter) filter); - - if (filter instanceof CompareTupleFilter) - return evalCompare((CompareTupleFilter) filter); - - // unable to evaluate - return all(); - } - - @SuppressWarnings("unchecked") - private ConciseSet evalCompare(CompareTupleFilter filter) { - int col = col(filter); - if (index[col] == null) - return all(); - - switch (filter.getOperator()) { - case ISNULL: - return index[col].getNull(); - case ISNOTNULL: - return all(); - case EQ: - return index[col].getEquals((ByteArray) filter.getFirstValue()); - case NEQ: - return all(); - case IN: - return index[col].getIn((Iterable<ByteArray>) filter.getValues()); - case NOTIN: - return all(); - case LT: - return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), false); - case LTE: - return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), true); - case GT: - return index[col].getRange((ByteArray) filter.getFirstValue(), false, null, false); - case GTE: - return index[col].getRange((ByteArray) filter.getFirstValue(), true, null, false); - default: - throw new IllegalStateException("Unsupported operator " + filter.getOperator()); - } - } - - private ConciseSet evalLogical(LogicalTupleFilter filter) { - List<? extends TupleFilter> children = filter.getChildren(); - - switch (filter.getOperator()) { - case AND: - return evalLogicalAnd(children); - case OR: - return evalLogicalOr(children); - case NOT: - return evalLogicalNot(children); - default: - throw new IllegalStateException("Unsupported operator " + filter.getOperator()); - } - } - - private ConciseSet evalLogicalAnd(List<? extends TupleFilter> children) { - ConciseSet set = all(); - - for (TupleFilter c : children) { - ConciseSet t = evaluate(c); - if (t == null) - continue; // because it's AND - - set.retainAll(t); - } - return set; - } - - private ConciseSet evalLogicalOr(List<? extends TupleFilter> children) { - ConciseSet set = new ConciseSet(); - - for (TupleFilter c : children) { - ConciseSet t = evaluate(c); - if (t == null) - return null; // because it's OR - - set.addAll(t); - } - return set; - } - - private ConciseSet evalLogicalNot(List<? extends TupleFilter> children) { - return all(); - } - - private ConciseSet all() { - return not(new ConciseSet()); - } - - private ConciseSet not(ConciseSet set) { - set.add(indexedBlocks); - set.complement(); - return set; - } - - private int col(CompareTupleFilter filter) { - return filter.getColumn().getColumnDesc().getZeroBasedIndex(); - } - - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java deleted file mode 100644 index bfacc0f..0000000 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.gridtable; - -import java.util.NavigableMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.kylin.common.util.ByteArray; - -import com.google.common.collect.Maps; - -import it.uniroma3.mat.extendedset.intset.ConciseSet; - -public class GTInvertedIndexOfColumn { - - final private IGTComparator comparator; - final private ReentrantReadWriteLock rwLock; - - private int nBlocks; - private NavigableMap<ByteArray, ConciseSet> rangeIndex; - private ConciseSet nullIndex; - - public GTInvertedIndexOfColumn(IGTComparator comparator) { - this.comparator = comparator; - this.rwLock = new ReentrantReadWriteLock(); - this.rangeIndex = Maps.newTreeMap(comparator); - this.nullIndex = new ConciseSet(); - } - - public void add(Iterable<ByteArray> codes, int blockId) { - rwLock.writeLock().lock(); - try { - for (ByteArray code : codes) { - if (comparator.isNull(code)) { - nullIndex.add(blockId); - continue; - } - ConciseSet set = rangeIndex.get(code); - if (set == null) { - set = new ConciseSet(); - rangeIndex.put(code.copy(), set); - } - set.add(blockId); - } - - if (blockId >= nBlocks) { - nBlocks = blockId + 1; - } - - } finally { - rwLock.writeLock().unlock(); - } - } - - public ConciseSet getNull() { - rwLock.readLock().lock(); - try { - return nullIndex.clone(); - } finally { - rwLock.readLock().unlock(); - } - } - - public ConciseSet getEquals(ByteArray code) { - rwLock.readLock().lock(); - try { - ConciseSet set = rangeIndex.get(code); - if (set == null) - return new ConciseSet(); - else - return set.clone(); - } finally { - rwLock.readLock().unlock(); - } - } - - public ConciseSet getIn(Iterable<ByteArray> codes) { - rwLock.readLock().lock(); - try { - ConciseSet r = new ConciseSet(); - for (ByteArray code : codes) { - ConciseSet set = rangeIndex.get(code); - if (set != null) - r.addAll(set); - } - return r; - } finally { - rwLock.readLock().unlock(); - } - } - - public ConciseSet getRange(ByteArray from, boolean fromInclusive, ByteArray to, boolean toInclusive) { - rwLock.readLock().lock(); - try { - ConciseSet r = new ConciseSet(); - if (from == null && to == null) { - r.add(nBlocks); - r.complement(); - return r; - } - NavigableMap<ByteArray, ConciseSet> subMap; - if (from == null) { - subMap = rangeIndex.headMap(to, toInclusive); - } else if (to == null) { - subMap = rangeIndex.tailMap(from, fromInclusive); - } else { - subMap = rangeIndex.subMap(from, fromInclusive, to, toInclusive); - } - for (ConciseSet set : subMap.values()) { - r.addAll(set); - } - return r; - } finally { - rwLock.readLock().unlock(); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java deleted file mode 100644 index 5e3e771..0000000 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.gridtable; - -import static org.junit.Assert.assertEquals; - -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.util.ArrayList; - -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.datatype.StringSerializer; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.apache.kylin.metadata.model.TblColRef; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.google.common.collect.Lists; - -import it.uniroma3.mat.extendedset.intset.ConciseSet; - -public class SimpleInvertedIndexTest extends LocalFileMetadataTestCase { - - GTInfo info; - GTInvertedIndex index; - ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList(); - ArrayList<ConciseSet> basicResults = Lists.newArrayList(); - - @BeforeClass - public static void setUp() throws Exception { - staticCreateTestMetadata(); - } - - @AfterClass - public static void after() throws Exception { - cleanAfterClass(); - } - - public SimpleInvertedIndexTest() { - - info = UnitTestSupport.advancedInfo(); - TblColRef colA = info.colRef(0); - - // block i contains value "i", the last is NULL - index = new GTInvertedIndex(info); - GTRowBlock mockBlock = GTRowBlock.allocate(info); - GTRowBlock.Writer writer = mockBlock.getWriter(); - GTRecord record = new GTRecord(info); - for (int i = 0; i < 10; i++) { - record.setValues(i < 9 ? "" + i : null, "", "", new Long(0), new BigDecimal(0)); - for (int j = 0; j < info.getRowBlockSize(); j++) { - writer.append(record); - } - writer.readyForFlush(); - index.add(mockBlock); - - writer.clearForNext(); - } - - basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL)); - basicResults.add(set(9)); - - basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL)); - basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); - - basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0)); - basicResults.add(set(0)); - - basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0)); - basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); - - basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5)); - basicResults.add(set(0, 5)); - - basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5)); - basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); - - basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3)); - basicResults.add(set(0, 1, 2)); - - basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3)); - basicResults.add(set(0, 1, 2, 3)); - - basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3)); - basicResults.add(set(4, 5, 6, 7, 8)); - - basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3)); - basicResults.add(set(3, 4, 5, 6, 7, 8)); - } - - @Test - public void testBasics() { - for (int i = 0; i < basicFilters.size(); i++) { - assertEquals(basicResults.get(i), index.filter(basicFilters.get(i))); - } - } - - @Test - public void testLogicalAnd() { - for (int i = 0; i < basicFilters.size(); i++) { - for (int j = 0; j < basicFilters.size(); j++) { - LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j)); - ConciseSet r = basicResults.get(i).clone(); - r.retainAll(basicResults.get(j)); - assertEquals(r, index.filter(f)); - } - } - } - - @Test - public void testLogicalOr() { - for (int i = 0; i < basicFilters.size(); i++) { - for (int j = 0; j < basicFilters.size(); j++) { - LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j)); - ConciseSet r = basicResults.get(i).clone(); - r.addAll(basicResults.get(j)); - assertEquals(r, index.filter(f)); - } - } - } - - @Test - public void testNotEvaluable() { - ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); - - CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0); - assertEquals(all, index.filter(notEvaluable)); - - LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable); - assertEquals(all, index.filter(or)); - - LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable); - assertEquals(basicResults.get(0), index.filter(and)); - } - - public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) { - CompareTupleFilter filter = new CompareTupleFilter(op); - filter.addChild(columnFilter(col)); - for (int i : ids) { - filter.addChild(constFilter(i)); - } - return filter; - } - - public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) { - LogicalTupleFilter filter = new LogicalTupleFilter(op); - for (TupleFilter f : filters) - filter.addChild(f); - return filter; - } - - public static ColumnTupleFilter columnFilter(TblColRef col) { - return new ColumnTupleFilter(col); - } - - public static ConstantTupleFilter constFilter(int id) { - byte[] space = new byte[10]; - ByteBuffer buf = ByteBuffer.wrap(space); - StringSerializer stringSerializer = new StringSerializer(DataType.getType("string")); - stringSerializer.serialize("" + id, buf); - ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position()); - return new ConstantTupleFilter(data); - } - - public static ConciseSet set(int... ints) { - ConciseSet set = new ConciseSet(); - for (int i : ints) - set.add(i); - return set; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-job/pom.xml ---------------------------------------------------------------------- diff --git a/core-job/pom.xml b/core-job/pom.xml index 36d34c8..9478555 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -39,6 +39,15 @@ <artifactId>kylin-core-cube</artifactId> </dependency> + <!-- Provided --> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-recipes</artifactId> http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/pom.xml ---------------------------------------------------------------------- diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 87c4438..0832e80 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -39,6 +39,29 @@ <artifactId>kylin-core-common</artifactId> </dependency> + <!-- Provided --> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + </dependency> + + <!-- Compiled --> <dependency> <groupId>net.sf.ehcache</groupId> <artifactId>ehcache</artifactId> http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java index 12162b2..fbdb0bb 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java @@ -21,6 +21,7 @@ package org.apache.kylin.dimension; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Map; @@ -33,7 +34,7 @@ import com.google.common.collect.Maps; /** * Encoding Boolean values to bytes */ -public class BooleanDimEnc extends DimensionEncoding { +public class BooleanDimEnc extends DimensionEncoding implements Serializable{ private static final long serialVersionUID = 1L; public static final String ENCODING_NAME = "boolean"; @@ -108,7 +109,7 @@ public class BooleanDimEnc extends DimensionEncoding { return new BooleanSerializer(); } - private class BooleanSerializer extends DataTypeSerializer<Object> { + public class BooleanSerializer extends DataTypeSerializer<Object> { private byte[] currentBuf() { byte[] buf = (byte[]) current.get(); http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java index 79e00ce..3b390d9 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java @@ -18,10 +18,12 @@ package org.apache.kylin.dimension; +import java.io.Serializable; + /** * This encoding is meant to be IDENTICAL to DateStrDictionary for 100% backward compatibility. */ -public class DateDimEnc extends AbstractDateDimEnc { +public class DateDimEnc extends AbstractDateDimEnc implements Serializable{ private static final long serialVersionUID = 1L; public static final int ID_9999_12_31 = 3652426; http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java index 48238dc..cfa10b9 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java @@ -21,6 +21,7 @@ package org.apache.kylin.dimension; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.io.Serializable; import java.nio.ByteBuffer; import org.apache.kylin.common.util.BytesUtil; @@ -29,7 +30,7 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class DictionaryDimEnc extends DimensionEncoding { +public class DictionaryDimEnc extends DimensionEncoding implements Serializable{ private static final long serialVersionUID = 1L; private static final Logger logger = LoggerFactory.getLogger(DictionaryDimEnc.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java index f7f02a0..9ce1577 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java @@ -21,6 +21,7 @@ package org.apache.kylin.dimension; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; @@ -29,7 +30,7 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class FixedLenDimEnc extends DimensionEncoding { +public class FixedLenDimEnc extends DimensionEncoding implements Serializable{ private static final long serialVersionUID = 1L; private static Logger logger = LoggerFactory.getLogger(FixedLenDimEnc.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java index f90a40e..a931450 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java @@ -21,6 +21,7 @@ package org.apache.kylin.dimension; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; @@ -43,7 +44,7 @@ import com.google.common.base.Preconditions; * <p> * Due to these limitations hex representation of hash values(with no padding, better with even characters) is more suitable */ -public class FixedLenHexDimEnc extends DimensionEncoding { +public class FixedLenHexDimEnc extends DimensionEncoding implements Serializable{ private static final long serialVersionUID = 1L; private static Logger logger = LoggerFactory.getLogger(FixedLenHexDimEnc.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java index aa954da..3650200 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java @@ -21,6 +21,7 @@ package org.apache.kylin.dimension; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; @@ -33,7 +34,7 @@ import org.slf4j.LoggerFactory; * deprecated use IntegerDimEnc instead * @deprecated */ -public class IntDimEnc extends DimensionEncoding { +public class IntDimEnc extends DimensionEncoding implements Serializable{ private static final long serialVersionUID = 1L; private static Logger logger = LoggerFactory.getLogger(IntDimEnc.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java index 0875a7f..e024696 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java @@ -21,6 +21,7 @@ package org.apache.kylin.dimension; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; @@ -36,7 +37,7 @@ import org.slf4j.LoggerFactory; * -2^(8*N-1) is not supported because the slot is reserved for null values. * -2^(8*N-1) will be encoded with warn, and its output will be null */ -public class IntegerDimEnc extends DimensionEncoding { +public class IntegerDimEnc extends DimensionEncoding implements Serializable{ private static final long serialVersionUID = 1L; private static Logger logger = LoggerFactory.getLogger(IntegerDimEnc.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java index c0c52d1..d998f44 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java @@ -26,13 +26,14 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; /** * not being used yet, prepared for future */ -public class OneMoreByteVLongDimEnc extends DimensionEncoding { +public class OneMoreByteVLongDimEnc extends DimensionEncoding implements Serializable{ private static final long serialVersionUID = 1L; private static Logger logger = LoggerFactory.getLogger(OneMoreByteVLongDimEnc.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java index 8c66d5d..4bef2b8 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java @@ -18,10 +18,12 @@ package org.apache.kylin.dimension; +import java.io.Serializable; + /** * This encoding is meant to be IDENTICAL to TimeStrDictionary for 100% backward compatibility. */ -public class TimeDimEnc extends AbstractDateDimEnc { +public class TimeDimEnc extends AbstractDateDimEnc implements Serializable{ private static final long serialVersionUID = 1L; public static final String ENCODING_NAME = "time"; http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java index 5d0e007..7f3a5f1 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java @@ -218,7 +218,7 @@ abstract public class MeasureTypeFactory<T> { } @SuppressWarnings("rawtypes") - private static class NeedRewriteOnlyMeasureType extends MeasureType { + public static class NeedRewriteOnlyMeasureType extends MeasureType { private Boolean needRewrite; http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java index 438a33f..c635cd6 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java @@ -77,7 +77,7 @@ public class HLLDistinctCountAggFunc { } @SuppressWarnings("serial") - private static class FixedValueHLLCMockup extends HLLCounter { + public static class FixedValueHLLCMockup extends HLLCounter { private Long value = null; http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java index 2f625af..d8fdc6e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java @@ -22,13 +22,14 @@ import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.io.Serializable; /** * Modified from Counter.java in https://github.com/addthis/stream-lib * * @param <T> */ -public class Counter<T> implements Externalizable { +public class Counter<T> implements Externalizable, Serializable{ protected T item; protected double count; http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index 86eea55..c04de0d 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -18,6 +18,7 @@ package org.apache.kylin.metadata.model; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -48,11 +49,11 @@ import com.google.common.collect.Sets; public class DataModelDesc extends RootPersistentEntity { private static final Logger logger = LoggerFactory.getLogger(DataModelDesc.class); - public static enum TableKind { + public static enum TableKind implements Serializable{ FACT, LOOKUP } - public static enum RealizationCapacity { + public static enum RealizationCapacity implements Serializable{ SMALL, MEDIUM, LARGE } http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java index 3ab9a46..c7666cb 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java @@ -113,7 +113,7 @@ public class JoinsTree implements Serializable { return chain.join; } - static class Chain implements Serializable { + public static class Chain implements Serializable { private static final long serialVersionUID = 1L; TableRef table; // pk side http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java index c6e6425..38c2de5 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java @@ -34,7 +34,7 @@ import java.io.Serializable; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class PartitionDesc implements Serializable { - public static enum PartitionType { + public static enum PartitionType implements Serializable{ APPEND, // UPDATE_INSERT // not used since 0.7.1 } http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java index 65c44df..689a314 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java @@ -18,10 +18,12 @@ package org.apache.kylin.metadata.model; +import java.io.Serializable; + /** * @author xduo * */ -public enum SegmentStatusEnum { +public enum SegmentStatusEnum implements Serializable{ NEW, READY, READY_PENDING } http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java index 9371f76..9d22dc6 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java @@ -19,6 +19,7 @@ package org.apache.kylin.metadata.model; import java.io.IOException; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -27,7 +28,7 @@ import org.apache.kylin.common.util.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class Segments<T extends ISegment> extends ArrayList<T> { +public class Segments<T extends ISegment> extends ArrayList<T> implements Serializable{ private static final long serialVersionUID = 1L; http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java index 27e2d57..872d2e4 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java @@ -18,7 +18,9 @@ package org.apache.kylin.metadata.realization; -public enum RealizationStatusEnum { +import java.io.Serializable; + +public enum RealizationStatusEnum implements Serializable{ DISABLED, READY, DESCBROKEN http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-storage/pom.xml ---------------------------------------------------------------------- diff --git a/core-storage/pom.xml b/core-storage/pom.xml index 9bd3f04..8c3da5f 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -39,8 +39,28 @@ <artifactId>kylin-core-cube</artifactId> </dependency> + <!-- Provided --> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + </dependency> + <!-- Env & Test --> <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-common</artifactId> <type>test-jar</type> http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/engine-spark/pom.xml ---------------------------------------------------------------------- diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index be2cdf6..7a478f3 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -67,11 +67,6 @@ <scope>provided</scope> </dependency> - <dependency> - <groupId>org.reflections</groupId> - <artifactId>reflections</artifactId> - </dependency> - <!-- Hadoop dependency --> <dependency> <groupId>org.apache.hadoop</groupId> @@ -116,45 +111,13 @@ <artifactId>maven-model</artifactId> <scope>test</scope> </dependency> + <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> + <groupId>org.reflections</groupId> + <artifactId>reflections</artifactId> + <version>0.9.10</version> + <scope>test</scope> </dependency> - - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <minimizeJar>false</minimizeJar> - <shadedArtifactAttached>true</shadedArtifactAttached> - <shadedClassifierName>job</shadedClassifierName> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - - </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java index 3d33aa8..20b39ee 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java @@ -19,15 +19,11 @@ package org.apache.kylin.engine.spark; import com.esotericsoftware.kryo.Kryo; -import com.google.common.collect.Sets; -import com.google.common.hash.Hashing; -import org.apache.kylin.measure.MeasureIngester; import org.apache.spark.serializer.KryoRegistrator; -import org.reflections.Reflections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Serializable; +import java.util.LinkedHashSet; import java.util.Set; /** @@ -39,7 +35,7 @@ public class KylinKryoRegistrator implements KryoRegistrator { @Override public void registerClasses(Kryo kryo) { - Set<Class> kyroClasses = Sets.newLinkedHashSet(); + Set<Class> kyroClasses = new LinkedHashSet(); kyroClasses.add(byte[].class); kyroClasses.add(int[].class); kyroClasses.add(byte[][].class); @@ -55,15 +51,11 @@ public class KylinKryoRegistrator implements KryoRegistrator { kyroClasses.add(java.util.HashMap.class); kyroClasses.add(java.util.TreeMap.class); kyroClasses.add(java.util.Properties.class); - kyroClasses.addAll(new Reflections("org.apache.kylin").getSubTypesOf(Serializable.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.dimension").getSubTypesOf(Serializable.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.cube").getSubTypesOf(Serializable.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.cube.model").getSubTypesOf(Object.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.metadata").getSubTypesOf(Object.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.model").getSubTypesOf(Object.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.measure").getSubTypesOf(Object.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.datatype").getSubTypesOf(org.apache.kylin.common.util.BytesSerializer.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.measure").getSubTypesOf(MeasureIngester.class)); + kyroClasses.add(java.math.MathContext.class); + kyroClasses.add(java.math.RoundingMode.class); + kyroClasses.add(java.util.concurrent.ConcurrentHashMap.class); + kyroClasses.add(java.util.Random.class); + kyroClasses.add(java.util.concurrent.atomic.AtomicLong.class); kyroClasses.add(org.apache.spark.sql.Row[].class); kyroClasses.add(org.apache.spark.sql.Row.class); @@ -74,65 +66,29 @@ public class KylinKryoRegistrator implements KryoRegistrator { kyroClasses.add(org.apache.spark.sql.types.DateType$.class); kyroClasses.add(org.apache.spark.sql.types.Metadata.class); kyroClasses.add(org.apache.spark.sql.types.StringType$.class); - kyroClasses.add(Hashing.murmur3_128().getClass()); kyroClasses.add(org.apache.spark.sql.execution.columnar.CachedBatch.class); kyroClasses.add(org.apache.spark.sql.types.Decimal.class); kyroClasses.add(scala.math.BigDecimal.class); - kyroClasses.add(java.math.MathContext.class); - kyroClasses.add(java.math.RoundingMode.class); - kyroClasses.add(java.util.concurrent.ConcurrentHashMap.class); - kyroClasses.add(java.util.Random.class); - kyroClasses.add(java.util.concurrent.atomic.AtomicLong.class); + kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class); kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc[].class); kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc[].class); kyroClasses.add(org.apache.kylin.metadata.model.TblColRef[].class); - kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class); - kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class); - kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class); - kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.PartitionType.class); - kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class); - kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class); + kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc[].class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer[].class); + kyroClasses.add(org.apache.kylin.measure.MeasureAggregator[].class); kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc[].class); kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc[].class); - kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc[].class); kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc[].class); - kyroClasses.add(org.apache.kylin.common.util.Array.class); - kyroClasses.add(org.apache.kylin.metadata.model.Segments.class); - kyroClasses.add(org.apache.kylin.metadata.realization.RealizationStatusEnum.class); - kyroClasses.add(org.apache.kylin.metadata.model.SegmentStatusEnum.class); - kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class); - kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class); - kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class); - kyroClasses.add(org.apache.kylin.measure.MeasureAggregator[].class); - kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer[].class); - kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class); - kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class); - kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class); - kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class); - kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class); - kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class); - kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class); - kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.DateDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class); - kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class); - kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class); - kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class); + + kylinClassByReflection1(kyroClasses); + kylinClassByReflection2(kyroClasses); + + kyroClasses.add(com.google.common.hash.Hashing.murmur3_128().getClass()); kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringArray.class); kyroClasses.add(org.roaringbitmap.buffer.MappeableContainer[].class); kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringBitmap.class); kyroClasses.add(org.roaringbitmap.buffer.MappeableArrayContainer.class); - kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class); - kyroClasses.add(org.apache.kylin.measure.topn.Counter.class); - kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class); - kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class); kyroClasses.add(com.tdunning.math.stats.AVLTreeDigest.class); kyroClasses.add(com.tdunning.math.stats.Centroid.class); @@ -151,6 +107,195 @@ public class KylinKryoRegistrator implements KryoRegistrator { // kryo.register(PercentileCounter.class, new JavaSerializer()); } + /** + * Generated by KyroMappingGenerator, method exceeds 150 lines, splits into two methods + * @param kyroClasses + */ + private void kylinClassByReflection1(Set<Class> kyroClasses) { + kyroClasses.add(org.apache.kylin.common.KylinConfig.class); + kyroClasses.add(org.apache.kylin.common.KylinConfigBase.class); + kyroClasses.add(org.apache.kylin.common.KylinConfigExt.class); + kyroClasses.add(org.apache.kylin.common.persistence.RootPersistentEntity.class); + kyroClasses.add(org.apache.kylin.common.util.Array.class); + kyroClasses.add(org.apache.kylin.common.util.ByteArray.class); + kyroClasses.add(org.apache.kylin.common.util.ByteArray.ImmutableByteArray.class); + kyroClasses.add(org.apache.kylin.common.util.Dictionary.class); + kyroClasses.add(org.apache.kylin.common.util.OrderedProperties.class); + kyroClasses.add(org.apache.kylin.common.util.Pair.class); + kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class); + kyroClasses.add(org.apache.kylin.cube.CubeInstance.class); + kyroClasses.add(org.apache.kylin.cube.CubeSegment.class); + kyroClasses.add(org.apache.kylin.cube.common.RowKeySplitter.class); + kyroClasses.add(org.apache.kylin.cube.cuboid.Cuboid.class); + kyroClasses.add(org.apache.kylin.cube.cuboid.CuboidScheduler.class); + kyroClasses.add(org.apache.kylin.cube.gridtable.TrimmedDimensionSerializer.class); + kyroClasses.add(org.apache.kylin.cube.kv.AbstractRowKeyEncoder.class); + kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class); + kyroClasses.add(org.apache.kylin.cube.kv.FuzzyKeyEncoder.class); + kyroClasses.add(org.apache.kylin.cube.kv.FuzzyMaskEncoder.class); + kyroClasses.add(org.apache.kylin.cube.kv.LazyRowKeyEncoder.class); + kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class); + kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class); + kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class); + kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.class); + kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich.class); + kyroClasses.add(org.apache.kylin.cube.model.DictionaryDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.DimensionDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.HBaseMappingDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.HierarchyDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.RowKeyDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.SelectRule.class); + kyroClasses.add(org.apache.kylin.cube.model.v1_4_0.CubeDesc.class); + kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.class); + kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictNode.class); + kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSlice.class); + kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSliceKey.class); + kyroClasses.add(org.apache.kylin.dict.CacheDictionary.class); + kyroClasses.add(org.apache.kylin.dict.DateStrDictionary.class); + kyroClasses.add(org.apache.kylin.dict.DictionaryInfo.class); + kyroClasses.add(org.apache.kylin.dict.NumberDictionary.class); + kyroClasses.add(org.apache.kylin.dict.NumberDictionary2.class); + kyroClasses.add(org.apache.kylin.dict.NumberDictionaryForestBuilder.Number2BytesConverter.class); + kyroClasses.add(org.apache.kylin.dict.StringBytesConverter.class); + kyroClasses.add(org.apache.kylin.dict.TimeStrDictionary.class); + kyroClasses.add(org.apache.kylin.dict.TrieDictionary.class); + kyroClasses.add(org.apache.kylin.dict.TrieDictionaryForest.class); + kyroClasses.add(org.apache.kylin.dict.lookup.SnapshotTable.class); + kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.BooleanSerializer.class); + kyroClasses.add(org.apache.kylin.dimension.DateDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.DictionarySerializer.class); + kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.FixedLenSerializer.class); + kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.FixedLenSerializer.class); + kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.IntegerSerializer.class); + kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.IntegerSerializer.class); + kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.VLongSerializer.class); + kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class); + kyroClasses.add(org.apache.kylin.engine.mr.common.BaseCuboidBuilder.class); + kyroClasses.add(org.apache.kylin.engine.mr.common.NDCuboidBuilder.class); + kyroClasses.add(org.apache.kylin.engine.spark.SparkCubingByLayer.class); + kyroClasses.add(org.apache.kylin.job.JobInstance.class); + kyroClasses.add(org.apache.kylin.job.dao.ExecutableOutputPO.class); + kyroClasses.add(org.apache.kylin.job.dao.ExecutablePO.class); + } + + /** + * Generated by KyroMappingGenerator + * @param kyroClasses + */ + private void kylinClassByReflection2(Set<Class> kyroClasses) { + kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class); + kyroClasses.add(org.apache.kylin.measure.MeasureAggregator.class); + kyroClasses.add(org.apache.kylin.measure.MeasureAggregators.class); + kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class); + kyroClasses.add(org.apache.kylin.measure.MeasureIngester.class); + kyroClasses.add(org.apache.kylin.measure.MeasureType.class); + kyroClasses.add(org.apache.kylin.measure.MeasureTypeFactory.NeedRewriteOnlyMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMaxAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMinAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalSumAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.DoubleIngester.class); + kyroClasses.add(org.apache.kylin.measure.basic.DoubleMaxAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.DoubleMinAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.DoubleSumAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongIngester.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongMaxAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongMinAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongSumAggregator.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapAggregator.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapSerializer.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class); + kyroClasses.add(org.apache.kylin.measure.dim.DimCountDistinctMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnSerializer.class); + kyroClasses.add(org.apache.kylin.measure.hllc.DenseRegister.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCAggregator.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCSerializer.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounter.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounterOld.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLDistinctCountAggFunc.FixedValueHLLCMockup.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HyperLogLogPlusTable.class); + kyroClasses.add(org.apache.kylin.measure.hllc.SingleValueRegister.class); + kyroClasses.add(org.apache.kylin.measure.hllc.SparseRegister.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileAggregator.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileCounter.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class); + kyroClasses.add(org.apache.kylin.measure.raw.RawAggregator.class); + kyroClasses.add(org.apache.kylin.measure.raw.RawMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.raw.RawSerializer.class); + kyroClasses.add(org.apache.kylin.measure.topn.Counter.class); + kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNAggregator.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNCounterSerializer.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNMeasureType.class); + kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryEntry.class); + kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryHistory.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.BigDecimalSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.BooleanSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DataType.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DateTimeSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleMutable.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.Int4Serializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.IntMutable.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.Long8Serializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.LongMutable.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.LongSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.StringSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class); + kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class); + kyroClasses.add(org.apache.kylin.metadata.model.DatabaseDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.ExternalFilterDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.FunctionDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.JoinDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.class); + kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.Chain.class); + kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.ModelDimensionDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.ParameterDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.PartitionType.class); + kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class); + kyroClasses.add(org.apache.kylin.metadata.model.Segments.class); + kyroClasses.add(org.apache.kylin.metadata.model.SegmentStatusEnum.class); + kyroClasses.add(org.apache.kylin.metadata.model.TableDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.ColumnStats.class); + kyroClasses.add(org.apache.kylin.metadata.model.TableRef.class); + kyroClasses.add(org.apache.kylin.metadata.model.TblColRef.class); + kyroClasses.add(org.apache.kylin.metadata.project.ProjectInstance.class); + kyroClasses.add(org.apache.kylin.metadata.project.RealizationEntry.class); + kyroClasses.add(org.apache.kylin.metadata.realization.RealizationStatusEnum.class); + kyroClasses.add(org.apache.kylin.metadata.streaming.StreamingConfig.class); + kyroClasses.add(org.apache.kylin.source.ReadableTable.TableSignature.class); + kyroClasses.add(org.apache.kylin.storage.hybrid.HybridInstance.class); + } + private static void addClassQuitely(Set<Class> kyroClasses, String className) { try { kyroClasses.add(Class.forName(className));
