KYLIN-2331 by layer spark cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5a8bc054 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5a8bc054 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5a8bc054 Branch: refs/heads/sparkcubing-rebase Commit: 5a8bc054147f634ded5f35522c7b5415894bcbda Parents: d7971d9 Author: shaofengshi <shaofeng...@apache.org> Authored: Sun Dec 25 15:59:16 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Tue Jan 3 14:40:24 2017 +0800 ---------------------------------------------------------------------- assembly/pom.xml | 5 +- .../main/config/assemblies/source-assembly.xml | 3 - .../java/org/apache/kylin/job/DeployUtil.java | 1 - build/conf/kylin-spark-conf.properties | 26 ++ build/conf/kylin.properties | 20 + .../apache/kylin/common/KylinConfigBase.java | 64 ++- .../kylin/common/persistence/ResourceStore.java | 35 ++ .../org/apache/kylin/common/util/Array.java | 2 +- .../apache/kylin/common/util/SplittedBytes.java | 2 +- .../java/org/apache/kylin/cube/CubeSegment.java | 2 +- .../kylin/cube/common/RowKeySplitter.java | 15 +- .../org/apache/kylin/cube/cuboid/Cuboid.java | 27 +- .../kylin/cube/cuboid/CuboidScheduler.java | 15 +- .../cube/inmemcubing/InMemCubeBuilder.java | 2 + .../kylin/cube/kv/AbstractRowKeyEncoder.java | 6 +- .../org/apache/kylin/cube/kv/CubeDimEncMap.java | 7 +- .../apache/kylin/cube/kv/RowKeyColumnIO.java | 2 +- .../org/apache/kylin/cube/kv/RowKeyEncoder.java | 19 +- .../kylin/cube/kv/RowKeyEncoderProvider.java | 2 +- .../kylin/cube/model/AggregationGroup.java | 4 +- .../org/apache/kylin/cube/model/CubeDesc.java | 2 +- .../cube/model/CubeJoinedFlatTableDesc.java | 2 +- .../cube/model/CubeJoinedFlatTableEnrich.java | 6 +- .../apache/kylin/cube/model/DictionaryDesc.java | 2 +- .../apache/kylin/cube/model/DimensionDesc.java | 13 +- .../kylin/cube/model/HBaseColumnDesc.java | 11 +- .../kylin/cube/model/HBaseColumnFamilyDesc.java | 9 +- .../kylin/cube/model/HBaseMappingDesc.java | 15 +- .../apache/kylin/cube/model/HierarchyDesc.java | 2 +- .../apache/kylin/cube/model/RowKeyColDesc.java | 2 +- .../org/apache/kylin/cube/model/RowKeyDesc.java | 17 +- .../org/apache/kylin/cube/model/SelectRule.java | 2 +- .../apache/kylin/gridtable/UnitTestSupport.java | 41 +- .../gridtable/AggregationCacheMemSizeTest.java | 6 +- .../gridtable/AggregationCacheSpillTest.java | 5 +- .../kylin/gridtable/SimpleGridTableTest.java | 11 +- .../gridtable/SimpleInvertedIndexTest.java | 3 +- .../metadata/measure/MeasureCodecTest.java | 6 +- .../apache/kylin/dict/AppendTrieDictionary.java | 6 +- .../org/apache/kylin/dict/NumberDictionary.java | 2 +- .../apache/kylin/dict/NumberDictionary2.java | 2 +- .../dict/NumberDictionaryForestBuilder.java | 4 +- .../apache/kylin/dict/StringBytesConverter.java | 2 +- .../kylin/job/execution/ExecutableManager.java | 21 +- .../kylin/dimension/AbstractDateDimEnc.java | 12 +- .../apache/kylin/dimension/BooleanDimEnc.java | 4 +- .../kylin/dimension/DictionaryDimEnc.java | 21 +- .../apache/kylin/dimension/FixedLenDimEnc.java | 4 +- .../kylin/dimension/FixedLenHexDimEnc.java | 4 +- .../org/apache/kylin/dimension/IntDimEnc.java | 4 +- .../apache/kylin/dimension/IntegerDimEnc.java | 4 +- .../kylin/dimension/OneMoreByteVLongDimEnc.java | 14 +- .../kylin/measure/BufferedMeasureCodec.java | 10 +- .../apache/kylin/measure/MeasureAggregator.java | 2 + .../kylin/measure/MeasureAggregators.java | 18 + .../org/apache/kylin/measure/MeasureCodec.java | 8 +- .../apache/kylin/measure/MeasureIngester.java | 12 +- .../org/apache/kylin/measure/MeasureType.java | 12 +- .../measure/basic/BigDecimalMaxAggregator.java | 14 + .../measure/basic/BigDecimalMinAggregator.java | 28 +- .../measure/basic/BigDecimalSumAggregator.java | 10 + .../kylin/measure/basic/DoubleIngester.java | 19 +- .../measure/basic/DoubleMaxAggregator.java | 20 +- .../measure/basic/DoubleMinAggregator.java | 20 +- .../measure/basic/DoubleSumAggregator.java | 18 +- .../kylin/measure/basic/LongIngester.java | 19 +- .../kylin/measure/basic/LongMaxAggregator.java | 20 +- .../kylin/measure/basic/LongMinAggregator.java | 20 +- .../kylin/measure/basic/LongSumAggregator.java | 18 +- .../kylin/measure/bitmap/BitmapAggregator.java | 13 + .../kylin/measure/bitmap/BitmapCounter.java | 8 +- .../kylin/measure/bitmap/BitmapMeasureType.java | 5 + .../kylin/measure/bitmap/BitmapSerializer.java | 2 +- .../ExtendedColumnMeasureType.java | 15 + .../kylin/measure/hllc/DenseRegister.java | 2 +- .../kylin/measure/hllc/HLLCAggregator.java | 7 + .../kylin/measure/hllc/HLLCMeasureType.java | 5 + .../kylin/measure/hllc/HLLCSerializer.java | 11 +- .../measure/hllc/HyperLogLogPlusTable.java | 2 +- .../kylin/measure/hllc/SingleValueRegister.java | 2 +- .../kylin/measure/hllc/SparseRegister.java | 2 +- .../apache/kylin/measure/raw/RawAggregator.java | 14 + .../apache/kylin/measure/raw/RawSerializer.java | 4 +- .../measure/topn/DoubleDeltaSerializer.java | 7 +- .../kylin/measure/topn/TopNAggregator.java | 9 + .../apache/kylin/measure/topn/TopNCounter.java | 2 +- .../kylin/measure/topn/TopNMeasureType.java | 4 +- .../metadata/datatype/BigDecimalSerializer.java | 2 +- .../metadata/datatype/BooleanSerializer.java | 36 +- .../metadata/datatype/DataTypeSerializer.java | 17 +- .../metadata/datatype/DateTimeSerializer.java | 30 +- .../metadata/datatype/DoubleSerializer.java | 28 +- .../kylin/metadata/datatype/Int4Serializer.java | 32 +- .../metadata/datatype/Long8Serializer.java | 31 +- .../kylin/metadata/datatype/LongSerializer.java | 32 +- .../kylin/metadata/model/FunctionDesc.java | 22 +- .../apache/kylin/metadata/model/JoinDesc.java | 7 +- .../kylin/metadata/model/JoinTableDesc.java | 4 +- .../apache/kylin/metadata/model/JoinsTree.java | 9 +- .../kylin/metadata/model/MeasureDesc.java | 9 +- .../metadata/model/ModelDimensionDesc.java | 10 +- .../kylin/metadata/model/ParameterDesc.java | 13 +- .../kylin/metadata/model/PartitionDesc.java | 6 +- .../apache/kylin/metadata/model/TableRef.java | 17 +- .../measure/AggregatorMemEstimateTest.java | 6 +- .../storage/gtrecord/DictGridTableTest.java | 41 +- .../kylin/engine/mr/BatchCubingJobBuilder2.java | 9 +- .../org/apache/kylin/engine/mr/HadoopUtil.java | 2 +- .../kylin/engine/mr/JobBuilderSupport.java | 12 +- .../engine/mr/common/BaseCuboidBuilder.java | 173 ++++++++ .../kylin/engine/mr/common/CubeStatsReader.java | 5 + .../kylin/engine/mr/common/NDCuboidBuilder.java | 96 +++++ .../engine/mr/steps/BaseCuboidMapperBase.java | 131 +----- .../kylin/engine/mr/steps/NDCuboidMapper.java | 58 +-- .../kylin/engine/mr/steps/CubeReducerTest.java | 3 +- engine-spark/pom.xml | 9 + .../engine/spark/SparkBatchCubingEngine2.java | 33 ++ .../spark/SparkBatchCubingJobBuilder2.java | 85 ++++ .../apache/kylin/engine/spark/SparkCubing.java | 88 +++- .../kylin/engine/spark/SparkCubingByLayer.java | 416 +++++++++++++++++++ .../kylin/engine/spark/SparkExecutable.java | 26 +- .../spark/cube/DefaultTupleConverter.java | 2 +- .../sandbox/kylin-spark-conf.properties | 28 ++ .../test_case_data/sandbox/kylin.properties | 10 + pom.xml | 2 +- .../kylin/rest/controller/CubeController.java | 8 +- server/pom.xml | 7 + .../apache/kylin/source/hive/HiveMRInput.java | 5 +- .../hbase/steps/RowValueDecoderTest.java | 7 +- tool/pom.xml | 4 + 130 files changed, 1738 insertions(+), 733 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/assembly/pom.xml ---------------------------------------------------------------------- diff --git a/assembly/pom.xml b/assembly/pom.xml index 65e3a8c..873abff 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -47,6 +47,10 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-engine-mr</artifactId> </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-engine-spark</artifactId> + </dependency> <!-- Env & Test --> <dependency> @@ -173,7 +177,6 @@ <shadedClassifierName>job</shadedClassifierName> <artifactSet> <excludes> - <exclude>io.netty:*</exclude> <exclude>org.apache.zookeeper:*</exclude> <exclude>net.sf.ehcache:*</exclude> <exclude>org.apache.httpcomponents:*</exclude> http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/assembly/src/main/config/assemblies/source-assembly.xml ---------------------------------------------------------------------- diff --git a/assembly/src/main/config/assemblies/source-assembly.xml b/assembly/src/main/config/assemblies/source-assembly.xml index fad45aa..92584b5 100644 --- a/assembly/src/main/config/assemblies/source-assembly.xml +++ b/assembly/src/main/config/assemblies/source-assembly.xml @@ -97,9 +97,6 @@ limitations under the License. <exclude>%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?docs(/.*)?] </exclude> - <!-- exclude unmaintained --> - <exclude>%regex[(?!((?!${project.build.directory}/)))?engine-spark(/.*)?] - </exclude> </excludes> </fileSet> <!-- LICENSE, NOTICE, DEPENDENCIES, git.properties, etc. calculated at build time --> http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java index a42b125..e8c7fae 100644 --- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java +++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java @@ -82,7 +82,6 @@ public class DeployUtil { config().overrideMRJobJarPath(jobJar.getAbsolutePath()); config().overrideCoprocessorLocalJar(coprocessorJar.getAbsolutePath()); - config().overrideSparkJobJarPath(getSparkJobJarFile().getAbsolutePath()); } private static String getPomVersion() { http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/build/conf/kylin-spark-conf.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin-spark-conf.properties b/build/conf/kylin-spark-conf.properties new file mode 100644 index 0000000..81567bb --- /dev/null +++ b/build/conf/kylin-spark-conf.properties @@ -0,0 +1,26 @@ +spark.yarn.submit.file.replication=1 +spark.yarn.executor.memoryOverhead=200 +spark.yarn.driver.memoryOverhead=384 +spark.master=yarn +spark.submit.deployMode=cluster +spark.eventLog.enabled=true +spark.yarn.scheduler.heartbeat.interval-ms=5000 +spark.yarn.preserve.staging.files=true +spark.yarn.queue=default +spark.yarn.containerLauncherMaxThreads=25 +spark.yarn.max.executor.failures=3 +spark.eventLog.dir=hdfs\:///kylin/spark-history +spark.history.kerberos.enabled=true +spark.history.provider=org.apache.spark.deploy.history.FsHistoryProvider +spark.history.ui.port=18080 +spark.history.fs.logDirectory=hdfs\:///kylin/spark-history +spark.executor.memory=4G +spark.storage.memoryFraction=0.3 +spark.executor.cores=4 +spark.executor.instances=8 +spark.history.kerberos.keytab=none +spark.history.kerberos.principal=none +#spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/apps/spark/spark-assembly-1.6.3-hadoop2.6.0.jar +spark.driver.extraJavaOptions=-Dhdp.version=current +spark.yarn.am.extraJavaOptions=-Dhdp.version=current +spark.executor.extraJavaOptions=-Dhdp.version=current http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 5e7d235..98b66cb 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -128,6 +128,26 @@ kylin.engine.mr.max-reducer-number=500 kylin.engine.mr.mapper-input-rows=1000000 +### Spark Engine ### + +# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run spark-submit +kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf + +# Spark install home, default be $KYLIN_HOME/spark/ +#kylin.engine.spark.spark-home= + +# Spark job submission properties file, default be $KYLIN_HOME/conf/kylin-spark-conf.properties +#kylin.engine.spark.properties-file= + +# Estimate the RDD partition numbers +kylin.engine.spark.rdd-partition-cut-mb=200 + +# Minimal partition numbers of rdd +kylin.engine.spark.min-partition=1 + +# Max partition numbers of rdd +kylin.engine.spark.max-partition=500 + ### CUBE | DICTIONARY ### # 'auto', 'inmem', 'layer' or 'random' for testing http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index d73b694..b0b18ce 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -640,6 +640,7 @@ abstract public class KylinConfigBase implements Serializable { // ref constants in IEngineAware r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine"); r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2"); + r.put(4, "org.apache.kylin.engine.spark.SparkBatchCubingEngine2"); return r; } @@ -716,29 +717,62 @@ abstract public class KylinConfigBase implements Serializable { // ENGINE.SPARK // ============================================================================ - public String getKylinSparkJobJarPath() { - final String jobJar = getOptional("kylin.engine.spark.job-jar"); - if (StringUtils.isNotEmpty(jobJar)) { - return jobJar; + public String getSparkHome() { + String sparkHome = getOptional("kylin.engine.spark.spark-home", "spark"); + File f = new File(sparkHome); + if (f.exists()) { + return f.getAbsolutePath(); + } else { + String home = getKylinHome(); + f = new File(home, sparkHome); + if (f.exists()) { + return f.getAbsolutePath(); + } } - String kylinHome = getKylinHome(); - if (StringUtils.isEmpty(kylinHome)) { - return ""; + + throw new IllegalArgumentException("Spark home '" + sparkHome + "' does not exist, check 'kylin.engine.spark.spark-home' in kylin.properties"); + + } + + public String getSparkHadoopConfDir() { + return getRequired("kylin.engine.spark.env.hadoop-conf-dir"); + } + + public String getSparkConfFile() { + String conf = getOptional("kylin.engine.spark.properties-file", "conf/kylin-spark-conf.properties"); + File f = new File(conf); + if (f.exists()) { + return f.getAbsolutePath(); + } else { + String home = getKylinHome(); + f = new File(home, conf); + if (f.exists()) { + return f.getAbsolutePath(); + } } - return getFileName(kylinHome + File.separator + "lib", SPARK_JOB_JAR_NAME_PATTERN); + + throw new IllegalArgumentException("Spark conf properties file '" + conf + "' does not exist."); } - public void overrideSparkJobJarPath(String path) { - logger.info("override " + "kylin.engine.spark.job-jar" + " to " + path); - System.setProperty("kylin.engine.spark.job-jar", path); + public String getSparkAdditionalJars() { + return getOptional("kylin.engine.spark.additional-jars", ""); } - public String getSparkHome() { - return getRequired("kylin.engine.spark.spark-home"); + public float getSparkRDDPartitionCutMB() { + return Float.valueOf(getOptional("kylin.engine.spark.rdd-partition-cut-mb", "200.0")); + } + + + public int getSparkMinPartition() { + return Integer.valueOf(getOptional("kylin.engine.spark.min-partition", "1")); + } + + public int getSparkMaxPartition() { + return Integer.valueOf(getOptional("kylin.engine.spark.max-partition", "500")); } - public String getSparkMaster() { - return getRequired("kylin.engine.spark.spark-master"); + public boolean isSparkSanityCheckEnabled() { + return Boolean.parseBoolean(getOptional("kylin.engine.spark.sanity-check-enabled", "false")); } // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java index 0580576..9549569 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java @@ -21,17 +21,21 @@ package org.apache.kylin.common.persistence; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.NavigableSet; import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.ClassUtil; +import org.apache.kylin.common.util.OptionsHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -311,4 +315,35 @@ abstract public class ResourceStore { return collector; } + public static String dumpResources(KylinConfig kylinConfig, Collection<String> dumpList) throws IOException { + File tmp = File.createTempFile("kylin_job_meta", ""); + FileUtils.forceDelete(tmp); // we need a directory, so delete the file first + + File metaDir = new File(tmp, "meta"); + metaDir.mkdirs(); + + // write kylin.properties + File kylinPropsFile = new File(metaDir, "kylin.properties"); + kylinConfig.writeProperties(kylinPropsFile); + + ResourceStore from = ResourceStore.getStore(kylinConfig); + KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()); + ResourceStore to = ResourceStore.getStore(localConfig); + for (String path : dumpList) { + RawResource res = from.getResource(path); + if (res == null) + throw new IllegalStateException("No resource found at -- " + path); + to.putResource(path, res.inputStream, res.timestamp); + res.inputStream.close(); + } + + String metaDirURI = OptionsHelper.convertToFileURL(metaDir.getAbsolutePath()); + if (metaDirURI.startsWith("/")) // note Path on windows is like "d:/../..." + metaDirURI = "file://" + metaDirURI; + else + metaDirURI = "file:///" + metaDirURI; + logger.info("meta dir is: " + metaDirURI); + + return metaDirURI; + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-common/src/main/java/org/apache/kylin/common/util/Array.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Array.java b/core-common/src/main/java/org/apache/kylin/common/util/Array.java index 7447b46..b25b764 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/Array.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/Array.java @@ -23,7 +23,7 @@ import java.util.Arrays; /* * An array with correct equals(), hashCode(), compareTo() and toString() */ -public class Array<T> implements Comparable<Array<T>> { +public class Array<T> implements Comparable<Array<T>>, java.io.Serializable { public T[] data; public Array(T[] data) { http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java b/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java index 8751b78..ae380cb 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/SplittedBytes.java @@ -22,7 +22,7 @@ package org.apache.kylin.common.util; * @author George Song (ysong1) * */ -public class SplittedBytes { +public class SplittedBytes implements java.io.Serializable { public SplittedBytes(int length) { this.value = new byte[length]; this.length = 0; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java index 36a6044..45310f0 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java @@ -51,7 +51,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegment { +public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegment, java.io.Serializable { @JsonBackReference private CubeInstance cubeInstance; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java index 67f1751..acebce4 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java @@ -26,9 +26,11 @@ import org.apache.kylin.cube.kv.CubeDimEncMap; import org.apache.kylin.cube.kv.RowConstants; import org.apache.kylin.cube.kv.RowKeyColumnIO; import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.cube.model.RowKeyColDesc; +import org.apache.kylin.dimension.IDimensionEncodingMap; import org.apache.kylin.metadata.model.TblColRef; -public class RowKeySplitter { +public class RowKeySplitter implements java.io.Serializable { private CubeDesc cubeDesc; private RowKeyColumnIO colIO; @@ -37,7 +39,6 @@ public class RowKeySplitter { private int[] splitOffsets; private int bufferSize; - private long lastSplittedCuboidId; private boolean enableSharding; private short shardId; @@ -64,7 +65,13 @@ public class RowKeySplitter { public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) { this.enableSharding = cubeSeg.isEnableSharding(); this.cubeDesc = cubeSeg.getCubeDesc(); - this.colIO = new RowKeyColumnIO(new CubeDimEncMap(cubeSeg)); + IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg); + + for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) { + dimEncoding.get(rowKeyColDesc.getColRef()); + } + + this.colIO = new RowKeyColumnIO(dimEncoding); this.splitBuffers = new SplittedBytes[splitLen]; this.splitOffsets = new int[splitLen]; @@ -105,7 +112,7 @@ public class RowKeySplitter { System.arraycopy(bytes, offset, cuboidIdSplit.value, 0, RowConstants.ROWKEY_CUBOIDID_LEN); offset += RowConstants.ROWKEY_CUBOIDID_LEN; - lastSplittedCuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length); + long lastSplittedCuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length); Cuboid cuboid = Cuboid.findById(cubeDesc, lastSplittedCuboidId); // rowkey columns http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java index 7503fbf..dd22d6a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/Cuboid.java @@ -18,15 +18,10 @@ package org.apache.kylin.cube.cuboid; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.google.common.collect.ComparisonChain; +import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping; @@ -37,12 +32,16 @@ import org.apache.kylin.cube.model.RowKeyColDesc; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.TblColRef; -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; -public class Cuboid implements Comparable<Cuboid> { +public class Cuboid implements Comparable<Cuboid>, java.io.Serializable { private final static Map<String, Map<Long, Cuboid>> CUBOID_CACHE = new ConcurrentHashMap<String, Map<Long, Cuboid>>(); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java index 733aded..ffb0a5e 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java @@ -21,6 +21,12 @@ package org.apache.kylin.cube.cuboid; /** */ +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.kylin.cube.model.AggregationGroup; +import org.apache.kylin.cube.model.CubeDesc; + import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -28,14 +34,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.kylin.cube.model.AggregationGroup; -import org.apache.kylin.cube.model.CubeDesc; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -public class CuboidScheduler { +public class CuboidScheduler implements java.io.Serializable { private final CubeDesc cubeDesc; private final long max; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java index 66a444e..b4cba39 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java @@ -462,6 +462,8 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { for (int i = 0; i < totalSum.length; i++) { if (totalSum[i] instanceof DoubleMutable) { totalSum[i] = Math.round(((DoubleMutable) totalSum[i]).get()); + } else if (totalSum[i] instanceof Double) { + totalSum[i] = Math.round(((Double) totalSum[i]).doubleValue()); } else if (totalSum[i] instanceof TopNCounter) { TopNCounter counter = (TopNCounter) totalSum[i]; Iterator<Counter> iterator = counter.iterator(); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java index bfe6eb4..2becde4 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/AbstractRowKeyEncoder.java @@ -18,8 +18,6 @@ package org.apache.kylin.cube.kv; -import java.util.Map; - import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.cube.CubeSegment; @@ -30,12 +28,14 @@ import org.apache.kylin.metadata.model.TblColRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; + /** * * @author xjiang * */ -public abstract class AbstractRowKeyEncoder { +public abstract class AbstractRowKeyEncoder implements java.io.Serializable { protected static final Logger logger = LoggerFactory.getLogger(AbstractRowKeyEncoder.class); public static final byte DEFAULT_BLANK_BYTE = DimensionEncoding.NULL; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java index a4d2d6f..bd9554a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java @@ -18,8 +18,7 @@ package org.apache.kylin.cube.kv; -import java.util.Map; - +import com.google.common.collect.Maps; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.model.CubeDesc; @@ -33,9 +32,9 @@ import org.apache.kylin.metadata.model.TblColRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Maps; +import java.util.Map; -public class CubeDimEncMap implements IDimensionEncodingMap { +public class CubeDimEncMap implements IDimensionEncodingMap, java.io.Serializable { private static final Logger logger = LoggerFactory.getLogger(CubeDimEncMap.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java index fbb93db..65911a0 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnIO.java @@ -29,7 +29,7 @@ import org.apache.kylin.metadata.model.TblColRef; * * @author yangli9 */ -public class RowKeyColumnIO { +public class RowKeyColumnIO implements java.io.Serializable { //private static final Logger logger = LoggerFactory.getLogger(RowKeyColumnIO.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java index bf20de1..a669fb1 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoder.java @@ -18,11 +18,7 @@ package org.apache.kylin.cube.kv; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - +import com.google.common.base.Preconditions; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.common.util.ImmutableBitSet; @@ -32,9 +28,12 @@ import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.gridtable.GTRecord; import org.apache.kylin.metadata.model.TblColRef; -import com.google.common.base.Preconditions; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; -public class RowKeyEncoder extends AbstractRowKeyEncoder { +public class RowKeyEncoder extends AbstractRowKeyEncoder implements java.io.Serializable { private int bodyLength = 0; private RowKeyColumnIO colIO; @@ -42,11 +41,13 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder { protected boolean enableSharding; private int uhcOffset = -1;//it's a offset to the beginning of body private int uhcLength = -1; + private int headerLength; public RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) { super(cubeSeg, cuboid); enableSharding = cubeSeg.isEnableSharding(); - Set<TblColRef> shardByColumns = cubeSeg.getShardByColumns(); + headerLength = cubeSeg.getRowKeyPreambleSize(); + Set<TblColRef> shardByColumns = cubeSeg.getCubeDesc().getShardByColumns(); if (shardByColumns.size() > 1) { throw new IllegalStateException("Does not support multiple UHC now"); } @@ -61,7 +62,7 @@ public class RowKeyEncoder extends AbstractRowKeyEncoder { } public int getHeaderLength() { - return cubeSeg.getRowKeyPreambleSize(); + return headerLength; } public int getBytesLength() { http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java index 2b1dea7..791b564 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyEncoderProvider.java @@ -25,7 +25,7 @@ import org.apache.kylin.cube.cuboid.Cuboid; import com.google.common.collect.Maps; -public class RowKeyEncoderProvider { +public class RowKeyEncoderProvider implements java.io.Serializable { private CubeSegment cubeSegment; private HashMap<Long, RowKeyEncoder> rowKeyEncoders; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java index 9bd082f..5f3c92c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java @@ -35,8 +35,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class AggregationGroup { - public static class HierarchyMask { +public class AggregationGroup implements java.io.Serializable { + public static class HierarchyMask implements java.io.Serializable { public long fullMask; // 00000111 public long[] allMasks; // 00000100,00000110,00000111 public long[] dims; // 00000100,00000010,00000001 http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 4686a15..0e54b98 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -100,7 +100,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { LOOKUP, PK_FK, EXTENDED_COLUMN } - public static class DeriveInfo { + public static class DeriveInfo implements java.io.Serializable { public DeriveType type; public JoinDesc join; public TblColRef[] columns; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java index 94e1a7c..2a68116 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java @@ -36,7 +36,7 @@ import com.google.common.collect.Maps; /** */ -public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc { +public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc, java.io.Serializable { protected final String tableName; protected final CubeDesc cubeDesc; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java index a1312b5..e829aeb 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java @@ -18,8 +18,6 @@ package org.apache.kylin.cube.model; -import java.util.List; - import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.FunctionDesc; @@ -28,10 +26,12 @@ import org.apache.kylin.metadata.model.ISegment; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; +import java.util.List; + /** * An enrich of IJoinedFlatTableDesc for cubes */ -public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc { +public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc, java.io.Serializable { private CubeDesc cubeDesc; private IJoinedFlatTableDesc flatDesc; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java index f471f9f..ca2183a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DictionaryDesc.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class DictionaryDesc { +public class DictionaryDesc implements java.io.Serializable { @JsonProperty("column") private String column; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java index cd75228..3488c36 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java @@ -20,7 +20,6 @@ package org.apache.kylin.cube.model; import java.util.Arrays; -import org.apache.commons.lang.NotImplementedException; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.JoinTableDesc; @@ -35,7 +34,7 @@ import com.google.common.base.Objects; /** */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class DimensionDesc { +public class DimensionDesc implements java.io.Serializable { @JsonProperty("name") private String name; @@ -139,16 +138,6 @@ public class DimensionDesc { } @Override - public boolean equals(Object o) { - throw new NotImplementedException(); - } - - @Override - public int hashCode() { - throw new NotImplementedException(); - } - - @Override public String toString() { return Objects.toStringHelper(this).add("name", name).add("table", table).add("column", column).add("derived", Arrays.toString(derived)).add("join", join).toString(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java index fb491f8..7007342 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnDesc.java @@ -18,19 +18,18 @@ package org.apache.kylin.cube.model; -import java.util.Arrays; - -import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.model.MeasureDesc; - import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.kylin.metadata.model.FunctionDesc; +import org.apache.kylin.metadata.model.MeasureDesc; + +import java.util.Arrays; /** */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class HBaseColumnDesc { +public class HBaseColumnDesc implements java.io.Serializable { @JsonProperty("qualifier") private String qualifier; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java index c5b2e19..85c2c17 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseColumnFamilyDesc.java @@ -18,18 +18,17 @@ package org.apache.kylin.cube.model; -import java.util.Arrays; - -import org.apache.kylin.metadata.model.MeasureDesc; - import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.kylin.metadata.model.MeasureDesc; + +import java.util.Arrays; /** */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class HBaseColumnFamilyDesc { +public class HBaseColumnFamilyDesc implements java.io.Serializable { @JsonProperty("name") private String name; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java index 2ef1e17..9ad8407 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java @@ -18,22 +18,21 @@ package org.apache.kylin.cube.model; -import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedList; - +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; /** */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class HBaseMappingDesc { +public class HBaseMappingDesc implements java.io.Serializable { @JsonProperty("column_family") private HBaseColumnFamilyDesc[] columnFamily; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java index f88d4d2..b72f220 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HierarchyDesc.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; /** */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class HierarchyDesc { +public class HierarchyDesc implements java.io.Serializable { @JsonProperty("level") private String level; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java index ef34a9b..71fd4b9 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java @@ -40,7 +40,7 @@ import com.google.common.base.Preconditions; * */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class RowKeyColDesc { +public class RowKeyColDesc implements java.io.Serializable { @JsonProperty("column") private String column; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java index f1a403d..00557c5 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java @@ -18,24 +18,23 @@ package org.apache.kylin.cube.model; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Objects; +import org.apache.commons.lang.ArrayUtils; +import org.apache.kylin.metadata.model.TblColRef; + import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import org.apache.commons.lang.ArrayUtils; -import org.apache.kylin.metadata.model.TblColRef; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Objects; - /** */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class RowKeyDesc { +public class RowKeyDesc implements java.io.Serializable { @JsonProperty("rowkey_columns") private RowKeyColDesc[] rowkeyColumns; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java index 63b0fc4..4a6c510 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/SelectRule.java @@ -20,7 +20,7 @@ package org.apache.kylin.cube.model; import com.fasterxml.jackson.annotation.JsonProperty; -public class SelectRule { +public class SelectRule implements java.io.Serializable { @JsonProperty("hierarchy_dims") public String[][] hierarchy_dims; @JsonProperty("mandatory_dims") http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java index b8d116c..6190f29 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java @@ -28,7 +28,6 @@ import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.gridtable.GTInfo.Builder; import org.apache.kylin.measure.hllc.HLLCounter; import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.datatype.LongMutable; public class UnitTestSupport { @@ -84,16 +83,16 @@ public class UnitTestSupport { String d_01_15 = datePlus("2015-01-15", i * 4); String d_01_16 = datePlus("2015-01-16", i * 4); String d_01_17 = datePlus("2015-01-17", i * 4); - result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_14, "Yang", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_14, "Luke", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_15, "Xu", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_15, "Dong", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_15, "Jason", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_16, "Mahone", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_16, "Shaofeng", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_16, "Qianhao", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_16, "George", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_17, "Kejia", "Food", new Long(10), new BigDecimal("10.5"))); } return result; } @@ -106,16 +105,16 @@ public class UnitTestSupport { String d_01_15 = datePlus("2015-01-15", i * 4); String d_01_16 = datePlus("2015-01-16", i * 4); String d_01_17 = datePlus("2015-01-17", i * 4); - result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_14, "Yang", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_14, "Luke", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_15, "Xu", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_15, "Dong", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_15, "Jason", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_16, "Mahone", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_16, "Shaofeng", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_16, "Qianhao", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_16, "George", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_17, "Kejia", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); } return result; } http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java index 3807b70..b164e54 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java @@ -36,8 +36,6 @@ import org.apache.kylin.measure.bitmap.BitmapAggregator; import org.apache.kylin.measure.bitmap.BitmapCounter; import org.apache.kylin.measure.hllc.HLLCAggregator; import org.apache.kylin.measure.hllc.HLLCounter; -import org.apache.kylin.metadata.datatype.DoubleMutable; -import org.apache.kylin.metadata.datatype.LongMutable; import org.github.jamm.MemoryMeter; import org.junit.Test; @@ -100,10 +98,10 @@ public class AggregationCacheMemSizeTest { private MeasureAggregator<?>[] createNoMemHungryAggrs() { LongSumAggregator longSum = new LongSumAggregator(); - longSum.aggregate(new LongMutable(10)); + longSum.aggregate(new Long(10)); DoubleSumAggregator doubleSum = new DoubleSumAggregator(); - doubleSum.aggregate(new DoubleMutable(10)); + doubleSum.aggregate(new Double(10)); BigDecimalSumAggregator decimalSum = new BigDecimalSumAggregator(); decimalSum.aggregate(new BigDecimal("12345678901234567890.123456789")); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java index d7787e0..74c2659 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java @@ -28,7 +28,6 @@ import java.util.List; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.metadata.datatype.LongMutable; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -91,7 +90,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase { for (GTRecord record : scanner) { assertNotNull(record); Object[] returnRecord = record.getValues(); - assertEquals(20, ((LongMutable) returnRecord[3]).get()); + assertEquals(20, ((Long) returnRecord[3]).longValue()); assertEquals(21, ((BigDecimal) returnRecord[4]).longValue()); count++; @@ -133,7 +132,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase { for (GTRecord record : scanner) { assertNotNull(record); Object[] returnRecord = record.getValues(); - assertEquals(80000, ((LongMutable) returnRecord[3]).get()); + assertEquals(80000, ((Long) returnRecord[3]).longValue()); assertEquals(84000, ((BigDecimal) returnRecord[4]).longValue()); count++; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java index f315b1e..4ac6644 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java @@ -29,7 +29,6 @@ import java.util.List; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.gridtable.memstore.GTSimpleMemStore; -import org.apache.kylin.metadata.datatype.LongMutable; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -97,7 +96,7 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { Object[] v = r.getValues(); assertTrue(((String) v[0]).startsWith("2015-")); assertTrue(((String) v[2]).equals("Food")); - assertTrue(((LongMutable) v[3]).get() == 10); + assertTrue(((Long) v[3]).longValue() == 10); assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5); System.out.println(r); } @@ -114,19 +113,19 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { Object[] v = r.getValues(); switch (i) { case 0: - assertTrue(((LongMutable) v[3]).get() == 20); + assertTrue(((Long) v[3]).longValue() == 20); assertTrue(((BigDecimal) v[4]).doubleValue() == 21.0); break; case 1: - assertTrue(((LongMutable) v[3]).get() == 30); + assertTrue(((Long) v[3]).longValue() == 30); assertTrue(((BigDecimal) v[4]).doubleValue() == 31.5); break; case 2: - assertTrue(((LongMutable) v[3]).get() == 40); + assertTrue(((Long) v[3]).longValue() == 40); assertTrue(((BigDecimal) v[4]).doubleValue() == 42.0); break; case 3: - assertTrue(((LongMutable) v[3]).get() == 10); + assertTrue(((Long) v[3]).longValue() == 10); assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5); break; default: http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java index 2cf58ac..5e3e771 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java @@ -27,7 +27,6 @@ import java.util.ArrayList; import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.datatype.LongMutable; import org.apache.kylin.metadata.datatype.StringSerializer; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; @@ -72,7 +71,7 @@ public class SimpleInvertedIndexTest extends LocalFileMetadataTestCase { GTRowBlock.Writer writer = mockBlock.getWriter(); GTRecord record = new GTRecord(info); for (int i = 0; i < 10; i++) { - record.setValues(i < 9 ? "" + i : null, "", "", new LongMutable(0), new BigDecimal(0)); + record.setValues(i < 9 ? "" + i : null, "", "", new Long(0), new BigDecimal(0)); for (int j = 0; j < info.getRowBlockSize(); j++) { writer.append(record); } http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java index fc20418..ad4d90b 100644 --- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java +++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java @@ -27,8 +27,6 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.measure.BufferedMeasureCodec; import org.apache.kylin.measure.bitmap.BitmapCounter; import org.apache.kylin.measure.hllc.HLLCounter; -import org.apache.kylin.metadata.datatype.DoubleMutable; -import org.apache.kylin.metadata.datatype.LongMutable; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.junit.AfterClass; @@ -54,8 +52,8 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase { MeasureDesc[] descs = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("bitmap") }; BufferedMeasureCodec codec = new BufferedMeasureCodec(descs); - DoubleMutable d = new DoubleMutable(1.0); - LongMutable l = new LongMutable(2); + Double d = new Double(1.0); + Long l = new Long(2); BigDecimal b = new BigDecimal("333.1234"); HLLCounter hllc = new HLLCounter(16); hllc.add("1234567"); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java index 32bfde6..a029f9f 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java @@ -141,7 +141,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> { } } - public static class DictSliceKey implements WritableComparable { + public static class DictSliceKey implements WritableComparable, java.io.Serializable { byte[] key; public static DictSliceKey wrap(byte[] key) { @@ -182,7 +182,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> { } } - public static class DictSlice<T> implements Writable { + public static class DictSlice<T> implements Writable, java.io.Serializable { public DictSlice() { } @@ -446,7 +446,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> { } } - public static class DictNode implements Writable { + public static class DictNode implements Writable, java.io.Serializable { public byte[] part; public int id = -1; public boolean isEndOfValue; http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java index 12efbd3..5da93ef 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java @@ -153,7 +153,7 @@ public class NumberDictionary<T> extends TrieDictionary<T> { } } - static ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>(); + static transient ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>(); // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java index b5ce1ff..80e9940 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary2.java @@ -24,7 +24,7 @@ package org.apache.kylin.dict; @SuppressWarnings("serial") public class NumberDictionary2<T> extends NumberDictionary<T> { - static ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>(); + static transient ThreadLocal<NumberBytesCodec> localCodec = new ThreadLocal<NumberBytesCodec>(); // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java index 5502a74..7c750f4 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java @@ -26,10 +26,10 @@ import org.apache.kylin.dict.NumberDictionary.NumberBytesCodec; */ public class NumberDictionaryForestBuilder extends TrieDictionaryForestBuilder<String> { - public static class Number2BytesConverter implements BytesConverter<String> { + public static class Number2BytesConverter implements BytesConverter<String>, java.io.Serializable { static final int MAX_DIGITS_BEFORE_DECIMAL_POINT = NumberDictionary.MAX_DIGITS_BEFORE_DECIMAL_POINT; - static final ThreadLocal<NumberBytesCodec> LOCAL = new ThreadLocal<NumberBytesCodec>(); + static final transient ThreadLocal<NumberBytesCodec> LOCAL = new ThreadLocal<NumberBytesCodec>(); static NumberBytesCodec getCodec() { NumberBytesCodec codec = LOCAL.get(); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java b/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java index 0bec6a1..9107a4c 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/StringBytesConverter.java @@ -20,7 +20,7 @@ package org.apache.kylin.dict; import org.apache.kylin.common.util.Bytes; -public class StringBytesConverter implements BytesConverter<String> { +public class StringBytesConverter implements BytesConverter<String>, java.io.Serializable { @Override public byte[] convertToBytes(String v) { http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java index 0273fd8..466cdad 100644 --- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java +++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java @@ -18,12 +18,9 @@ package org.apache.kylin.job.execution; -import java.lang.reflect.Constructor; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.ClassUtil; import org.apache.kylin.job.dao.ExecutableDao; @@ -34,9 +31,11 @@ import org.apache.kylin.job.exception.PersistentException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; +import java.lang.reflect.Constructor; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** */ @@ -352,6 +351,10 @@ public class ExecutableManager { } } } + + if (job.getStatus() == ExecutableState.SUCCEED) { + updateJobOutput(job.getId(), ExecutableState.READY, null, null); + } } public void pauseJob(String jobId) { http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java index ec6347f..a54bcda 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/AbstractDateDimEnc.java @@ -18,6 +18,10 @@ package org.apache.kylin.dimension; +import org.apache.kylin.common.util.BytesUtil; +import org.apache.kylin.common.util.DateFormat; +import org.apache.kylin.metadata.datatype.DataTypeSerializer; + import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; @@ -25,10 +29,6 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.DateFormat; -import org.apache.kylin.metadata.datatype.DataTypeSerializer; - public class AbstractDateDimEnc extends DimensionEncoding { private static final long serialVersionUID = 1L; @@ -81,11 +81,9 @@ public class AbstractDateDimEnc extends DimensionEncoding { @Override public DataTypeSerializer<Object> asDataTypeSerializer() { return new DataTypeSerializer<Object>() { - // be thread-safe and avoid repeated obj creation - private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); private byte[] currentBuf() { - byte[] buf = current.get(); + byte[] buf = (byte[]) current.get(); if (buf == null) { buf = new byte[fixedLen]; current.set(buf); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java index c3f4c11..75e50a1 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java @@ -109,11 +109,9 @@ public class BooleanDimEnc extends DimensionEncoding { } private class BooleanSerializer extends DataTypeSerializer<Object> { - // be thread-safe and avoid repeated obj creation - private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); private byte[] currentBuf() { - byte[] buf = current.get(); + byte[] buf = (byte[]) current.get(); if (buf == null) { buf = new byte[fixedLen]; current.set(buf); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java index 500b410..48238dc 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java @@ -39,12 +39,15 @@ public class DictionaryDimEnc extends DimensionEncoding { // ============================================================================ // could use a lazy loading trick here, to prevent loading all dictionaries of a segment at once - private final Dictionary<String> dict; - private final int fixedLen; + private Dictionary<String> dict; + private int fixedLen; // used in encode(), when a value does not exist in dictionary - private final int roundingFlag; - private final byte defaultByte; + private int roundingFlag; + private byte defaultByte; + + public DictionaryDimEnc() { + } public DictionaryDimEnc(Dictionary<String> dict) { this(dict, 0, NULL); @@ -145,12 +148,18 @@ public class DictionaryDimEnc extends DimensionEncoding { @Override public void writeExternal(ObjectOutput out) throws IOException { - throw new UnsupportedOperationException(); + out.writeInt(fixedLen); + out.writeInt(roundingFlag); + out.write(defaultByte); + out.writeObject(dict); } @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - throw new UnsupportedOperationException(); + this.fixedLen = in.readInt(); + this.roundingFlag = in.readInt(); + this.defaultByte = in.readByte(); + this.dict = (Dictionary<String>) in.readObject(); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java index b219766..f7f02a0 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java @@ -129,11 +129,9 @@ public class FixedLenDimEnc extends DimensionEncoding { } public class FixedLenSerializer extends DataTypeSerializer<Object> { - // be thread-safe and avoid repeated obj creation - private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); private byte[] currentBuf() { - byte[] buf = current.get(); + byte[] buf = (byte[]) current.get(); if (buf == null) { buf = new byte[fixedLen]; current.set(buf); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java index 83118fc..f90a40e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java @@ -224,11 +224,9 @@ public class FixedLenHexDimEnc extends DimensionEncoding { } public class FixedLenSerializer extends DataTypeSerializer<Object> { - // be thread-safe and avoid repeated obj creation - private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); private byte[] currentBuf() { - byte[] buf = current.get(); + byte[] buf = (byte[]) current.get(); if (buf == null) { buf = new byte[bytelen]; current.set(buf); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java index f25f2a6..aa954da 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java @@ -109,11 +109,9 @@ public class IntDimEnc extends DimensionEncoding { } public class IntegerSerializer extends DataTypeSerializer<Object> { - // be thread-safe and avoid repeated obj creation - private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); private byte[] currentBuf() { - byte[] buf = current.get(); + byte[] buf = (byte[]) current.get(); if (buf == null) { buf = new byte[fixedLen]; current.set(buf); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java index 090dc83..0875a7f 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java @@ -135,11 +135,9 @@ public class IntegerDimEnc extends DimensionEncoding { } public class IntegerSerializer extends DataTypeSerializer<Object> { - // be thread-safe and avoid repeated obj creation - private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); private byte[] currentBuf() { - byte[] buf = current.get(); + byte[] buf = (byte[]) current.get(); if (buf == null) { buf = new byte[fixedLen]; current.set(buf); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java index 993aac3..c0c52d1 100644 --- a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java +++ b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java @@ -18,17 +18,17 @@ package org.apache.kylin.dimension; +import org.apache.kylin.common.util.BytesUtil; +import org.apache.kylin.metadata.datatype.DataTypeSerializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; import java.nio.ByteBuffer; import java.util.Arrays; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.metadata.datatype.DataTypeSerializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * not being used yet, prepared for future */ @@ -118,11 +118,9 @@ public class OneMoreByteVLongDimEnc extends DimensionEncoding { } public class VLongSerializer extends DataTypeSerializer<Object> { - // be thread-safe and avoid repeated obj creation - private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>(); private byte[] currentBuf() { - byte[] buf = current.get(); + byte[] buf = (byte[]) current.get(); if (buf == null) { buf = new byte[byteLen]; current.set(buf); http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java b/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java index 8a5481c..44e5708 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/BufferedMeasureCodec.java @@ -18,26 +18,26 @@ package org.apache.kylin.measure; +import org.apache.kylin.metadata.datatype.DataType; +import org.apache.kylin.metadata.model.MeasureDesc; + import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.util.Collection; -import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.model.MeasureDesc; - /** * This class embeds a reusable byte buffer for measure encoding, and is not thread-safe. * The buffer will grow to accommodate BufferOverflowException until a limit. * The problem here to solve is some measure type cannot provide accurate DataTypeSerializer.maxLength() */ @SuppressWarnings({ "unchecked" }) -public class BufferedMeasureCodec { +public class BufferedMeasureCodec implements java.io.Serializable { public static final int DEFAULT_BUFFER_SIZE = 1024 * 1024; // 1 MB public static final int MAX_BUFFER_SIZE = 1 * 1024 * DEFAULT_BUFFER_SIZE; // 1 GB final private MeasureCodec codec; - private ByteBuffer buf; + private transient ByteBuffer buf; final private int[] measureSizes; public BufferedMeasureCodec(Collection<MeasureDesc> measureDescs) { http://git-wip-us.apache.org/repos/asf/kylin/blob/5a8bc054/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java index 7b74225..929eff0 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureAggregator.java @@ -71,6 +71,8 @@ abstract public class MeasureAggregator<V> implements Serializable { abstract public void aggregate(V value); + abstract public V aggregate(V value1, V value2); + abstract public V getState(); // get an estimate of memory consumption UPPER BOUND
