Repository: kylin Updated Branches: refs/heads/KYLIN-2428 900436e36 -> 62463b741
KYLIN-2428 Generate kyro mapping by Reflection Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/62463b74 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/62463b74 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/62463b74 Branch: refs/heads/KYLIN-2428 Commit: 62463b7413f34718ee29005c414b0b86bb751d8e Parents: 900436e Author: Billy Liu <billy...@apache.org> Authored: Tue Feb 14 07:33:13 2017 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Tue Feb 14 07:33:13 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/metadata/model/JoinsTree.java | 2 +- engine-spark/pom.xml | 7 + .../engine/spark/KylinKryoRegistrator.java | 187 ++++++++++++++++++- .../engine/spark/util/KyroMappingGenerator.java | 51 +++++ 4 files changed, 245 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/62463b74/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java index 3ab9a46..c7666cb 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java @@ -113,7 +113,7 @@ public class JoinsTree implements Serializable { return chain.join; } - static class Chain implements Serializable { + public static class Chain implements Serializable { private static final long serialVersionUID = 1L; TableRef table; // pk side http://git-wip-us.apache.org/repos/asf/kylin/blob/62463b74/engine-spark/pom.xml ---------------------------------------------------------------------- diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index 258232d..7a478f3 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -111,6 +111,13 @@ <artifactId>maven-model</artifactId> <scope>test</scope> </dependency> + + <dependency> + <groupId>org.reflections</groupId> + <artifactId>reflections</artifactId> + <version>0.9.10</version> + <scope>test</scope> + </dependency> </dependencies> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/62463b74/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java index c99ee6e..d9bfe99 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java @@ -61,7 +61,7 @@ public class KylinKryoRegistrator implements KryoRegistrator { kyroClasses.add(org.apache.spark.sql.types.DateType$.class); kyroClasses.add(org.apache.spark.sql.types.Metadata.class); kyroClasses.add(org.apache.spark.sql.types.StringType$.class); -// kyroClasses.add(Hashing.murmur3_128().getClass()); + // kyroClasses.add(Hashing.murmur3_128().getClass()); kyroClasses.add(org.apache.spark.sql.execution.columnar.CachedBatch.class); kyroClasses.add(org.apache.spark.sql.types.Decimal.class); kyroClasses.add(scala.math.BigDecimal.class); @@ -146,6 +146,9 @@ public class KylinKryoRegistrator implements KryoRegistrator { kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class); kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class); kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class); + kylinClassByReflection1(kyroClasses); + kylinClassByReflection2(kyroClasses); + kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringArray.class); kyroClasses.add(org.roaringbitmap.buffer.MappeableContainer[].class); kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringBitmap.class); @@ -168,6 +171,188 @@ public class KylinKryoRegistrator implements KryoRegistrator { // kryo.register(PercentileCounter.class, new JavaSerializer()); } + /** + * Generated by KyroMappingGenerator, method exceeds 150 lines, splits into two methods + * @param kyroClasses + */ + private void kylinClassByReflection1(Set<Class> kyroClasses) { + kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc[].class); + kyroClasses.add(org.apache.kylin.cube.model.RowKeyDesc.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMaxAggregator.class); + kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.class); + kyroClasses.add(org.apache.kylin.source.ReadableTable.TableSignature.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class); + kyroClasses.add(org.apache.kylin.metadata.model.ParameterDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc.class); + kyroClasses.add(org.apache.kylin.measure.raw.RawSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.LongMutable.class); + kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.FixedLenSerializer.class); + kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.class); + kyroClasses.add(org.apache.kylin.dict.TrieDictionary.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.Int4Serializer.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class); + kyroClasses.add(org.apache.kylin.measure.hllc.DenseRegister.class); + kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class); + kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnSerializer.class); + kyroClasses.add(org.apache.kylin.dict.lookup.SnapshotTable.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.MeasureType.class); + kyroClasses.add(org.apache.kylin.cube.gridtable.TrimmedDimensionSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.project.RealizationEntry.class); + kyroClasses.add(org.apache.kylin.cube.kv.AbstractRowKeyEncoder.class); + kyroClasses.add(org.apache.kylin.common.KylinConfigBase.class); + kyroClasses.add(org.apache.kylin.cube.model.v1_4_0.CubeDesc.class); + kyroClasses.add(org.apache.kylin.cube.kv.FuzzyKeyEncoder.class); + kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class); + kyroClasses.add(org.apache.kylin.cube.model.DimensionDesc.class); + kyroClasses.add(org.apache.kylin.measure.raw.RawAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.DoubleSumAggregator.class); + kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.IntegerSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.model.ModelDimensionDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.SelectRule.class); + kyroClasses.add(org.apache.kylin.metadata.model.DatabaseDesc.class); + kyroClasses.add(org.apache.kylin.common.util.Array.class); + kyroClasses.add(org.apache.kylin.engine.mr.common.NDCuboidBuilder.class); + kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryHistory.class); + kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounterOld.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongMinAggregator.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileCounter.class); + kyroClasses.add(org.apache.kylin.metadata.project.ProjectInstance.class); + kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileAggregator.class); + kyroClasses.add(org.apache.kylin.common.util.ByteArray.class); + kyroClasses.add(org.apache.kylin.metadata.model.FunctionDesc.class); + kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounter.class); + kyroClasses.add(org.apache.kylin.cube.kv.LazyRowKeyEncoder.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCAggregator.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapMeasureType.class); + kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCSerializer.class); + kyroClasses.add(org.apache.kylin.cube.cuboid.CuboidScheduler.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapAggregator.class); + kyroClasses.add(org.apache.kylin.job.dao.ExecutablePO.class); + kyroClasses.add(org.apache.kylin.measure.raw.RawMeasureType.class); + kyroClasses.add(org.apache.kylin.common.util.OrderedProperties.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapSerializer.class); + kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class); + kyroClasses.add(org.apache.kylin.cube.cuboid.Cuboid.class); + kyroClasses.add(org.apache.kylin.measure.MeasureAggregator.class); + kyroClasses.add(org.apache.kylin.cube.model.DictionaryDesc.class); + kyroClasses.add(org.apache.kylin.measure.dim.DimCountDistinctMeasureType.class); + kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMinAggregator.class); + kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.ColumnStats.class); + kyroClasses.add(org.apache.kylin.metadata.streaming.StreamingConfig.class); + kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.class); + kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.IntegerSerializer.class); + kyroClasses.add(org.apache.kylin.common.KylinConfig.class); + kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class); + kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.class); + kyroClasses.add(org.apache.kylin.cube.CubeInstance.class); + } + + /** + * Generated by KyroMappingGenerator + * @param kyroClasses + */ + private void kylinClassByReflection2(Set<Class> kyroClasses) { + kyroClasses.add(org.apache.kylin.measure.basic.DoubleMaxAggregator.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DateTimeSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.BigDecimalSerializer.class); + kyroClasses.add(org.apache.kylin.dict.NumberDictionary2.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.IntMutable.class); + kyroClasses.add(org.apache.kylin.storage.hybrid.HybridInstance.class); + kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.class); + kyroClasses.add(org.apache.kylin.cube.CubeSegment.class); + kyroClasses.add(org.apache.kylin.common.util.ByteArray.ImmutableByteArray.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleMutable.class); + kyroClasses.add(org.apache.kylin.common.util.Pair.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNAggregator.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DataType.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongSumAggregator.class); + kyroClasses.add(org.apache.kylin.measure.hllc.SparseRegister.class); + kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSlice.class); + kyroClasses.add(org.apache.kylin.dict.TrieDictionaryForest.class); + kyroClasses.add(org.apache.kylin.measure.basic.DoubleMinAggregator.class); + kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.class); + kyroClasses.add(org.apache.kylin.measure.MeasureAggregators.class); + kyroClasses.add(org.apache.kylin.dict.CacheDictionary.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.Long8Serializer.class); + kyroClasses.add(org.apache.kylin.dict.NumberDictionaryForestBuilder.Number2BytesConverter.class); + kyroClasses.add(org.apache.kylin.metadata.model.TableRef.class); + kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryEntry.class); + kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class); + kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType.class); + kyroClasses.add(org.apache.kylin.common.KylinConfigExt.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class); + kyroClasses.add(org.apache.kylin.common.util.Dictionary.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNCounterSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.class); + kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.Chain.class); + kyroClasses.add(org.apache.kylin.cube.model.HBaseMappingDesc.class); + kyroClasses.add(org.apache.kylin.job.dao.ExecutableOutputPO.class); + kyroClasses.add(org.apache.kylin.job.JobInstance.class); + kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.FixedLenSerializer.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNMeasureType.class); + kyroClasses.add(org.apache.kylin.metadata.model.JoinDesc.class); + kyroClasses.add(org.apache.kylin.dict.NumberDictionary.class); + kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.VLongSerializer.class); + kyroClasses.add(org.apache.kylin.dict.TimeStrDictionary.class); + kyroClasses.add(org.apache.kylin.measure.hllc.SingleValueRegister.class); + kyroClasses.add(org.apache.kylin.dict.StringBytesConverter.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCMeasureType.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleSerializer.class); + kyroClasses.add(org.apache.kylin.engine.spark.SparkCubingByLayer.class); + kyroClasses.add(org.apache.kylin.cube.common.RowKeySplitter.class); + kyroClasses.add(org.apache.kylin.measure.basic.DoubleIngester.class); + kyroClasses.add(org.apache.kylin.cube.kv.FuzzyMaskEncoder.class); + kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.DictionarySerializer.class); + kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer.class); + kyroClasses.add(org.apache.kylin.dict.DictionaryInfo.class); + kyroClasses.add(org.apache.kylin.metadata.model.TableDesc.class); + kyroClasses.add(org.apache.kylin.engine.mr.common.BaseCuboidBuilder.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.LongSerializer.class); + kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class); + kyroClasses.add(org.apache.kylin.metadata.model.ExternalFilterDesc.class); + kyroClasses.add(org.apache.kylin.dict.DateStrDictionary.class); + kyroClasses.add(org.apache.kylin.measure.MeasureIngester.class); + kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictNode.class); + kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.BooleanSerializer.class); + kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSliceKey.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongMaxAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalSumAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongIngester.class); + kyroClasses.add(org.apache.kylin.common.persistence.RootPersistentEntity.class); + kyroClasses.add(org.apache.kylin.metadata.model.TblColRef.class); + kyroClasses.add(org.apache.kylin.cube.model.HierarchyDesc.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HyperLogLogPlusTable.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.StringSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.LongSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.Long8Serializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.BooleanSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DateTimeSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.BigDecimalSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.Int4Serializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.StringSerializer.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class); + kyroClasses.add(org.apache.kylin.measure.basic.DoubleIngester.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongIngester.class); + } + private static void addClassQuitely(Set<Class> kyroClasses, String className) { try { kyroClasses.add(Class.forName(className)); http://git-wip-us.apache.org/repos/asf/kylin/blob/62463b74/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java b/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java new file mode 100644 index 0000000..b4960e9 --- /dev/null +++ b/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.engine.spark.util; + +import org.apache.kylin.common.util.BytesSerializer; +import org.apache.kylin.measure.MeasureIngester; +import org.reflections.Reflections; + +import java.io.Serializable; +import java.util.Set; + +/** + * Generate Kyro Registrator class + */ +public class KyroMappingGenerator { + public static void main(String[] args) { + Set<Class<? extends Serializable>> subTypesOfSerializable = new Reflections("org.apache.kylin").getSubTypesOf(Serializable.class); + String begin = "kyroClasses.add("; + String end = ".class);"; + for (Class clazz : subTypesOfSerializable) { + if (clazz.getCanonicalName() != null) + System.out.println(begin + clazz.getCanonicalName() + end); + } + Set<Class<? extends BytesSerializer>> subTypesOfBytes = new Reflections("org.apache.kylin.metadata.datatype").getSubTypesOf(BytesSerializer.class); + for (Class clazz : subTypesOfBytes) { + if (clazz.getCanonicalName() != null) + System.out.println(begin + clazz.getCanonicalName() + end); + } + Set<Class<? extends MeasureIngester>> subTypesOfMeasure = new Reflections("org.apache.kylin.measure").getSubTypesOf(MeasureIngester.class); + for (Class clazz : subTypesOfMeasure) { + if (clazz.getCanonicalName() != null) + System.out.println(begin + clazz.getCanonicalName() + end); + } + } +}
