Repository: kylin Updated Branches: refs/heads/KYLIN-2428 e911ef473 -> d7b2b8b25
KYLIN-2428 update kyro mapping Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d7b2b8b2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d7b2b8b2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d7b2b8b2 Branch: refs/heads/KYLIN-2428 Commit: d7b2b8b2574f8516e5176363630f1854c9e7bca4 Parents: e911ef4 Author: Billy Liu <billy...@apache.org> Authored: Mon Feb 13 13:41:40 2017 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Mon Feb 13 13:41:40 2017 +0800 ---------------------------------------------------------------------- .../engine/spark/KylinKryoRegistrator.java | 41 ++++++++++---------- .../spark/SparkBatchCubingJobBuilder2.java | 1 + 2 files changed, 22 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d7b2b8b2/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java index f5f4f36..71c08f4 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java @@ -80,35 +80,43 @@ public class KylinKryoRegistrator implements KryoRegistrator { kyroClasses.add(java.util.Random.class); kyroClasses.add(java.util.concurrent.atomic.AtomicLong.class); + kyroClasses.add(org.apache.kylin.common.util.Array.class); + kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class); + kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class); kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc[].class); kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc[].class); kyroClasses.add(org.apache.kylin.metadata.model.TblColRef[].class); + kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc[].class); kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class); kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class); kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class); kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.PartitionType.class); - kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class); - kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class); - kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc[].class); - kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc[].class); - kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc[].class); - kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc[].class); - kyroClasses.add(org.apache.kylin.common.util.Array.class); kyroClasses.add(org.apache.kylin.metadata.model.Segments.class); kyroClasses.add(org.apache.kylin.metadata.realization.RealizationStatusEnum.class); kyroClasses.add(org.apache.kylin.metadata.model.SegmentStatusEnum.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer[].class); kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class); - kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class); kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class); kyroClasses.add(org.apache.kylin.measure.MeasureAggregator[].class); - kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer[].class); - kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class); kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class); - kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class); - kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class); + kyroClasses.add(org.apache.kylin.measure.topn.Counter.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class); + kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class); + kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc[].class); + kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc[].class); + kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc[].class); + kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class); + kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class); + kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class); kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class); kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class); - kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class); kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.class); kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.class); kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.class); @@ -118,17 +126,10 @@ public class KylinKryoRegistrator implements KryoRegistrator { kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class); kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class); kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class); - kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class); - kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class); - kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class); kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringArray.class); kyroClasses.add(org.roaringbitmap.buffer.MappeableContainer[].class); kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringBitmap.class); kyroClasses.add(org.roaringbitmap.buffer.MappeableArrayContainer.class); - kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class); - kyroClasses.add(org.apache.kylin.measure.topn.Counter.class); - kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class); - kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class); kyroClasses.add(com.tdunning.math.stats.AVLTreeDigest.class); kyroClasses.add(com.tdunning.math.stats.Centroid.class); http://git-wip-us.apache.org/repos/asf/kylin/blob/d7b2b8b2/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java index 327d215..00b9378 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java @@ -60,6 +60,7 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 { StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.HBaseConfiguration")); // hbase-common.jar StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.util.ByteStringer")); // hbase-protocol.jar StringUtil.appendWithSeparator(jars, findJar("com.yammer.metrics.core.Gauge")); // metrics-core.jar + StringUtil.appendWithSeparator(jars, findJar("com.google.common.collect.Maps")); //guava.jar StringUtil.appendWithSeparator(jars, seg.getConfig().getSparkAdditionalJars()); sparkExecutable.setJars(jars.toString());
