http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java index 3d33aa8..20b39ee 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java @@ -19,15 +19,11 @@ package org.apache.kylin.engine.spark; import com.esotericsoftware.kryo.Kryo; -import com.google.common.collect.Sets; -import com.google.common.hash.Hashing; -import org.apache.kylin.measure.MeasureIngester; import org.apache.spark.serializer.KryoRegistrator; -import org.reflections.Reflections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Serializable; +import java.util.LinkedHashSet; import java.util.Set; /** @@ -39,7 +35,7 @@ public class KylinKryoRegistrator implements KryoRegistrator { @Override public void registerClasses(Kryo kryo) { - Set<Class> kyroClasses = Sets.newLinkedHashSet(); + Set<Class> kyroClasses = new LinkedHashSet(); kyroClasses.add(byte[].class); kyroClasses.add(int[].class); kyroClasses.add(byte[][].class); @@ -55,15 +51,11 @@ public class KylinKryoRegistrator implements KryoRegistrator { kyroClasses.add(java.util.HashMap.class); kyroClasses.add(java.util.TreeMap.class); kyroClasses.add(java.util.Properties.class); - kyroClasses.addAll(new Reflections("org.apache.kylin").getSubTypesOf(Serializable.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.dimension").getSubTypesOf(Serializable.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.cube").getSubTypesOf(Serializable.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.cube.model").getSubTypesOf(Object.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.metadata").getSubTypesOf(Object.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.model").getSubTypesOf(Object.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.measure").getSubTypesOf(Object.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.datatype").getSubTypesOf(org.apache.kylin.common.util.BytesSerializer.class)); - kyroClasses.addAll(new Reflections("org.apache.kylin.measure").getSubTypesOf(MeasureIngester.class)); + kyroClasses.add(java.math.MathContext.class); + kyroClasses.add(java.math.RoundingMode.class); + kyroClasses.add(java.util.concurrent.ConcurrentHashMap.class); + kyroClasses.add(java.util.Random.class); + kyroClasses.add(java.util.concurrent.atomic.AtomicLong.class); kyroClasses.add(org.apache.spark.sql.Row[].class); kyroClasses.add(org.apache.spark.sql.Row.class); @@ -74,65 +66,29 @@ public class KylinKryoRegistrator implements KryoRegistrator { kyroClasses.add(org.apache.spark.sql.types.DateType$.class); kyroClasses.add(org.apache.spark.sql.types.Metadata.class); kyroClasses.add(org.apache.spark.sql.types.StringType$.class); - kyroClasses.add(Hashing.murmur3_128().getClass()); kyroClasses.add(org.apache.spark.sql.execution.columnar.CachedBatch.class); kyroClasses.add(org.apache.spark.sql.types.Decimal.class); kyroClasses.add(scala.math.BigDecimal.class); - kyroClasses.add(java.math.MathContext.class); - kyroClasses.add(java.math.RoundingMode.class); - kyroClasses.add(java.util.concurrent.ConcurrentHashMap.class); - kyroClasses.add(java.util.Random.class); - kyroClasses.add(java.util.concurrent.atomic.AtomicLong.class); + kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class); kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc[].class); kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc[].class); kyroClasses.add(org.apache.kylin.metadata.model.TblColRef[].class); - kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class); - kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class); - kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class); - kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.PartitionType.class); - kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class); - kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class); + kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc[].class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer[].class); + kyroClasses.add(org.apache.kylin.measure.MeasureAggregator[].class); kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc[].class); kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc[].class); - kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc[].class); kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc[].class); - kyroClasses.add(org.apache.kylin.common.util.Array.class); - kyroClasses.add(org.apache.kylin.metadata.model.Segments.class); - kyroClasses.add(org.apache.kylin.metadata.realization.RealizationStatusEnum.class); - kyroClasses.add(org.apache.kylin.metadata.model.SegmentStatusEnum.class); - kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class); - kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class); - kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class); - kyroClasses.add(org.apache.kylin.measure.MeasureAggregator[].class); - kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer[].class); - kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class); - kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class); - kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class); - kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class); - kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class); - kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class); - kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class); - kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.DateDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class); - kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class); - kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class); - kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class); - kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class); + + kylinClassByReflection1(kyroClasses); + kylinClassByReflection2(kyroClasses); + + kyroClasses.add(com.google.common.hash.Hashing.murmur3_128().getClass()); kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringArray.class); kyroClasses.add(org.roaringbitmap.buffer.MappeableContainer[].class); kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringBitmap.class); kyroClasses.add(org.roaringbitmap.buffer.MappeableArrayContainer.class); - kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class); - kyroClasses.add(org.apache.kylin.measure.topn.Counter.class); - kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class); - kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class); kyroClasses.add(com.tdunning.math.stats.AVLTreeDigest.class); kyroClasses.add(com.tdunning.math.stats.Centroid.class); @@ -151,6 +107,195 @@ public class KylinKryoRegistrator implements KryoRegistrator { // kryo.register(PercentileCounter.class, new JavaSerializer()); } + /** + * Generated by KyroMappingGenerator, method exceeds 150 lines, splits into two methods + * @param kyroClasses + */ + private void kylinClassByReflection1(Set<Class> kyroClasses) { + kyroClasses.add(org.apache.kylin.common.KylinConfig.class); + kyroClasses.add(org.apache.kylin.common.KylinConfigBase.class); + kyroClasses.add(org.apache.kylin.common.KylinConfigExt.class); + kyroClasses.add(org.apache.kylin.common.persistence.RootPersistentEntity.class); + kyroClasses.add(org.apache.kylin.common.util.Array.class); + kyroClasses.add(org.apache.kylin.common.util.ByteArray.class); + kyroClasses.add(org.apache.kylin.common.util.ByteArray.ImmutableByteArray.class); + kyroClasses.add(org.apache.kylin.common.util.Dictionary.class); + kyroClasses.add(org.apache.kylin.common.util.OrderedProperties.class); + kyroClasses.add(org.apache.kylin.common.util.Pair.class); + kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class); + kyroClasses.add(org.apache.kylin.cube.CubeInstance.class); + kyroClasses.add(org.apache.kylin.cube.CubeSegment.class); + kyroClasses.add(org.apache.kylin.cube.common.RowKeySplitter.class); + kyroClasses.add(org.apache.kylin.cube.cuboid.Cuboid.class); + kyroClasses.add(org.apache.kylin.cube.cuboid.CuboidScheduler.class); + kyroClasses.add(org.apache.kylin.cube.gridtable.TrimmedDimensionSerializer.class); + kyroClasses.add(org.apache.kylin.cube.kv.AbstractRowKeyEncoder.class); + kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class); + kyroClasses.add(org.apache.kylin.cube.kv.FuzzyKeyEncoder.class); + kyroClasses.add(org.apache.kylin.cube.kv.FuzzyMaskEncoder.class); + kyroClasses.add(org.apache.kylin.cube.kv.LazyRowKeyEncoder.class); + kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class); + kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class); + kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class); + kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.class); + kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich.class); + kyroClasses.add(org.apache.kylin.cube.model.DictionaryDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.DimensionDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.HBaseMappingDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.HierarchyDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.RowKeyDesc.class); + kyroClasses.add(org.apache.kylin.cube.model.SelectRule.class); + kyroClasses.add(org.apache.kylin.cube.model.v1_4_0.CubeDesc.class); + kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.class); + kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictNode.class); + kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSlice.class); + kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSliceKey.class); + kyroClasses.add(org.apache.kylin.dict.CacheDictionary.class); + kyroClasses.add(org.apache.kylin.dict.DateStrDictionary.class); + kyroClasses.add(org.apache.kylin.dict.DictionaryInfo.class); + kyroClasses.add(org.apache.kylin.dict.NumberDictionary.class); + kyroClasses.add(org.apache.kylin.dict.NumberDictionary2.class); + kyroClasses.add(org.apache.kylin.dict.NumberDictionaryForestBuilder.Number2BytesConverter.class); + kyroClasses.add(org.apache.kylin.dict.StringBytesConverter.class); + kyroClasses.add(org.apache.kylin.dict.TimeStrDictionary.class); + kyroClasses.add(org.apache.kylin.dict.TrieDictionary.class); + kyroClasses.add(org.apache.kylin.dict.TrieDictionaryForest.class); + kyroClasses.add(org.apache.kylin.dict.lookup.SnapshotTable.class); + kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.BooleanSerializer.class); + kyroClasses.add(org.apache.kylin.dimension.DateDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.DictionarySerializer.class); + kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.FixedLenSerializer.class); + kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.FixedLenSerializer.class); + kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.IntegerSerializer.class); + kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.IntegerSerializer.class); + kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class); + kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.VLongSerializer.class); + kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class); + kyroClasses.add(org.apache.kylin.engine.mr.common.BaseCuboidBuilder.class); + kyroClasses.add(org.apache.kylin.engine.mr.common.NDCuboidBuilder.class); + kyroClasses.add(org.apache.kylin.engine.spark.SparkCubingByLayer.class); + kyroClasses.add(org.apache.kylin.job.JobInstance.class); + kyroClasses.add(org.apache.kylin.job.dao.ExecutableOutputPO.class); + kyroClasses.add(org.apache.kylin.job.dao.ExecutablePO.class); + } + + /** + * Generated by KyroMappingGenerator + * @param kyroClasses + */ + private void kylinClassByReflection2(Set<Class> kyroClasses) { + kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class); + kyroClasses.add(org.apache.kylin.measure.MeasureAggregator.class); + kyroClasses.add(org.apache.kylin.measure.MeasureAggregators.class); + kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class); + kyroClasses.add(org.apache.kylin.measure.MeasureIngester.class); + kyroClasses.add(org.apache.kylin.measure.MeasureType.class); + kyroClasses.add(org.apache.kylin.measure.MeasureTypeFactory.NeedRewriteOnlyMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMaxAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMinAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalSumAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.DoubleIngester.class); + kyroClasses.add(org.apache.kylin.measure.basic.DoubleMaxAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.DoubleMinAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.DoubleSumAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongIngester.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongMaxAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongMinAggregator.class); + kyroClasses.add(org.apache.kylin.measure.basic.LongSumAggregator.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapAggregator.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapSerializer.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class); + kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class); + kyroClasses.add(org.apache.kylin.measure.dim.DimCountDistinctMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnSerializer.class); + kyroClasses.add(org.apache.kylin.measure.hllc.DenseRegister.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCAggregator.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCSerializer.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounter.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounterOld.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HLLDistinctCountAggFunc.FixedValueHLLCMockup.class); + kyroClasses.add(org.apache.kylin.measure.hllc.HyperLogLogPlusTable.class); + kyroClasses.add(org.apache.kylin.measure.hllc.SingleValueRegister.class); + kyroClasses.add(org.apache.kylin.measure.hllc.SparseRegister.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileAggregator.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileCounter.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class); + kyroClasses.add(org.apache.kylin.measure.raw.RawAggregator.class); + kyroClasses.add(org.apache.kylin.measure.raw.RawMeasureType.class); + kyroClasses.add(org.apache.kylin.measure.raw.RawSerializer.class); + kyroClasses.add(org.apache.kylin.measure.topn.Counter.class); + kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNAggregator.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNCounterSerializer.class); + kyroClasses.add(org.apache.kylin.measure.topn.TopNMeasureType.class); + kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryEntry.class); + kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryHistory.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.BigDecimalSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.BooleanSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DataType.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DateTimeSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleMutable.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.Int4Serializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.IntMutable.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.Long8Serializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.LongMutable.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.LongSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.datatype.StringSerializer.class); + kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class); + kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class); + kyroClasses.add(org.apache.kylin.metadata.model.DatabaseDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.ExternalFilterDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.FunctionDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.JoinDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.class); + kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.Chain.class); + kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.ModelDimensionDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.ParameterDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.PartitionType.class); + kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class); + kyroClasses.add(org.apache.kylin.metadata.model.Segments.class); + kyroClasses.add(org.apache.kylin.metadata.model.SegmentStatusEnum.class); + kyroClasses.add(org.apache.kylin.metadata.model.TableDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.class); + kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.ColumnStats.class); + kyroClasses.add(org.apache.kylin.metadata.model.TableRef.class); + kyroClasses.add(org.apache.kylin.metadata.model.TblColRef.class); + kyroClasses.add(org.apache.kylin.metadata.project.ProjectInstance.class); + kyroClasses.add(org.apache.kylin.metadata.project.RealizationEntry.class); + kyroClasses.add(org.apache.kylin.metadata.realization.RealizationStatusEnum.class); + kyroClasses.add(org.apache.kylin.metadata.streaming.StreamingConfig.class); + kyroClasses.add(org.apache.kylin.source.ReadableTable.TableSignature.class); + kyroClasses.add(org.apache.kylin.storage.hybrid.HybridInstance.class); + } + private static void addClassQuitely(Set<Class> kyroClasses, String className) { try { kyroClasses.add(Class.forName(className));
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java index 327d215..f68194e 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java @@ -18,8 +18,8 @@ package org.apache.kylin.engine.spark; -import org.apache.hadoop.util.ClassUtil; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.ClassUtil; import org.apache.kylin.common.util.StringUtil; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.engine.EngineFactory; @@ -53,13 +53,14 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 { StringBuilder jars = new StringBuilder(); - StringUtil.appendWithSeparator(jars, findJar("org.htrace.HTraceConfiguration")); // htrace-core.jar - StringUtil.appendWithSeparator(jars, findJar("org.apache.htrace.Trace")); // htrace-core.jar - StringUtil.appendWithSeparator(jars, findJar("org.cloudera.htrace.HTraceConfiguration")); // htrace-core.jar - StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.client.HConnection")); // hbase-client.jar - StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.HBaseConfiguration")); // hbase-common.jar - StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.util.ByteStringer")); // hbase-protocol.jar - StringUtil.appendWithSeparator(jars, findJar("com.yammer.metrics.core.Gauge")); // metrics-core.jar + StringUtil.appendWithSeparator(jars, findJar("org.htrace.HTraceConfiguration", null)); // htrace-core.jar + StringUtil.appendWithSeparator(jars, findJar("org.apache.htrace.Trace", null)); // htrace-core.jar + StringUtil.appendWithSeparator(jars, findJar("org.cloudera.htrace.HTraceConfiguration", null)); // htrace-core.jar + StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.client.HConnection", null)); // hbase-client.jar + StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.HBaseConfiguration", null)); // hbase-common.jar + StringUtil.appendWithSeparator(jars, findJar("org.apache.hadoop.hbase.util.ByteStringer", null)); // hbase-protocol.jar + StringUtil.appendWithSeparator(jars, findJar("com.yammer.metrics.core.Gauge", null)); // metrics-core.jar + StringUtil.appendWithSeparator(jars, findJar("com.google.common.collect.Maps", "guava")); //guava.jar StringUtil.appendWithSeparator(jars, seg.getConfig().getSparkAdditionalJars()); sparkExecutable.setJars(jars.toString()); @@ -73,9 +74,9 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 { } - private String findJar(String className) { + private String findJar(String className, String perferLibraryName) { try { - return ClassUtil.findContainingJar(Class.forName(className)); + return ClassUtil.findContainingJar(Class.forName(className), perferLibraryName); } catch (ClassNotFoundException e) { logger.warn("failed to locate jar for class " + className + ", ignore it", e); } http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java index c7422ed..f7ed2d0 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java @@ -17,7 +17,6 @@ */ package org.apache.kylin.engine.spark; -import com.google.common.collect.Lists; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; @@ -72,6 +71,7 @@ import java.io.File; import java.io.FileFilter; import java.io.Serializable; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -333,7 +333,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa } } - private static final java.lang.Iterable<Tuple2<ByteArray, Object[]>> EMTPY_ITERATOR = Lists.newArrayListWithCapacity(0); + private static final java.lang.Iterable<Tuple2<ByteArray, Object[]>> EMTPY_ITERATOR = new ArrayList(0); class CuboidFlatMap implements PairFlatMapFunction<Tuple2<ByteArray, Object[]>, ByteArray, Object[]> { @@ -370,7 +370,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa return EMTPY_ITERATOR; } - List<Tuple2<ByteArray, Object[]>> tuples = Lists.newArrayListWithCapacity(myChildren.size()); + List<Tuple2<ByteArray, Object[]>> tuples = new ArrayList(myChildren.size()); for (Long child : myChildren) { Cuboid childCuboid = Cuboid.findById(cubeDesc, child); Pair<Integer, ByteArray> result = ndCuboidBuilder.buildKey(parentCuboid, childCuboid, rowKeySplitter.getSplitBuffers()); http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java index 733a472..015b480 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java @@ -20,8 +20,7 @@ package org.apache.kylin.engine.spark; import java.io.File; import java.util.Map; -import jodd.util.StringUtil; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.CliCommandExecutor; import org.apache.kylin.common.util.Logger; @@ -31,8 +30,6 @@ import org.apache.kylin.job.execution.ExecutableContext; import org.apache.kylin.job.execution.ExecuteResult; import org.slf4j.LoggerFactory; -import com.google.common.base.Preconditions; - /** */ public class SparkExecutable extends AbstractExecutable { @@ -74,12 +71,16 @@ public class SparkExecutable extends AbstractExecutable { @Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final KylinConfig config = context.getConfig(); - Preconditions.checkNotNull(config.getSparkHome()); - Preconditions.checkNotNull(config.getKylinJobJarPath()); + if (config.getSparkHome() == null) { + throw new NullPointerException(); + } + if (config.getKylinJobJarPath() == null) { + throw new NullPointerException(); + } String jars = this.getParam(JARS); String hadoopConf = "/etc/hadoop/conf"; - if (StringUtil.isNotEmpty(config.getHadoopConfDir())) { + if (StringUtils.isNotEmpty(config.getHadoopConfDir())) { hadoopConf = config.getHadoopConfDir(); } else { String hiveConf = ClassLoader.getSystemClassLoader().getResource("hive-site.xml").getFile().toString(); @@ -104,7 +105,7 @@ public class SparkExecutable extends AbstractExecutable { } StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit --class org.apache.kylin.common.util.SparkEntry "); + stringBuilder.append("export HADOOP_CONF_DIR=%s && %s/bin/spark-submit --class org.apache.kylin.engine.spark.util.SparkEntry "); Map<String, String> sparkConfs = config.getSparkConfigOverride(); for (Map.Entry<String, String> entry : sparkConfs.entrySet()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java index 531670f..a2740bf 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/cube/ListBackedCuboidWriter.java @@ -23,8 +23,6 @@ import java.util.ArrayList; import org.apache.kylin.engine.spark.SparkCuboidWriter; import org.apache.kylin.gridtable.GTRecord; -import com.google.common.collect.Lists; - import scala.Tuple2; /** @@ -35,7 +33,7 @@ public class ListBackedCuboidWriter implements SparkCuboidWriter { private final TupleConverter tupleConverter; public ListBackedCuboidWriter(TupleConverter tupleConverter) { - this.result = Lists.newArrayList(); + this.result = new ArrayList(); this.tupleConverter = tupleConverter; } http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java index 13464a0..a8a4d28 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/IteratorUtils.java @@ -24,8 +24,6 @@ import java.util.NoSuchElementException; import org.apache.spark.api.java.function.Function; -import com.google.common.collect.Lists; - import scala.Tuple2; /** @@ -47,7 +45,7 @@ public class IteratorUtils { if (!hasNext()) { throw new NoSuchElementException(); } - final LinkedList<V> values = Lists.newLinkedList(); + final LinkedList<V> values = new LinkedList(); K currentKey = current._1(); values.add(current._2()); while (input.hasNext()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java new file mode 100644 index 0000000..14788fc --- /dev/null +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/util/SparkEntry.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kylin.engine.spark.util; + +import org.apache.commons.lang.StringUtils; + +import org.apache.kylin.common.util.AbstractApplication; + +/** + */ +public final class SparkEntry { + + public static void main(String[] args) throws Exception { + System.out.println("SparkEntry args:" + StringUtils.join(args, " ")); + if (!(args.length >= 2)) { + throw new IllegalArgumentException(String.valueOf("-className is required")); + } + if (!(args[0].equals("-className"))) { + throw new IllegalArgumentException(String.valueOf("-className is required")); + } + final String className = args[1]; + final Object o = Class.<AbstractApplication> forName(className).newInstance(); + if (!(o instanceof AbstractApplication)) { + throw new IllegalArgumentException(String.valueOf(className + " is not a subClass of AbstractSparkApplication")); + } + String[] appArgs = new String[args.length - 2]; + for (int i = 2; i < args.length; i++) { + appArgs[i - 2] = args[i]; + } + AbstractApplication abstractApplication = (AbstractApplication) o; + abstractApplication.execute(appArgs); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java b/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java new file mode 100644 index 0000000..b181d33 --- /dev/null +++ b/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.engine.spark.util; + +import org.apache.kylin.common.util.BytesSerializer; +import org.apache.kylin.measure.MeasureIngester; +import org.reflections.Reflections; + +import java.io.Serializable; +import java.util.Set; +import java.util.TreeSet; + +/** + * Generate Kyro Registrator class, the output will be added into KylinKyroRegistrator manually. No runtime dependency with Reflections. + */ +public class KyroMappingGenerator { + public static void main(String[] args) { + Set<Class<? extends Serializable>> subTypesOfSerializable = new Reflections("org.apache.kylin").getSubTypesOf(Serializable.class); + String begin = "kyroClasses.add("; + String end = ".class);"; + TreeSet<String> sortedSet = new TreeSet(); + for (Class clazz : subTypesOfSerializable) { + if (clazz.getCanonicalName() != null) + sortedSet.add(clazz.getCanonicalName()); + } + Set<Class<? extends BytesSerializer>> subTypesOfBytes = new Reflections("org.apache.kylin.metadata.datatype").getSubTypesOf(BytesSerializer.class); + for (Class clazz : subTypesOfBytes) { + if (clazz.getCanonicalName() != null) + sortedSet.add(clazz.getCanonicalName()); + } + Set<Class<? extends MeasureIngester>> subTypesOfMeasure = new Reflections("org.apache.kylin.measure").getSubTypesOf(MeasureIngester.class); + for (Class clazz : subTypesOfMeasure) { + if (clazz.getCanonicalName() != null) + sortedSet.add(clazz.getCanonicalName()); + } + for (String className : sortedSet) { + System.out.println(begin + className + end); + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 874ead6..02890a6 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -43,7 +43,7 @@ </dependency> <dependency> <groupId>org.apache.calcite.avatica</groupId> - <artifactId>avatica</artifactId> + <artifactId>avatica-core</artifactId> </dependency> <dependency> <groupId>junit</groupId> @@ -69,9 +69,7 @@ <artifactSet> <excludes> - <exclude>com.google.protobuf:*</exclude> - <exclude>commons-logging:*</exclude> - <exclude>commons-codec:*</exclude> + <exclude>org.slf4j:jcl-over-slf4j:*</exclude> </excludes> </artifactSet> @@ -85,9 +83,17 @@ <shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern> </relocation> <relocation> + <pattern>com.google.protobuf</pattern> + <shadedPattern>${shadeBase}.com.google.protobuf</shadedPattern> + </relocation> + <relocation> <pattern>org.apache.http</pattern> <shadedPattern>${shadeBase}.org.apache.http</shadedPattern> </relocation> + <relocation> + <pattern>org.apache.commons</pattern> + <shadedPattern>${shadeBase}.org.apache.commons</shadedPattern> + </relocation> </relocations> <filters> <filter> http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b82eee2..a103ae0 100644 --- a/pom.xml +++ b/pom.xml @@ -55,15 +55,34 @@ <!-- HBase versions --> <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version> + + <!-- Kafka versions --> <kafka.version>0.10.1.0</kafka.version> - <!-- Hadoop deps, keep compatible with hadoop2.version --> + <!-- Spark versions --> + <spark.version>1.6.3</spark.version> + <kryo.version>4.0.0</kryo.version> + <!-- + <reflections.version>0.9.10</reflections.version> + --> + + <!-- Calcite Version --> + <calcite.version>1.11.0</calcite.version> + <avatica.version>1.9.0</avatica.version> + + <!-- Hadoop Common deps, keep compatible with hadoop2.version --> <zookeeper.version>3.4.6</zookeeper.version> <curator.version>2.7.1</curator.version> - <jackson.version>2.2.4</jackson.version> <jsr305.version>3.0.1</jsr305.version> <guava.version>14.0</guava.version> + <jsch.version>0.1.53</jsch.version> <commons-cli.version>1.2</commons-cli.version> + <commons-lang.version>2.6</commons-lang.version> + <commons-io.version>2.5</commons-io.version> + <commons-math3.version>3.6.1</commons-math3.version> + + <!-- Calcite deps, keep compatible with calcite.version --> + <jackson.version>2.6.3</jackson.version> <!-- Test Dependency versions --> <antlr.version>3.4</antlr.version> @@ -75,26 +94,16 @@ <jamm.version>0.3.1</jamm.version> <!-- Commons --> - <commons-lang.version>2.6</commons-lang.version> <commons-lang3.version>3.4</commons-lang3.version> <commons-collections.version>3.2.2</commons-collections.version> - <commons-io.version>2.5</commons-io.version> <commons-daemon.version>1.0.15</commons-daemon.version> <commons-email.version>1.4</commons-email.version> - <commons-math3.version>3.6.1</commons-math3.version> - - <!-- Spark --> - <spark.version>1.6.3</spark.version> <!-- Utility --> <log4j.version>1.2.17</log4j.version> <slf4j.version>1.7.21</slf4j.version> - <reflections.version>0.9.10</reflections.version> - <jsch.version>0.1.53</jsch.version> <xerces.version>2.11.0</xerces.version> <xalan.version>2.7.2</xalan.version> - <extendedset.version>1.3.4</extendedset.version> - <kryo.version>4.0.0</kryo.version> <ehcache.version>2.10.2.2.21</ehcache.version> <apache-httpclient.version>4.2.5</apache-httpclient.version> <roaring.version>0.6.18</roaring.version> @@ -111,10 +120,6 @@ <opensaml.version>2.6.4</opensaml.version> <aspectj.version>1.8.9</aspectj.version> - <!-- Calcite Version --> - <calcite.version>1.11.0</calcite.version> - <avatica.version>1.9.0</avatica.version> - <!-- Sonar --> <sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin> <sonar.dynamicAnalysis>reuseReports</sonar.dynamicAnalysis> @@ -279,6 +284,11 @@ </dependency> <dependency> <groupId>org.apache.kylin</groupId> + <artifactId>kylin-tool-assembly</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> <artifactId>kylin-it</artifactId> <version>${project.version}</version> </dependency> @@ -390,6 +400,66 @@ <version>${hadoop2.version}</version> <optional>true</optional> </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>${zookeeper.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-framework</artifactId> + <version>${curator.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-recipes</artifactId> + <version>${curator.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + <version>${jsr305.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>${guava.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.jcraft</groupId> + <artifactId>jsch</artifactId> + <version>${jsch.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + <version>${commons-cli.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>${commons-lang.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-math3</artifactId> + <version>${commons-math3.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>${commons-io.version}</version> + <scope>provided</scope> + </dependency> <!-- HBase2 dependencies --> <dependency> @@ -461,6 +531,11 @@ </dependency> <dependency> <groupId>org.apache.calcite.avatica</groupId> + <artifactId>avatica-core</artifactId> + <version>${avatica.version}</version> + </dependency> + <dependency> + <groupId>org.apache.calcite.avatica</groupId> <artifactId>avatica</artifactId> <version>${avatica.version}</version> </dependency> @@ -476,23 +551,39 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson.version}</version> + </dependency> + <!-- Spark dependency --> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> <version>${spark.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_2.10</artifactId> <version>${spark.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-hive_2.10</artifactId> <version>${spark.version}</version> + <scope>provided</scope> </dependency> <dependency> + <groupId>com.esotericsoftware</groupId> + <artifactId>kryo-shaded</artifactId> + <version>${kryo.version}</version> + <scope>provided</scope> + </dependency> + <!-- Kafka dependency --> + <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.10</artifactId> <version>${kafka.version}</version> @@ -501,26 +592,6 @@ <!-- Other dependencies --> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>${junit.version}</version> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <version>${zookeeper.version}</version> - </dependency> - <dependency> - <groupId>commons-cli</groupId> - <artifactId>commons-cli</artifactId> - <version>${commons-cli.version}</version> - </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <version>${commons-lang.version}</version> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> <version>${commons-lang3.version}</version> @@ -531,21 +602,15 @@ <version>${commons-collections.version}</version> </dependency> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-math3</artifactId> - <version>${commons-math3.version}</version> - </dependency> - - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <version>${commons-io.version}</version> - </dependency> - <dependency> <groupId>commons-daemon</groupId> <artifactId>commons-daemon</artifactId> <version>${commons-daemon.version}</version> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-email</artifactId> + <version>${commons-email.version}</version> + </dependency> <!-- Logging --> <dependency> @@ -571,40 +636,18 @@ <version>${slf4j.version}</version> </dependency> + <!-- Test --> <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <version>${jackson.version}</version> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-email</artifactId> - <version>${commons-email.version}</version> - </dependency> - <dependency> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> - <version>${jsr305.version}</version> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>${guava.version}</version> - </dependency> - <dependency> - <groupId>org.reflections</groupId> - <artifactId>reflections</artifactId> - <version>${reflections.version}</version> - </dependency> - <dependency> - <groupId>com.jcraft</groupId> - <artifactId>jsch</artifactId> - <version>${jsch.version}</version> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${junit.version}</version> + <scope>test</scope> </dependency> <dependency> <groupId>org.dbunit</groupId> <artifactId>dbunit</artifactId> <version>${dbunit.version}</version> + <scope>test</scope> </dependency> <dependency> <groupId>org.apache.maven</groupId> @@ -615,6 +658,7 @@ <groupId>com.h2database</groupId> <artifactId>h2</artifactId> <version>${h2.version}</version> + <scope>test</scope> </dependency> <dependency> <groupId>xerces</groupId> @@ -627,31 +671,10 @@ <version>${xalan.version}</version> </dependency> <dependency> - <groupId>com.n3twork.druid</groupId> - <artifactId>extendedset</artifactId> - <version>${extendedset.version}</version> - </dependency> - <dependency> - <groupId>com.esotericsoftware</groupId> - <artifactId>kryo-shaded</artifactId> - <version>${kryo.version}</version> - </dependency> - <dependency> <groupId>com.github.jbellis</groupId> <artifactId>jamm</artifactId> <version>${jamm.version}</version> </dependency> - - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-framework</artifactId> - <version>${curator.version}</version> - </dependency> - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-recipes</artifactId> - <version>${curator.version}</version> - </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> @@ -1036,6 +1059,7 @@ <module>jdbc</module> <module>assembly</module> <module>tool</module> + <module>tool-assembly</module> <module>kylin-it</module> <module>tomcat-ext</module> </modules> http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/query/pom.xml ---------------------------------------------------------------------- diff --git a/query/pom.xml b/query/pom.xml index 6ab74a7..d35c74c 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -47,10 +47,16 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-storage</artifactId> </dependency> + + <!-- Provided --> <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> <!-- Env & Test --> <dependency> http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/server/pom.xml ---------------------------------------------------------------------- diff --git a/server/pom.xml b/server/pom.xml index 454ed6b..3c96e60 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -233,6 +233,10 @@ <artifactId>jetty-all</artifactId> <groupId>org.eclipse.jetty.aggregate</groupId> </exclusion> + <exclusion> + <artifactId>calcite-core</artifactId> + <groupId>org.apache.calcite</groupId> + </exclusion> </exclusions> </dependency> http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/source-kafka/pom.xml ---------------------------------------------------------------------- diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml index e2fe448..da5e033 100644 --- a/source-kafka/pom.xml +++ b/source-kafka/pom.xml @@ -35,23 +35,19 @@ <dependencies> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-engine-mr</artifactId> + <artifactId>kylin-core-common</artifactId> </dependency> - <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-common</artifactId> + <artifactId>kylin-engine-mr</artifactId> </dependency> + <!-- Provided --> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.10</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> </dependency> + <!-- Env & Test --> <dependency> <groupId>org.apache.hadoop</groupId> @@ -76,9 +72,5 @@ <type>test-jar</type> <scope>test</scope> </dependency> - - - </dependencies> - </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/storage-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml index 3aea531..29ca7e5 100644 --- a/storage-hbase/pom.xml +++ b/storage-hbase/pom.xml @@ -32,8 +32,11 @@ <version>2.0.0-SNAPSHOT</version> </parent> - <dependencies> + <properties> + <shadeBase>org.apache.kylin.coprocessor.shaded</shadeBase> + </properties> + <dependencies> <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-engine-mr</artifactId> @@ -119,14 +122,20 @@ <include>org.apache.kylin:kylin-core-metadata</include> <include>org.apache.kylin:kylin-core-dictionary</include> <include>org.apache.kylin:kylin-core-cube</include> - <include>com.ning:compress-lzf</include> <include>org.roaringbitmap:RoaringBitmap</include> <include>com.tdunning:t-digest</include> - <!-- below for inverted index only --> - <include>com.n3twork.druid:extendedset</include> - <include>org.apache.commons:commons-lang3</include> </includes> </artifactSet> + <relocations> + <relocation> + <pattern>com.tdunning</pattern> + <shadedPattern>${shadeBase}.com.tdunning</shadedPattern> + </relocation> + <relocation> + <pattern>org.roaringbitmap</pattern> + <shadedPattern>${shadeBase}.org.roaringbitmap</shadedPattern> + </relocation> + </relocations> <filters> <filter> <artifact>*:*</artifact> http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/tool-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/tool-assembly/pom.xml b/tool-assembly/pom.xml new file mode 100644 index 0000000..67d92d4 --- /dev/null +++ b/tool-assembly/pom.xml @@ -0,0 +1,135 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <artifactId>kylin-tool-assembly</artifactId> + <name>Apache Kylin - Tool Assembly</name> + <packaging>jar</packaging> + <description>Apache Kylin - Tool Assembly</description> + + <parent> + <artifactId>kylin</artifactId> + <groupId>org.apache.kylin</groupId> + <version>2.0.0-SNAPSHOT</version> + </parent> + + <properties> + <shadeBase>org.apache.kylin.tool.shaded</shadeBase> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-tool</artifactId> + </dependency> + + <!-- As KylinConfigCLI will be called before Hbase/Hive/Hadoop dependency loaded, has to define the following as compile in tool --> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <scope>compile</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>assembly</shadedClassifierName> + <artifactSet> + <includes> + <!-- shade the httpcore to avoid the lower version conflict with HBase one --> + <include>org.apache.httpcomponents:httpcore</include> + <include>org.slf4j:slf4j-api</include> + <include>org.slf4j:slf4j-log4j12</include> + <include>log4j:log4j</include> + <include>commons-io:commons-io</include> + <include>commons-lang:commons-lang</include> + <include>com.google.guava:guava</include> + <include>org.apache.kylin:*</include> + </includes> + </artifactSet> + <relocations> + <relocation> + <pattern>com.google.common</pattern> + <shadedPattern>${shadeBase}.com.google.common</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.commons.io</pattern> + <shadedPattern>${shadeBase}.org.apache.commons.io</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.commons.lang</pattern> + <shadedPattern>${shadeBase}.org.apache.commons.lang</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.http</pattern> + <shadedPattern>${shadeBase}.org.apache.http</shadedPattern> + </relocation> + </relocations> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/kylin/blob/4f940010/tool/pom.xml ---------------------------------------------------------------------- diff --git a/tool/pom.xml b/tool/pom.xml index 91040d4..278c2b8 100644 --- a/tool/pom.xml +++ b/tool/pom.xml @@ -81,43 +81,4 @@ </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <shadedArtifactAttached>true</shadedArtifactAttached> - <shadedClassifierName>assembly</shadedClassifierName> - <artifactSet> - <includes> - <!-- shade the httpcore to avoid the lower version conflict with HBase one --> - <include>org.apache.httpcomponents:httpcore</include> - <include>org.apache.kylin:kylin-tool</include> - </includes> - </artifactSet> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </project>
