Use java built-in class to replace LongMutable/DoubleMutable;
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55ba39af Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55ba39af Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55ba39af Branch: refs/heads/sparkcubing-rebase Commit: 55ba39afb6ded85e5f8d82df2ba6828dc4b68819 Parents: 0a648fb Author: shaofengshi <shaofeng...@apache.org> Authored: Sun Dec 25 22:48:43 2016 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Sun Dec 25 22:48:43 2016 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 11 ++++++ .../apache/kylin/common/KylinConfigBase.java | 17 +++++++- .../cube/inmemcubing/InMemCubeBuilder.java | 2 + .../apache/kylin/gridtable/UnitTestSupport.java | 41 ++++++++++---------- .../gridtable/AggregationCacheMemSizeTest.java | 6 +-- .../gridtable/AggregationCacheSpillTest.java | 5 +-- .../kylin/gridtable/SimpleGridTableTest.java | 11 +++--- .../gridtable/SimpleInvertedIndexTest.java | 2 +- .../metadata/measure/MeasureCodecTest.java | 6 +-- .../kylin/measure/basic/DoubleIngester.java | 16 +++----- .../measure/basic/DoubleMaxAggregator.java | 15 ++++--- .../measure/basic/DoubleMinAggregator.java | 15 ++++--- .../measure/basic/DoubleSumAggregator.java | 18 +++++---- .../kylin/measure/basic/LongIngester.java | 16 +++----- .../kylin/measure/basic/LongMaxAggregator.java | 15 ++++--- .../kylin/measure/basic/LongMinAggregator.java | 15 ++++--- .../kylin/measure/basic/LongSumAggregator.java | 18 +++++---- .../metadata/datatype/BooleanSerializer.java | 29 ++++---------- .../metadata/datatype/DateTimeSerializer.java | 25 ++++-------- .../metadata/datatype/DoubleSerializer.java | 25 ++++-------- .../kylin/metadata/datatype/Int4Serializer.java | 25 ++++-------- .../metadata/datatype/Long8Serializer.java | 24 ++++-------- .../kylin/metadata/datatype/LongSerializer.java | 25 ++++-------- .../measure/AggregatorMemEstimateTest.java | 4 +- .../storage/gtrecord/DictGridTableTest.java | 41 ++++++++++---------- .../kylin/engine/mr/steps/CubeReducerTest.java | 3 +- .../kylin/engine/spark/SparkCubingByLayer.java | 32 +++++---------- .../test_case_data/sandbox/kylin.properties | 6 +-- .../hbase/steps/RowValueDecoderTest.java | 7 ++-- 29 files changed, 199 insertions(+), 276 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 131a725..dabe113 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -128,6 +128,17 @@ kylin.engine.mr.max-reducer-number=500 kylin.engine.mr.mapper-input-rows=1000000 +### Spark Engine ### + +# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run spark-submit +kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf + +# Spark install home, default be $KYLIN_HOME/spark/ +#kylin.engine.spark.spark-home= + +# Spark job submission properties file, default be $KYLIN_HOME/conf/kylin-spark-conf.properties +#kylin.engine.spark.properties-file= + ### CUBE | DICTIONARY ### # 'auto', 'inmem', 'layer' or 'random' for testing http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index c32550f..5bec9b3 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -714,7 +714,20 @@ abstract public class KylinConfigBase implements Serializable { // ============================================================================ public String getSparkHome() { - return getRequired("kylin.engine.spark.spark-home"); + String sparkHome = getOptional("kylin.engine.spark.spark-home", "spark"); + File f = new File(sparkHome); + if (f.exists()) { + return f.getAbsolutePath(); + } else { + String home = getKylinHome(); + f = new File(home, sparkHome); + if (f.exists()) { + return f.getAbsolutePath(); + } + } + + throw new IllegalArgumentException("Spark home '" + sparkHome + "' does not exist, check 'kylin.engine.spark.spark-home' in kylin.properties"); + } public String getSparkHadoopConfDir() { @@ -734,7 +747,7 @@ abstract public class KylinConfigBase implements Serializable { } } - throw new IllegalArgumentException("Spark conf file '" + conf + "' does not exist."); + throw new IllegalArgumentException("Spark conf properties file '" + conf + "' does not exist."); } public String getSparkAdditionalJars() { http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java index 66a444e..b4cba39 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java @@ -462,6 +462,8 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder { for (int i = 0; i < totalSum.length; i++) { if (totalSum[i] instanceof DoubleMutable) { totalSum[i] = Math.round(((DoubleMutable) totalSum[i]).get()); + } else if (totalSum[i] instanceof Double) { + totalSum[i] = Math.round(((Double) totalSum[i]).doubleValue()); } else if (totalSum[i] instanceof TopNCounter) { TopNCounter counter = (TopNCounter) totalSum[i]; Iterator<Counter> iterator = counter.iterator(); http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java index b8d116c..6190f29 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java @@ -28,7 +28,6 @@ import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.gridtable.GTInfo.Builder; import org.apache.kylin.measure.hllc.HLLCounter; import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.datatype.LongMutable; public class UnitTestSupport { @@ -84,16 +83,16 @@ public class UnitTestSupport { String d_01_15 = datePlus("2015-01-15", i * 4); String d_01_16 = datePlus("2015-01-16", i * 4); String d_01_17 = datePlus("2015-01-17", i * 4); - result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5"))); - result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_14, "Yang", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_14, "Luke", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_15, "Xu", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_15, "Dong", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_15, "Jason", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_16, "Mahone", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_16, "Shaofeng", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_16, "Qianhao", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_16, "George", "Food", new Long(10), new BigDecimal("10.5"))); + result.add(newRec(info, d_01_17, "Kejia", "Food", new Long(10), new BigDecimal("10.5"))); } return result; } @@ -106,16 +105,16 @@ public class UnitTestSupport { String d_01_15 = datePlus("2015-01-15", i * 4); String d_01_16 = datePlus("2015-01-16", i * 4); String d_01_17 = datePlus("2015-01-17", i * 4); - result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); - result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_14, "Yang", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_14, "Luke", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_15, "Xu", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_15, "Dong", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_15, "Jason", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_16, "Mahone", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_16, "Shaofeng", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_16, "Qianhao", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_16, "George", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); + result.add(newRec(info, d_01_17, "Kejia", "Food", new Long(10), new BigDecimal("10.5"), new HLLCounter(14))); } return result; } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java index 8ffe055..cd292a2 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java @@ -27,8 +27,6 @@ import org.apache.kylin.measure.bitmap.BitmapAggregator; import org.apache.kylin.measure.bitmap.BitmapCounter; import org.apache.kylin.measure.hllc.HLLCAggregator; import org.apache.kylin.measure.hllc.HLLCounter; -import org.apache.kylin.metadata.datatype.DoubleMutable; -import org.apache.kylin.metadata.datatype.LongMutable; import org.github.jamm.MemoryMeter; import org.junit.Test; @@ -92,10 +90,10 @@ public class AggregationCacheMemSizeTest { private MeasureAggregator<?>[] createNoMemHungryAggrs() { LongSumAggregator longSum = new LongSumAggregator(); - longSum.aggregate(new LongMutable(10)); + longSum.aggregate(new Long(10)); DoubleSumAggregator doubleSum = new DoubleSumAggregator(); - doubleSum.aggregate(new DoubleMutable(10)); + doubleSum.aggregate(new Double(10)); BigDecimalSumAggregator decimalSum = new BigDecimalSumAggregator(); decimalSum.aggregate(new BigDecimal("12345678901234567890.123456789")); http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java index d7787e0..74c2659 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java @@ -28,7 +28,6 @@ import java.util.List; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.metadata.datatype.LongMutable; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -91,7 +90,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase { for (GTRecord record : scanner) { assertNotNull(record); Object[] returnRecord = record.getValues(); - assertEquals(20, ((LongMutable) returnRecord[3]).get()); + assertEquals(20, ((Long) returnRecord[3]).longValue()); assertEquals(21, ((BigDecimal) returnRecord[4]).longValue()); count++; @@ -133,7 +132,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase { for (GTRecord record : scanner) { assertNotNull(record); Object[] returnRecord = record.getValues(); - assertEquals(80000, ((LongMutable) returnRecord[3]).get()); + assertEquals(80000, ((Long) returnRecord[3]).longValue()); assertEquals(84000, ((BigDecimal) returnRecord[4]).longValue()); count++; http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java index fd571d0..5dd7214 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleGridTableTest.java @@ -28,7 +28,6 @@ import java.util.List; import org.apache.kylin.common.util.ImmutableBitSet; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.gridtable.memstore.GTSimpleMemStore; -import org.apache.kylin.metadata.datatype.LongMutable; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -96,7 +95,7 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { Object[] v = r.getValues(); assertTrue(((String) v[0]).startsWith("2015-")); assertTrue(((String) v[2]).equals("Food")); - assertTrue(((LongMutable) v[3]).get() == 10); + assertTrue(((Long) v[3]).longValue() == 10); assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5); System.out.println(r); } @@ -113,19 +112,19 @@ public class SimpleGridTableTest extends LocalFileMetadataTestCase { Object[] v = r.getValues(); switch (i) { case 0: - assertTrue(((LongMutable) v[3]).get() == 20); + assertTrue(((Long) v[3]).longValue() == 20); assertTrue(((BigDecimal) v[4]).doubleValue() == 21.0); break; case 1: - assertTrue(((LongMutable) v[3]).get() == 30); + assertTrue(((Long) v[3]).longValue() == 30); assertTrue(((BigDecimal) v[4]).doubleValue() == 31.5); break; case 2: - assertTrue(((LongMutable) v[3]).get() == 40); + assertTrue(((Long) v[3]).longValue() == 40); assertTrue(((BigDecimal) v[4]).doubleValue() == 42.0); break; case 3: - assertTrue(((LongMutable) v[3]).get() == 10); + assertTrue(((Long) v[3]).longValue() == 10); assertTrue(((BigDecimal) v[4]).doubleValue() == 10.5); break; default: http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java index 1026414..388a08b 100644 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java +++ b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java @@ -71,7 +71,7 @@ public class SimpleInvertedIndexTest extends LocalFileMetadataTestCase { GTRowBlock.Writer writer = mockBlock.getWriter(); GTRecord record = new GTRecord(info); for (int i = 0; i < 10; i++) { - record.setValues(i < 9 ? "" + i : null, "", "", new LongMutable(0), new BigDecimal(0)); + record.setValues(i < 9 ? "" + i : null, "", "", new Long(0), new BigDecimal(0)); for (int j = 0; j < info.getRowBlockSize(); j++) { writer.append(record); } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java index 0f3f3a9..d2122d6 100644 --- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java +++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java @@ -27,8 +27,6 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.measure.BufferedMeasureCodec; import org.apache.kylin.measure.bitmap.BitmapCounter; import org.apache.kylin.measure.hllc.HLLCounter; -import org.apache.kylin.metadata.datatype.DoubleMutable; -import org.apache.kylin.metadata.datatype.LongMutable; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.junit.AfterClass; @@ -54,8 +52,8 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase { MeasureDesc descs[] = new MeasureDesc[] { measure("double"), measure("long"), measure("decimal"), measure("HLLC16"), measure("bitmap") }; BufferedMeasureCodec codec = new BufferedMeasureCodec(descs); - DoubleMutable d = new DoubleMutable(1.0); - LongMutable l = new LongMutable(2); + Double d = new Double(1.0); + Long l = new Long(2); BigDecimal b = new BigDecimal("333.1234"); HLLCounter hllc = new HLLCounter(16); hllc.add("1234567"); http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java index e42f275..90ecb0d 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java @@ -22,30 +22,24 @@ import java.util.Map; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.measure.MeasureIngester; -import org.apache.kylin.metadata.datatype.DoubleMutable; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; -public class DoubleIngester extends MeasureIngester<DoubleMutable> { - - // avoid repeated object creation - private DoubleMutable current = new DoubleMutable(); +public class DoubleIngester extends MeasureIngester<Double> { @Override - public DoubleMutable valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { + public Double valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { if (values.length > 1) throw new IllegalArgumentException(); - DoubleMutable l = current; if (values[0] == null || values[0].length() == 0) - l.set(0L); + return new Double(0); else - l.set(Double.parseDouble(values[0])); - return l; + return Double.parseDouble(values[0]); } @Override public void reset() { - current = new DoubleMutable(); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java index 25911e8..af24390 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMaxAggregator.java @@ -19,14 +19,13 @@ package org.apache.kylin.measure.basic; import org.apache.kylin.measure.MeasureAggregator; -import org.apache.kylin.metadata.datatype.DoubleMutable; /** */ @SuppressWarnings("serial") -public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> { +public class DoubleMaxAggregator extends MeasureAggregator<Double> { - DoubleMutable max = null; + Double max = null; @Override public void reset() { @@ -34,15 +33,15 @@ public class DoubleMaxAggregator extends MeasureAggregator<DoubleMutable> { } @Override - public void aggregate(DoubleMutable value) { + public void aggregate(Double value) { if (max == null) - max = new DoubleMutable(value.get()); - else if (max.get() < value.get()) - max.set(value.get()); + max = value; + else if (max < value) + max = value; } @Override - public DoubleMutable getState() { + public Double getState() { return max; } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java index be97deb..0ef4734 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleMinAggregator.java @@ -19,14 +19,13 @@ package org.apache.kylin.measure.basic; import org.apache.kylin.measure.MeasureAggregator; -import org.apache.kylin.metadata.datatype.DoubleMutable; /** */ @SuppressWarnings("serial") -public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> { +public class DoubleMinAggregator extends MeasureAggregator<Double> { - DoubleMutable min = null; + Double min = null; @Override public void reset() { @@ -34,15 +33,15 @@ public class DoubleMinAggregator extends MeasureAggregator<DoubleMutable> { } @Override - public void aggregate(DoubleMutable value) { + public void aggregate(Double value) { if (min == null) - min = new DoubleMutable(value.get()); - else if (min.get() > value.get()) - min.set(value.get()); + min = value; + else if (min > value) + min = value; } @Override - public DoubleMutable getState() { + public Double getState() { return min; } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java index 29eb787..58f714e 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleSumAggregator.java @@ -19,28 +19,30 @@ package org.apache.kylin.measure.basic; import org.apache.kylin.measure.MeasureAggregator; -import org.apache.kylin.metadata.datatype.DoubleMutable; /** */ @SuppressWarnings("serial") -public class DoubleSumAggregator extends MeasureAggregator<DoubleMutable> { +public class DoubleSumAggregator extends MeasureAggregator<Double> { - DoubleMutable sum = new DoubleMutable(); + Double sum = null; @Override public void reset() { -// sum.set(0.0); - sum = new DoubleMutable(); + sum = null; } @Override - public void aggregate(DoubleMutable value) { - sum.set(sum.get() + value.get()); + public void aggregate(Double value) { + if (sum == null) { + sum = value; + } else { + sum = sum + value; + } } @Override - public DoubleMutable getState() { + public Double getState() { return sum; } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java index 439f096..f7382b9 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java @@ -22,30 +22,24 @@ import java.util.Map; import org.apache.kylin.common.util.Dictionary; import org.apache.kylin.measure.MeasureIngester; -import org.apache.kylin.metadata.datatype.LongMutable; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; -public class LongIngester extends MeasureIngester<LongMutable> { - - // avoid repeated object creation - private LongMutable current = new LongMutable(); +public class LongIngester extends MeasureIngester<Long> { @Override - public LongMutable valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { + public Long valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { if (values.length > 1) throw new IllegalArgumentException(); - LongMutable l = current; if (values[0] == null || values[0].length() == 0) - l.set(0L); + return new Long(0L); else - l.set(Long.parseLong(values[0])); - return l; + return Long.parseLong(values[0]); } @Override public void reset() { - current = new LongMutable(); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java index ca44f15..c7556e3 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMaxAggregator.java @@ -19,14 +19,13 @@ package org.apache.kylin.measure.basic; import org.apache.kylin.measure.MeasureAggregator; -import org.apache.kylin.metadata.datatype.LongMutable; /** */ @SuppressWarnings("serial") -public class LongMaxAggregator extends MeasureAggregator<LongMutable> { +public class LongMaxAggregator extends MeasureAggregator<Long> { - LongMutable max = null; + Long max = null; @Override public void reset() { @@ -34,15 +33,15 @@ public class LongMaxAggregator extends MeasureAggregator<LongMutable> { } @Override - public void aggregate(LongMutable value) { + public void aggregate(Long value) { if (max == null) - max = new LongMutable(value.get()); - else if (max.get() < value.get()) - max.set(value.get()); + max = value; + else if (max < value) + max = value; } @Override - public LongMutable getState() { + public Long getState() { return max; } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java index dadc64e..135ce10 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongMinAggregator.java @@ -19,14 +19,13 @@ package org.apache.kylin.measure.basic; import org.apache.kylin.measure.MeasureAggregator; -import org.apache.kylin.metadata.datatype.LongMutable; /** */ @SuppressWarnings("serial") -public class LongMinAggregator extends MeasureAggregator<LongMutable> { +public class LongMinAggregator extends MeasureAggregator<Long> { - LongMutable min = null; + Long min = null; @Override public void reset() { @@ -34,15 +33,15 @@ public class LongMinAggregator extends MeasureAggregator<LongMutable> { } @Override - public void aggregate(LongMutable value) { + public void aggregate(Long value) { if (min == null) - min = new LongMutable(value.get()); - else if (min.get() > value.get()) - min.set(value.get()); + min = value; + else if (min > value) + min = value; } @Override - public LongMutable getState() { + public Long getState() { return min; } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java index 0a56af8..28d45c0 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongSumAggregator.java @@ -19,28 +19,30 @@ package org.apache.kylin.measure.basic; import org.apache.kylin.measure.MeasureAggregator; -import org.apache.kylin.metadata.datatype.LongMutable; /** */ @SuppressWarnings("serial") -public class LongSumAggregator extends MeasureAggregator<LongMutable> { +public class LongSumAggregator extends MeasureAggregator<Long> { - LongMutable sum = new LongMutable(); + Long sum = null; @Override public void reset() { -// sum.set(0); - sum = new LongMutable(); + sum = null; } @Override - public void aggregate(LongMutable value) { - sum.set(sum.get() + value.get()); + public void aggregate(Long value) { + if (sum == null) { + sum = value; + } else { + sum = sum + value; + } } @Override - public LongMutable getState() { + public Long getState() { return sum; } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java index 998e5e9..3d485d2 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BooleanSerializer.java @@ -23,7 +23,7 @@ import org.apache.commons.lang.BooleanUtils; import java.nio.ByteBuffer; -public class BooleanSerializer extends DataTypeSerializer<LongMutable> { +public class BooleanSerializer extends DataTypeSerializer<Long> { public final static String[] TRUE_VALUE_SET = { "true", "t", "on", "yes" }; @@ -31,24 +31,13 @@ public class BooleanSerializer extends DataTypeSerializer<LongMutable> { } @Override - public void serialize(LongMutable value, ByteBuffer out) { - out.putLong(value.get()); - } - - private LongMutable current() { - LongMutable l = (LongMutable) current.get(); - if (l == null) { - l = new LongMutable(); - current.set(l); - } - return l; + public void serialize(Long value, ByteBuffer out) { + out.putLong(value); } @Override - public LongMutable deserialize(ByteBuffer in) { - LongMutable l = current(); - l.set(in.getLong()); - return l; + public Long deserialize(ByteBuffer in) { + return in.getLong(); } @Override @@ -67,12 +56,10 @@ public class BooleanSerializer extends DataTypeSerializer<LongMutable> { } @Override - public LongMutable valueOf(String str) { - LongMutable l = current(); + public Long valueOf(String str) { if (str == null) - l.set(0L); + return Long.valueOf(0L); else - l.set(BooleanUtils.toInteger(ArrayUtils.contains(TRUE_VALUE_SET, str.toLowerCase()))); - return l; + return Long.valueOf(BooleanUtils.toInteger(ArrayUtils.contains(TRUE_VALUE_SET, str.toLowerCase()))); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java index a5719bd..5101766 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DateTimeSerializer.java @@ -22,30 +22,19 @@ import org.apache.kylin.common.util.DateFormat; import java.nio.ByteBuffer; -public class DateTimeSerializer extends DataTypeSerializer<LongMutable> { +public class DateTimeSerializer extends DataTypeSerializer<Long> { public DateTimeSerializer(DataType type) { } @Override - public void serialize(LongMutable value, ByteBuffer out) { - out.putLong(value.get()); - } - - private LongMutable current() { - LongMutable l = (LongMutable) current.get(); - if (l == null) { - l = new LongMutable(); - current.set(l); - } - return l; + public void serialize(Long value, ByteBuffer out) { + out.putLong(value); } @Override - public LongMutable deserialize(ByteBuffer in) { - LongMutable l = current(); - l.set(in.getLong()); - return l; + public Long deserialize(ByteBuffer in) { + return in.getLong(); } @Override @@ -64,8 +53,8 @@ public class DateTimeSerializer extends DataTypeSerializer<LongMutable> { } @Override - public LongMutable valueOf(String str) { - return new LongMutable(DateFormat.stringToMillis(str)); + public Long valueOf(String str) { + return Long.valueOf(DateFormat.stringToMillis(str)); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java index cda4ff5..20cfff3 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DoubleSerializer.java @@ -22,30 +22,19 @@ import java.nio.ByteBuffer; /** */ -public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> { +public class DoubleSerializer extends DataTypeSerializer<Double> { public DoubleSerializer(DataType type) { } @Override - public void serialize(DoubleMutable value, ByteBuffer out) { - out.putDouble(value.get()); - } - - private DoubleMutable current() { - DoubleMutable d = (DoubleMutable) current.get(); - if (d == null) { - d = new DoubleMutable(); - current.set(d); - } - return d; + public void serialize(Double value, ByteBuffer out) { + out.putDouble(value); } @Override - public DoubleMutable deserialize(ByteBuffer in) { - DoubleMutable d = current(); - d.set(in.getDouble()); - return d; + public Double deserialize(ByteBuffer in) { + return in.getDouble(); } @Override @@ -64,7 +53,7 @@ public class DoubleSerializer extends DataTypeSerializer<DoubleMutable> { } @Override - public DoubleMutable valueOf(String str) { - return new DoubleMutable(Double.parseDouble(str)); + public Double valueOf(String str) { + return Double.parseDouble(str); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java index c726ab6..0e82e11 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Int4Serializer.java @@ -24,30 +24,19 @@ import java.nio.ByteBuffer; /** */ -public class Int4Serializer extends DataTypeSerializer<IntMutable> { +public class Int4Serializer extends DataTypeSerializer<Integer> { public Int4Serializer(DataType type) { } @Override - public void serialize(IntMutable value, ByteBuffer out) { - BytesUtil.writeUnsigned(value.get(), 4, out); - } - - private IntMutable current() { - IntMutable l = (IntMutable) current.get(); - if (l == null) { - l = new IntMutable(); - current.set(l); - } - return l; + public void serialize(Integer value, ByteBuffer out) { + BytesUtil.writeUnsigned(value, 4, out); } @Override - public IntMutable deserialize(ByteBuffer in) { - IntMutable l = current(); - l.set(BytesUtil.readUnsigned(in, 4)); - return l; + public Integer deserialize(ByteBuffer in) { + return BytesUtil.readUnsigned(in, 4); } @Override @@ -66,7 +55,7 @@ public class Int4Serializer extends DataTypeSerializer<IntMutable> { } @Override - public IntMutable valueOf(String str) { - return new IntMutable(Integer.parseInt(str)); + public Integer valueOf(String str) { + return Integer.parseInt(str); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java index 186cdb9..7dd5aa7 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/Long8Serializer.java @@ -24,30 +24,20 @@ import java.nio.ByteBuffer; /** */ -public class Long8Serializer extends DataTypeSerializer<LongMutable> { +public class Long8Serializer extends DataTypeSerializer<Long> { public Long8Serializer(DataType type) { } @Override - public void serialize(LongMutable value, ByteBuffer out) { - BytesUtil.writeLong(value.get(), out); + public void serialize(Long value, ByteBuffer out) { + BytesUtil.writeLong(value, out); } - private LongMutable current() { - LongMutable l = (LongMutable) current.get(); - if (l == null) { - l = new LongMutable(); - current.set(l); - } - return l; - } @Override - public LongMutable deserialize(ByteBuffer in) { - LongMutable l = current(); - l.set(BytesUtil.readLong(in)); - return l; + public Long deserialize(ByteBuffer in) { + return BytesUtil.readLong(in); } @Override @@ -66,7 +56,7 @@ public class Long8Serializer extends DataTypeSerializer<LongMutable> { } @Override - public LongMutable valueOf(String str) { - return new LongMutable(Long.parseLong(str)); + public Long valueOf(String str) { + return Long.parseLong(str); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java index d8f3f37..605dcd7 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/LongSerializer.java @@ -24,30 +24,19 @@ import java.nio.ByteBuffer; /** */ -public class LongSerializer extends DataTypeSerializer<LongMutable> { +public class LongSerializer extends DataTypeSerializer<Long> { public LongSerializer(DataType type) { } @Override - public void serialize(LongMutable value, ByteBuffer out) { - BytesUtil.writeVLong(value.get(), out); - } - - private LongMutable current() { - LongMutable l = (LongMutable) current.get(); - if (l == null) { - l = new LongMutable(); - current.set(l); - } - return l; + public void serialize(Long value, ByteBuffer out) { + BytesUtil.writeVLong(value, out); } @Override - public LongMutable deserialize(ByteBuffer in) { - LongMutable l = current(); - l.set(BytesUtil.readVLong(in)); - return l; + public Long deserialize(ByteBuffer in) { + return BytesUtil.readVLong(in); } @Override @@ -72,7 +61,7 @@ public class LongSerializer extends DataTypeSerializer<LongMutable> { } @Override - public LongMutable valueOf(String str) { - return new LongMutable(Long.parseLong(str)); + public Long valueOf(String str) { + return Long.parseLong(str); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java index 0f22610..af70196 100644 --- a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java @@ -52,7 +52,7 @@ public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase { } private List<? extends MeasureAggregator> basicAggregators() { - LongMutable longVal = new LongMutable(1000); + Long longVal = new Long(1000); LongMinAggregator longMin = new LongMinAggregator(); LongMaxAggregator longMax = new LongMaxAggregator(); LongSumAggregator longSum = new LongSumAggregator(); @@ -60,7 +60,7 @@ public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase { longMax.aggregate(longVal); longSum.aggregate(longVal); - DoubleMutable doubleVal = new DoubleMutable(1.0); + Double doubleVal = new Double(1.0); DoubleMinAggregator doubleMin = new DoubleMinAggregator(); DoubleMaxAggregator doubleMax = new DoubleMaxAggregator(); DoubleSumAggregator doubleSum = new DoubleSumAggregator(); http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java index bac9c10..b90e3a1 100644 --- a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java +++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java @@ -52,7 +52,6 @@ import org.apache.kylin.gridtable.GridTable; import org.apache.kylin.gridtable.IGTScanner; import org.apache.kylin.gridtable.memstore.GTSimpleMemStore; import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.datatype.LongMutable; import org.apache.kylin.metadata.filter.ColumnTupleFilter; import org.apache.kylin.metadata.filter.CompareTupleFilter; import org.apache.kylin.metadata.filter.ConstantTupleFilter; @@ -499,16 +498,16 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { GTRecord r = new GTRecord(table.getInfo()); GTBuilder builder = table.rebuild(); - builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5"))); - builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-14", "30", "Yang", new Long(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-14", "30", "Luke", new Long(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-15", "20", "Dong", new Long(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-15", "20", "Jason", new Long(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-15", "30", "Xu", new Long(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-16", "20", "Mahone", new Long(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-16", "20", "Qianhao", new Long(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-16", "30", "George", new Long(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new Long(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-17", "10", "Kejia", new Long(10), new BigDecimal("10.5"))); builder.close(); return table; @@ -524,34 +523,34 @@ public class DictGridTableTest extends LocalFileMetadataTestCase { for (int i = 0; i < 100000; i++) { for (int j = 0; j < 10; j++) - builder.write(r.setValues("2015-01-14", "30", "Yang", new LongMutable(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-14", "30", "Yang", new Long(10), new BigDecimal("10.5"))); for (int j = 0; j < 10; j++) - builder.write(r.setValues("2015-01-14", "30", "Luke", new LongMutable(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-14", "30", "Luke", new Long(10), new BigDecimal("10.5"))); for (int j = 0; j < 10; j++) - builder.write(r.setValues("2015-01-15", "20", "Dong", new LongMutable(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-15", "20", "Dong", new Long(10), new BigDecimal("10.5"))); for (int j = 0; j < 10; j++) - builder.write(r.setValues("2015-01-15", "20", "Jason", new LongMutable(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-15", "20", "Jason", new Long(10), new BigDecimal("10.5"))); for (int j = 0; j < 10; j++) - builder.write(r.setValues("2015-01-15", "30", "Xu", new LongMutable(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-15", "30", "Xu", new Long(10), new BigDecimal("10.5"))); for (int j = 0; j < 10; j++) - builder.write(r.setValues("2015-01-16", "20", "Mahone", new LongMutable(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-16", "20", "Mahone", new Long(10), new BigDecimal("10.5"))); for (int j = 0; j < 10; j++) - builder.write(r.setValues("2015-01-16", "20", "Qianhao", new LongMutable(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-16", "20", "Qianhao", new Long(10), new BigDecimal("10.5"))); for (int j = 0; j < 10; j++) - builder.write(r.setValues("2015-01-16", "30", "George", new LongMutable(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-16", "30", "George", new Long(10), new BigDecimal("10.5"))); for (int j = 0; j < 10; j++) - builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new LongMutable(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-16", "30", "Shaofeng", new Long(10), new BigDecimal("10.5"))); for (int j = 0; j < 10; j++) - builder.write(r.setValues("2015-01-17", "10", "Kejia", new LongMutable(10), new BigDecimal("10.5"))); + builder.write(r.setValues("2015-01-17", "10", "Kejia", new Long(10), new BigDecimal("10.5"))); } builder.close(); http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java index 97dd750..29ca9b8 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java @@ -41,7 +41,6 @@ import org.apache.kylin.measure.BufferedMeasureCodec; import org.apache.kylin.measure.MeasureAggregator; import org.apache.kylin.measure.MeasureIngester; import org.apache.kylin.measure.MeasureType; -import org.apache.kylin.metadata.datatype.LongMutable; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.junit.After; @@ -161,7 +160,7 @@ public class CubeReducerTest extends LocalFileMetadataTestCase { } private Text newValueText(BufferedMeasureCodec codec, String sum, String min, String max, int count, int item_count) { - Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new LongMutable(count), new LongMutable(item_count) }; + Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new Long(count), new Long(item_count) }; ByteBuffer buf = codec.encode(values); http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java index 7467e86..8615a13 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java @@ -71,8 +71,6 @@ import scala.Tuple2; import javax.annotation.Nullable; import java.io.File; import java.io.FileFilter; -import java.io.IOException; -import java.io.ObjectInputStream; import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Collection; @@ -309,31 +307,24 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa BufferedMeasureCodec codec; CubeDesc cubeDesc; int measureNum; - transient ThreadLocal<MeasureAggregators> current = new ThreadLocal<>(); + MeasureAggregators aggregators; CuboidReducerFunction2(int measureNum, CubeDesc cubeDesc, BufferedMeasureCodec codec) { this.codec = codec; this.cubeDesc = cubeDesc; this.measureNum = measureNum; + this.aggregators = new MeasureAggregators(cubeDesc.getMeasures()); } @Override public Object[] call(Object[] input1, Object[] input2) throws Exception { - if (current.get() == null) { - current.set(new MeasureAggregators(cubeDesc.getMeasures())); - } Object[] result = new Object[measureNum]; - current.get().reset(); - current.get().aggregate(input1); - current.get().aggregate(input2); - current.get().collectStates(result); + aggregators.reset(); + aggregators.aggregate(input1); + aggregators.aggregate(input2); + aggregators.collectStates(result); return result; } - - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { - in.defaultReadObject(); - current = new ThreadLocal(); - } } class BaseCuboidReducerFunction2 extends CuboidReducerFunction2 { @@ -346,14 +337,11 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa @Override public Object[] call(Object[] input1, Object[] input2) throws Exception { - if (current.get() == null) { - current.set(new MeasureAggregators(cubeDesc.getMeasures())); - } - current.get().reset(); Object[] result = new Object[measureNum]; - current.get().aggregate(input1, needAggr); - current.get().aggregate(input2, needAggr); - current.get().collectStates(result); + aggregators.reset(); + aggregators.aggregate(input1, needAggr); + aggregators.aggregate(input2, needAggr); + aggregators.collectStates(result); return result; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index db8eb7a..74a6a8a 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -157,9 +157,7 @@ kylin.env=DEV kylin.source.hive.keep-flat-table=true ### Spark ### -#kylin.engine.spark.spark-home=/usr/hdp/2.2.4.2-2/spark #kylin.engine.spark.env.hadoop-conf-dir=/etc/hadoop/conf -kylin.engine.spark.env.hadoop-conf-dir=/Users/shishaofeng/workspace/kylin-15/examples/test_case_data/sandbox +kylin.engine.spark.env.hadoop-conf-dir=../examples/test_case_data/sandbox kylin.engine.spark.spark-home=/Users/shishaofeng/spark-1.6.3-bin-hadoop2.6 -kylin.engine.spark.properties-file=/Users/shishaofeng/workspace/kylin-15/examples/test_case_data/sandbox/kylin-spark-conf.properties -kylin.engine.spark.conf.jars= \ No newline at end of file +kylin.engine.spark.properties-file=../examples/test_case_data/sandbox/kylin-spark-conf.properties \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/55ba39af/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java index f0ad6e0..9b1a00d 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java @@ -31,7 +31,6 @@ import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.cube.model.HBaseColumnDesc; import org.apache.kylin.measure.BufferedMeasureCodec; import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.datatype.LongMutable; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.junit.After; @@ -60,8 +59,8 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase { BigDecimal sum = new BigDecimal("333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); - LongMutable count = new LongMutable(2); - LongMutable item_count = new LongMutable(100); + Long count = new Long(2); + Long item_count = new Long(100); ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count }); buf.flip(); @@ -91,7 +90,7 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase { BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); LongWritable count = new LongWritable(2); - LongMutable item_count = new LongMutable(100); + Long item_count = new Long(100); codec.encode(new Object[] { sum, min, max, count, item_count }); }
