minor, fix spark serialization issues
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1902da89 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1902da89 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1902da89 Branch: refs/heads/2.x-staging Commit: 1902da892003067f02f5d085588847ccfca48671 Parents: 070d42e Author: lidongsjtu <lid...@apache.org> Authored: Wed Jan 20 13:12:04 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Wed Jan 20 15:34:13 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/hll/HyperLogLogPlusCounter.java | 2 +- .../main/java/org/apache/kylin/engine/spark/SparkCubing.java | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/1902da89/core-common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/core-common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java index 11ae78f..ef91509 100644 --- a/core-common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java +++ b/core-common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java @@ -42,7 +42,7 @@ public class HyperLogLogPlusCounter implements Serializable, Comparable<HyperLog private final int p; private final int m; - private transient final HashFunction hashFunc; + private final HashFunction hashFunc; byte[] registers; public HyperLogLogPlusCounter() { http://git-wip-us.apache.org/repos/asf/kylin/blob/1902da89/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java index 19da936..7f5ab6b 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java @@ -254,7 +254,7 @@ public class SparkCubing extends AbstractApplication { List<String>, HashMap<Long, HyperLogLogPlusCounter>>() { - final HashFunction hashFunction = Hashing.murmur3_32(); + final HashFunction hashFunction = Hashing.murmur3_128(); @Override public HashMap<Long, HyperLogLogPlusCounter> call(HashMap<Long, HyperLogLogPlusCounter> v1, List<String> v2) throws Exception { @@ -343,12 +343,11 @@ public class SparkCubing extends AbstractApplication { prepare(); final CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName); - final CubeDesc cubeDesc = cubeInstance.getDescriptor(); LinkedBlockingQueue<List<String>> blockingQueue = new LinkedBlockingQueue(); System.out.println("load properties finished"); AbstractInMemCubeBuilder inMemCubeBuilder = new DoggedCubeBuilder(cubeInstance.getDescriptor(), dictionaryMap); - final SparkCuboidWriter sparkCuboidWriter = new BufferedCuboidWriter(new DefaultTupleConverter(cubeSegment, columnLengthMap)); + final SparkCuboidWriter sparkCuboidWriter = new BufferedCuboidWriter(new DefaultTupleConverter(cubeInstance.getSegmentById(segmentId), columnLengthMap)); Executors.newCachedThreadPool().submit(inMemCubeBuilder.buildAsRunnable(blockingQueue, sparkCuboidWriter)); try { while (listIterator.hasNext()) { @@ -518,7 +517,6 @@ public class SparkCubing extends AbstractApplication { kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.model").getSubTypesOf(Object.class)); kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.measure").getSubTypesOf(Object.class)); kyroClasses.add(HashMap.class); - kyroClasses.add(org.apache.kylin.cube.CubeSegment.class); kyroClasses.add(org.apache.spark.sql.Row[].class); kyroClasses.add(org.apache.spark.sql.Row.class); kyroClasses.add(org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema.class);
