minor, adjust out-of-box kylin.properties for sparking cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9ff78bce Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9ff78bce Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9ff78bce Branch: refs/heads/master-hbase0.98 Commit: 9ff78bce72a56b63dc9e1c8f0b4cd59d0c01b011 Parents: d59b8ae Author: Li Yang <liy...@apache.org> Authored: Fri Feb 24 16:15:52 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Fri Feb 24 16:20:33 2017 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 17 ++++++++--------- .../apache/kylin/engine/spark/SparkExecutable.java | 4 ++-- webapp/app/js/model/cubeConfig.js | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/9ff78bce/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index d7ba0b3..46fe86b 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -203,8 +203,9 @@ kylin.security.saml.context-path=/kylin ### Spark Engine Configs ### -# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" before run spark-submit -#kylin.env.hadoop-conf-dir=/etc/hive/conf +# Hadoop conf folder, will export this as "HADOOP_CONF_DIR" to run spark-submit +# This must contain site xmls of core, yarn, hive, and hbase in one folder +#kylin.env.hadoop-conf-dir=/etc/hadoop/conf # Estimate the RDD partition numbers kylin.engine.spark.rdd-partition-cut-mb=10 @@ -215,19 +216,17 @@ kylin.engine.spark.min-partition=1 # Max partition numbers of rdd kylin.engine.spark.max-partition=5000 -### Spark conf (default is in spark/conf/spark-defaults.conf) +## Spark conf (default is in spark/conf/spark-defaults.conf) kylin.engine.spark-conf.spark.master=yarn kylin.engine.spark-conf.spark.submit.deployMode=cluster -kylin.engine.spark-conf.spark.yarn.executor.memoryOverhead=1024 -kylin.engine.spark-conf.spark.yarn.driver.memoryOverhead=384 kylin.engine.spark-conf.spark.yarn.queue=default -kylin.engine.spark-conf.spark.executor.memory=4G -kylin.engine.spark-conf.spark.executor.cores=4 -kylin.engine.spark-conf.spark.executor.instances=8 -kylin.engine.spark-conf.spark.storage.memoryFraction=0.3 +kylin.engine.spark-conf.spark.executor.memory=1G +kylin.engine.spark-conf.spark.executor.cores=2 +kylin.engine.spark-conf.spark.executor.instances=1 kylin.engine.spark-conf.spark.eventLog.enabled=true kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history + ## manually upload spark-assembly jar to HDFS and then set this property will avoid repeatedly uploading jar at runtime #kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar #kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec http://git-wip-us.apache.org/repos/asf/kylin/blob/9ff78bce/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java index cf7438c..5ad21cf 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java @@ -71,7 +71,7 @@ public class SparkExecutable extends AbstractExecutable { @Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final KylinConfig config = context.getConfig(); - if (config.getSparkHome() == null) { + if (KylinConfig.getSparkHome() == null) { throw new NullPointerException(); } if (config.getKylinJobJarPath() == null) { @@ -117,7 +117,7 @@ public class SparkExecutable extends AbstractExecutable { stringBuilder.append("--files %s --jars %s %s %s"); try { - String cmd = String.format(stringBuilder.toString(), hadoopConf, config.getSparkHome(), hbaseConfFile.getAbsolutePath(), jars, jobJar, formatArgs()); + String cmd = String.format(stringBuilder.toString(), hadoopConf, KylinConfig.getSparkHome(), hbaseConfFile.getAbsolutePath(), jars, jobJar, formatArgs()); logger.info("cmd:" + cmd); final StringBuilder output = new StringBuilder(); CliCommandExecutor exec = new CliCommandExecutor(); http://git-wip-us.apache.org/repos/asf/kylin/blob/9ff78bce/webapp/app/js/model/cubeConfig.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js index 47d0f46..eec3f10 100644 --- a/webapp/app/js/model/cubeConfig.js +++ b/webapp/app/js/model/cubeConfig.js @@ -25,7 +25,7 @@ KylinApp.constant('cubeConfig', { cubePartitionTypes: ['APPEND'], engineType:[ {name:'MapReduce',value: 2}, - {name:'Spark(Beta)',value: 4} + {name:'Spark (Beta)',value: 4} ], joinTypes: [ {name: 'Left', value: 'left'},
