[kylin] branch kylin-on-parquet-v2 updated: KYLIN-4765 Set spark.sql.shuffle.partition to 1 for debug on local

[xxyu]

This is an automated email from the ASF dual-hosted git repository.

xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this 
push:
     new 67a1c7c  KYLIN-4765 Set spark.sql.shuffle.partition to 1 for debug on 
local
67a1c7c is described below

commit 67a1c7c6c28e1e22f6d4e7c3b20dbbb9b80a52fe
Author: rupengwang <wangrup...@live.cn>
AuthorDate: Mon Sep 21 12:24:59 2020 +0800

    KYLIN-4765 Set spark.sql.shuffle.partition to 1 for debug on local
---
 .../org/apache/kylin/engine/spark/application/SparkApplication.java    | 3 +++
 .../java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java  | 1 -
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git 
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
 
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
index a2e1ad8..3da1271 100644
--- 
a/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
+++ 
b/kylin-spark-project/kylin-spark-engine/src/main/java/org/apache/kylin/engine/spark/application/SparkApplication.java
@@ -142,6 +142,9 @@ public abstract class SparkApplication {
                 }
             } else if (!isJobOnCluster(sparkConf)) {
                 sparkConf.set("spark.master", "local");
+                if 
(!config.getSparkConfigOverride().containsKey("spark.sql.shuffle.partitions")) {
+                    sparkConf.set("spark.sql.shuffle.partitions", "1");
+                }
             }
 
             // for wrapping credential
diff --git 
a/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java
 
b/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java
index 4550e19..9e2328c 100644
--- 
a/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java
+++ 
b/kylin-spark-project/kylin-spark-engine/src/test/java/org/apache/kylin/engine/spark/LocalWithSparkSessionTest.java
@@ -128,7 +128,6 @@ public class LocalWithSparkSessionTest extends 
LocalFileMetadataTestCase impleme
         sparkConf = new 
SparkConf().setAppName(UUID.randomUUID().toString()).setMaster("local[4]");
         sparkConf.set("spark.serializer", 
"org.apache.spark.serializer.JavaSerializer");
         sparkConf.set(StaticSQLConf.CATALOG_IMPLEMENTATION().key(), 
"in-memory");
-        sparkConf.set("spark.sql.shuffle.partitions", "1");
         sparkConf.set("spark.memory.fraction", "0.1");
         // opt memory
         sparkConf.set("spark.shuffle.detectCorrupt", "false");