Repository: zeppelin Updated Branches: refs/heads/master d11221fb8 -> 223d225e0
[ZEPPELIN-1383][ Interpreters][r-interpreter] remove SparkInterpreter.getSystemDefault and update relative code ### What is this PR for? clean some redundant code: remove `SparkInterpreter.getSystemDefault` methods, and replace it with `InterpreterProperty.getValue` ### What type of PR is it? Improvement ### Todos N/A ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-1383 remove SparkInterpreter.getSystemDefault and update relative code ### How should this be tested? Existing tests. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: WeichenXu <weichenxu...@outlook.com> Closes #1372 from WeichenXu123/remove_spark_interpreter_getSystemDefault and squashes the following commits: 204a34c [WeichenXu] improve code stype. 841b757 [WeichenXu] update. Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/223d225e Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/223d225e Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/223d225e Branch: refs/heads/master Commit: 223d225e0119d9e3013206a211df6b70a71b6a66 Parents: d11221f Author: WeichenXu <weichenxu...@outlook.com> Authored: Sun Aug 28 00:44:26 2016 -0700 Committer: Jongyoul Lee <jongy...@apache.org> Committed: Wed Aug 31 17:07:57 2016 +0900 ---------------------------------------------------------------------- .../zeppelin/rinterpreter/RInterpreter.scala | 8 +++--- .../apache/zeppelin/spark/SparkInterpreter.java | 27 +++----------------- .../interpreter/InterpreterProperty.java | 1 - .../interpreter/InterpreterPropertyBuilder.java | 7 +++++ 4 files changed, 15 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/223d225e/r/src/main/scala/org/apache/zeppelin/rinterpreter/RInterpreter.scala ---------------------------------------------------------------------- diff --git a/r/src/main/scala/org/apache/zeppelin/rinterpreter/RInterpreter.scala b/r/src/main/scala/org/apache/zeppelin/rinterpreter/RInterpreter.scala index f0558a9..9504573 100644 --- a/r/src/main/scala/org/apache/zeppelin/rinterpreter/RInterpreter.scala +++ b/r/src/main/scala/org/apache/zeppelin/rinterpreter/RInterpreter.scala @@ -111,10 +111,10 @@ object RInterpreter { // These are the additional properties we need on top of the ones provided by the spark interpreters lazy val props: Map[String, InterpreterProperty] = new InterpreterPropertyBuilder() - .add("rhadoop.cmd", SparkInterpreter.getSystemDefault("HADOOP_CMD", "rhadoop.cmd", ""), "Usually /usr/bin/hadoop") - .add("rhadooop.streamingjar", SparkInterpreter.getSystemDefault("HADOOP_STREAMING", "rhadooop.streamingjar", ""), "Usually /usr/lib/hadoop/contrib/streaming/hadoop-streaming-<version>.jar") - .add("rscala.debug", SparkInterpreter.getSystemDefault("RSCALA_DEBUG", "rscala.debug","false"), "Whether to turn on rScala debugging") // TEST: Implemented but not tested - .add("rscala.timeout", SparkInterpreter.getSystemDefault("RSCALA_TIMEOUT", "rscala.timeout","60"), "Timeout for rScala") // TEST: Implemented but not tested + .add("rhadoop.cmd", "HADOOP_CMD", "rhadoop.cmd", "", "Usually /usr/bin/hadoop") + .add("rhadooop.streamingjar", "HADOOP_STREAMING", "rhadooop.streamingjar", "", "Usually /usr/lib/hadoop/contrib/streaming/hadoop-streaming-<version>.jar") + .add("rscala.debug", "RSCALA_DEBUG", "rscala.debug","false", "Whether to turn on rScala debugging") // TEST: Implemented but not tested + .add("rscala.timeout", "RSCALA_TIMEOUT", "rscala.timeout","60", "Timeout for rScala") // TEST: Implemented but not tested .build def getProps() = { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/223d225e/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index e6ce17f..9a54912 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -49,6 +49,7 @@ import org.apache.spark.ui.jobs.JobProgressListener; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterProperty; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterUtils; @@ -447,10 +448,11 @@ public class SparkInterpreter extends Interpreter { } private void setupConfForPySpark(SparkConf conf) { - String pysparkBasePath = getSystemDefault("SPARK_HOME", null, null); + String pysparkBasePath = new InterpreterProperty("SPARK_HOME", null, null, null).getValue(); File pysparkPath; if (null == pysparkBasePath) { - pysparkBasePath = getSystemDefault("ZEPPELIN_HOME", "zeppelin.home", "../"); + pysparkBasePath = + new InterpreterProperty("ZEPPELIN_HOME", "zeppelin.home", "../", null).getValue(); pysparkPath = new File(pysparkBasePath, "interpreter" + File.separator + "spark" + File.separator + "pyspark"); } else { @@ -500,27 +502,6 @@ public class SparkInterpreter extends Interpreter { return null != System.getenv("SPARK_SUBMIT"); } - public static String getSystemDefault( - String envName, - String propertyName, - String defaultValue) { - - if (envName != null && !envName.isEmpty()) { - String envValue = System.getenv().get(envName); - if (envValue != null) { - return envValue; - } - } - - if (propertyName != null && !propertyName.isEmpty()) { - String propValue = System.getProperty(propertyName); - if (propValue != null) { - return propValue; - } - } - return defaultValue; - } - public boolean printREPLOutput() { return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.printREPLOutput")); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/223d225e/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java index 488f2a1..5067586 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java @@ -71,7 +71,6 @@ public class InterpreterProperty { } public String getValue() { - //TODO(jongyoul): Remove SparkInterpreter's getSystemDefault method if (envName != null && !envName.isEmpty()) { String envValue = System.getenv().get(envName); if (envValue != null) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/223d225e/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterPropertyBuilder.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterPropertyBuilder.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterPropertyBuilder.java index f077b4e..f33dc7c 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterPropertyBuilder.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterPropertyBuilder.java @@ -31,6 +31,13 @@ public class InterpreterPropertyBuilder { return this; } + public InterpreterPropertyBuilder add(String name, String envName, String propertyName, + String defaultValue, String description){ + properties.put(name, + new InterpreterProperty(envName, propertyName, defaultValue, description)); + return this; + } + public Map<String, InterpreterProperty> build(){ return properties; }
