Repository: zeppelin Updated Branches: refs/heads/master c2a91e96c -> a5b437d7f
[ZEPPELIN-1609] using pyspark(python3) with livy interperter ### What is this PR for? using pyspark(python3) with livy interperter ### What type of PR is it? Improvement ### Todos ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-1609 ### How should this be tested? %livy.pyspark3 3/2 check result is "1.5" ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? - NO * Is there breaking changes for older versions? - NO * Does this needs documentation? - NO Author: purechoc <purechoc...@gmail.com> Author: purechoc <purec...@ncsoft.com> Closes #1587 from purechoc/zeppelin-livy-pyspark3 and squashes the following commits: 6f7391f [purechoc] update template 7c0f70d [purechoc] refactor code 476ae25 [purechoc] Merge remote-tracking branch 'upstream3/master' into zeppelin-livy-pyspark3 55d486f [purechoc] Trigger e5db640 [purechoc] add properties 51140f2 [purechoc] Trigger 3fc98cd [purechoc] add configuration d0e2293 [purechoc] Trigger 7e42108 [purechoc] add %livy.python3 3775e47 [purechoc] add %livy.python3 c0babf6 [purechoc] add %livy.python3 Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/a5b437d7 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/a5b437d7 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/a5b437d7 Branch: refs/heads/master Commit: a5b437d7fe5adcdb4546b91af5f77b0252d2fd64 Parents: c2a91e9 Author: purechoc <purechoc...@gmail.com> Authored: Thu Dec 29 09:08:18 2016 +0900 Committer: Felix Cheung <felixche...@apache.org> Committed: Tue Jan 3 21:54:09 2017 -0800 ---------------------------------------------------------------------- conf/zeppelin-site.xml.template | 2 +- .../zeppelin/livy/LivyPySpark3Interpreter.java | 46 ++++++++++++++++++++ .../src/main/resources/interpreter-setting.json | 20 +++++++++ .../zeppelin/conf/ZeppelinConfiguration.java | 1 + 4 files changed, 68 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a5b437d7/conf/zeppelin-site.xml.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 2edb056..08104c7 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -190,7 +190,7 @@ <property> <name>zeppelin.interpreters</name> - <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlIn terpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value> + <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlIn terpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value> <description>Comma separated interpreter configurations. First interpreter become a default</description> </property> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a5b437d7/livy/src/main/java/org/apache/zeppelin/livy/LivyPySpark3Interpreter.java ---------------------------------------------------------------------- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyPySpark3Interpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyPySpark3Interpreter.java new file mode 100644 index 0000000..4a0314c --- /dev/null +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyPySpark3Interpreter.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.livy; + +import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + + +/** + * Livy PySpark interpreter for Zeppelin. + */ +public class LivyPySpark3Interpreter extends BaseLivyInterprereter { + + public LivyPySpark3Interpreter(Properties property) { + super(property); + } + + @Override + public String getSessionKind() { + return "pyspark3"; + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a5b437d7/livy/src/main/resources/interpreter-setting.json ---------------------------------------------------------------------- diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json index 8a4e4d7..4be537a 100644 --- a/livy/src/main/resources/interpreter-setting.json +++ b/livy/src/main/resources/interpreter-setting.json @@ -155,6 +155,26 @@ }, { "group": "livy", + "name": "pyspark3", + "className": "org.apache.zeppelin.livy.LivyPySpark3Interpreter", + "properties": { + }, + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "scoped", + "isExistingProcess": false, + "setPermission": false, + "users": [] + }, + "editor": { + "language": "python", + "editOnDblClick": false + } + }, + { + "group": "livy", "name": "sparkr", "className": "org.apache.zeppelin.livy.LivySparkRInterpreter", "properties": { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a5b437d7/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 499c338..001585d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -535,6 +535,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { + "org.apache.zeppelin.livy.LivySparkInterpreter," + "org.apache.zeppelin.livy.LivySparkSQLInterpreter," + "org.apache.zeppelin.livy.LivyPySparkInterpreter," + + "org.apache.zeppelin.livy.LivyPySpark3Interpreter," + "org.apache.zeppelin.livy.LivySparkRInterpreter," + "org.apache.zeppelin.alluxio.AlluxioInterpreter," + "org.apache.zeppelin.file.HDFSFileInterpreter,"
