Repository: zeppelin Updated Branches: refs/heads/branch-0.7 6af18b7c8 -> 2aea45e66
[ZEPPELIN-2309] Backport maven and npm repo config to 0.7 ### What is this PR for? Backport configuration settings for maven central and npm repository to 0.7. ### What type of PR is it? Improvement ### What is the Jira issue? [ZEPPELIN-2309] Author: Andrea Peruffo <andrea.peruffo1...@gmail.com> Closes #2183 from andreaTP/backportConfs and squashes the following commits: bf8df5d [Andrea Peruffo] [ZEPPELIN-2278] Env variable to configure Npm registry c50c964 [Andrea Peruffo] [ZEPPELIN-2277] Env variable to configure maven central repo Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/2aea45e6 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/2aea45e6 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/2aea45e6 Branch: refs/heads/branch-0.7 Commit: 2aea45e66a99efff7682b04a7a9cc819936c70d1 Parents: 6af18b7 Author: Andrea Peruffo <andrea.peruffo1...@gmail.com> Authored: Fri Mar 17 10:59:44 2017 +0100 Committer: Lee moon soo <m...@apache.org> Committed: Fri Mar 24 14:12:30 2017 -0700 ---------------------------------------------------------------------- conf/zeppelin-env.cmd.template | 2 ++ conf/zeppelin-env.sh.template | 2 ++ conf/zeppelin-site.xml.template | 18 +++++++++++++ docs/install/configuration.md | 28 ++++++++++++++++---- .../java/org/apache/zeppelin/dep/Booter.java | 10 ++++++- .../apache/zeppelin/server/ZeppelinServer.java | 3 ++- .../zeppelin/conf/ZeppelinConfiguration.java | 23 +++++++++++----- .../helium/HeliumVisualizationFactory.java | 16 ++++++++--- .../helium/HeliumVisualizationFactoryTest.java | 8 +++++- 9 files changed, 92 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2aea45e6/conf/zeppelin-env.cmd.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template index 5fc3acf..3d06733 100644 --- a/conf/zeppelin-env.cmd.template +++ b/conf/zeppelin-env.cmd.template @@ -37,6 +37,8 @@ REM set ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION REM AWS KMS key region REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default. REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0. REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading +REM set ZEPPELIN_INTERPRETER_DEP_MVNREPO REM Maven principal repository for interpreter's additional dependency loading +REM set ZEPPELIN_HELIUM_NPM_REGISTRY REM Remote Npm registry for Helium dependency loader REM set ZEPPELIN_NOTEBOOK_STORAGE REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). REM set ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC REM If there are multiple notebook storages, should we treat the first one as the only source of truth? http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2aea45e6/conf/zeppelin-env.sh.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index 7e777b6..e1222bc 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -38,6 +38,8 @@ # export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default. # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. # export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading +# export ZEPPELIN_INTERPRETER_DEP_MVNREPO # Remote principal repository for interpreter's additional dependency loading +# export ZEPPELIN_HELIUM_NPM_REGISTRY # Remote Npm registry for Helium dependency loader # export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). # export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth? # export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2aea45e6/conf/zeppelin-site.xml.template ---------------------------------------------------------------------- diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 7faacac..1593f7f 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -199,6 +199,24 @@ </property> <property> + <name>zeppelin.interpreter.dep.mvnRepo</name> + <value>http://repo1.maven.org/maven2/</value> + <description>Remote principal repository for interpreter's additional dependency loading</description> +</property> + +<property> + <name>zeppelin.dep.localrepo</name> + <value>local-repo</value> + <description>Local repository for dependency loader</description> +</property> + +<property> + <name>zeppelin.helium.npm.registry</name> + <value>http://registry.npmjs.org/</value> + <description>Remote Npm registry for Helium dependency loader</description> +</property> + +<property> <name>zeppelin.interpreters</name> <value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlIn terpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value> <description>Comma separated interpreter configurations. First interpreter become a default</description> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2aea45e6/docs/install/configuration.md ---------------------------------------------------------------------- diff --git a/docs/install/configuration.md b/docs/install/configuration.md index 80a7555..5012803 100644 --- a/docs/install/configuration.md +++ b/docs/install/configuration.md @@ -26,7 +26,7 @@ limitations under the License. ## Zeppelin Properties There are two locations you can configure Apache Zeppelin. -* **Environment variables** can be defined `conf/zeppelin-env.sh`(`conf\zeppelin-env.cmd` for Windows). +* **Environment variables** can be defined `conf/zeppelin-env.sh`(`conf\zeppelin-env.cmd` for Windows). * **Java properties** can ba defined in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority. @@ -43,7 +43,7 @@ If both are defined, then the **environment variables** will take priority. <td>zeppelin.server.port</td> <td>8080</td> <td>Zeppelin server port </br> - <span style="font-style:italic; color: gray"> Note: Please make sure you're not using the same port with + <span style="font-style:italic; color: gray"> Note: Please make sure you're not using the same port with <a href="https://zeppelin.apache.org/contribution/webapplication.html#dev-mode"; target="_blank">Zeppelin web application development port</a> (default: 9000).</span></td> </tr> <tr> @@ -251,8 +251,26 @@ If both are defined, then the **environment variables** will take priority. <td>Interpreter directory</td> </tr> <tr> - <td>ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</td> - <td>zeppelin.websocket.max.text.message.size</td> + <td><h6 class="properties">ZEPPELIN_INTERPRETER_DEP_MVNREPO</h6></td> + <td><h6 class="properties">zeppelin.interpreter.dep.mvnRepo</h6></td> + <td>http://repo1.maven.org/maven2/</td> + <td>Remote principal repository for interpreter's additional dependency loading</td> + </tr> + <tr> + <td><h6 class="properties">ZEPPELIN_DEP_LOCALREPO</h6></td> + <td><h6 class="properties">zeppelin.dep.localrepo</h6></td> + <td>local-repo</td> + <td>Local repository for dependency loader.<br>ex)visualiztion modules of npm.</td> + </tr> + <tr> + <td><h6 class="properties">ZEPPELIN_HELIUM_NPM_REGISTRY</h6></td> + <td><h6 class="properties">zeppelin.helium.npm.registry</h6></td> + <td>http://registry.npmjs.org/</td> + <td>Remote Npm registry for Helium dependency loader</td> + </tr> + <tr> + <td><h6 class="properties">ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</h6></td> + <td><h6 class="properties">zeppelin.websocket.max.text.message.size</h6></td> <td>1024000</td> <td>Size (in characters) of the maximum text message that can be received by websocket.</td> </tr> @@ -360,7 +378,7 @@ The following properties needs to be updated in the `zeppelin-site.xml` in order ### Obfuscating Passwords using the Jetty Password Tool Security best practices advise to not use plain text passwords and Jetty provides a password tool to help obfuscating the passwords used to access the KeyStore and TrustStore. - + The Password tool documentation can be found [here](http://www.eclipse.org/jetty/documentation/current/configuring-security-secure-passwords.html). After using the tool: http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2aea45e6/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java ---------------------------------------------------------------------- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java index 0fd0ea2..5bc58ed 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java @@ -67,7 +67,15 @@ public class Booter { } public static RemoteRepository newCentralRepository() { - return new RemoteRepository("central", "default", "http://repo1.maven.org/maven2/";); + String mvnRepo = System.getenv("ZEPPELIN_INTERPRETER_DEP_MVNREPO"); + if (mvnRepo == null) { + mvnRepo = System.getProperty("zeppelin.interpreter.dep.mvnRepo"); + } + if (mvnRepo == null) { + mvnRepo = "http://repo1.maven.org/maven2/";; + } + + return new RemoteRepository("central", "default", mvnRepo); } public static RemoteRepository newLocalRepository() { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2aea45e6/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 4bcb422..8a7ae0a 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -112,11 +112,13 @@ public class ZeppelinServer extends Application { * packaged into binary package. */ heliumVisualizationFactory = new HeliumVisualizationFactory( + conf, new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)), new File(conf.getRelativeDir("lib/node_modules/zeppelin-tabledata")), new File(conf.getRelativeDir("lib/node_modules/zeppelin-vis"))); } else { heliumVisualizationFactory = new HeliumVisualizationFactory( + conf, new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)), new File(conf.getRelativeDir("zeppelin-web/src/app/tabledata")), new File(conf.getRelativeDir("zeppelin-web/src/app/visualization"))); @@ -400,4 +402,3 @@ public class ZeppelinServer extends Application { return !new File(conf.getRelativeDir("zeppelin-web")).isDirectory(); } } - http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2aea45e6/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 0c3ecac..83d37ee 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -105,7 +105,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { conf = new ZeppelinConfiguration(); } } - + LOG.info("Server Host: " + conf.getServerAddress()); if (conf.useSsl() == false) { LOG.info("Server Port: " + conf.getServerPort()); @@ -352,7 +352,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { public String getNotebookDir() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_DIR); } - + public String getUser() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_USER); } @@ -360,7 +360,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { public String getBucketName() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_BUCKET); } - + public String getEndpoint() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_ENDPOINT); } @@ -372,7 +372,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { public String getS3KMSKeyRegion() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION); } - + public String getS3EncryptionMaterialsProviderClass() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP); } @@ -401,6 +401,10 @@ public class ZeppelinConfiguration extends XMLConfiguration { return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_LOCALREGISTRY_DEFAULT); } + public String getHeliumNpmRegistry() { + return getString(ConfVars.ZEPPELIN_HELIUM_NPM_REGISTRY); + } + public String getNotebookAuthorizationPath() { return getRelativeDir(String.format("%s/notebook-authorization.json", getConfDir())); } @@ -426,6 +430,10 @@ public class ZeppelinConfiguration extends XMLConfiguration { return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO); } + public String getInterpreterMvnRepoPath() { + return getString(ConfVars.ZEPPELIN_INTERPRETER_DEP_MVNREPO); + } + public String getRelativeDir(ConfVars c) { return getRelativeDir(getString(c)); } @@ -441,7 +449,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { public boolean isWindowsPath(String path){ return path.matches("^[A-Za-z]:\\\\.*"); } - + public boolean isAnonymousAllowed() { return getBoolean(ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED); } @@ -449,7 +457,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { public boolean isNotebokPublic() { return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC); } - + public String getConfDir() { return getString(ConfVars.ZEPPELIN_CONF_DIR); } @@ -567,6 +575,8 @@ public class ZeppelinConfiguration extends XMLConfiguration { ZEPPELIN_INTERPRETER_JSON("zeppelin.interpreter.setting", "interpreter-setting.json"), ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"), ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"), + ZEPPELIN_INTERPRETER_DEP_MVNREPO("zeppelin.interpreter.dep.mvnRepo", + "http://repo1.maven.org/maven2/";), ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000), ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10), ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh," @@ -599,6 +609,7 @@ public class ZeppelinConfiguration extends XMLConfiguration { ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"), ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"), ZEPPELIN_HELIUM_LOCALREGISTRY_DEFAULT("zeppelin.helium.localregistry.default", "helium"), + ZEPPELIN_HELIUM_NPM_REGISTRY("zeppelin.helium.npm.registry", "http://registry.npmjs.org/";), // Allows a way to specify a ',' separated list of allowed origins for rest and websockets // i.e. http://localhost:8080 ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"), http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2aea45e6/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumVisualizationFactory.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumVisualizationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumVisualizationFactory.java index 316fc2a..9f63bd9 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumVisualizationFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumVisualizationFactory.java @@ -33,6 +33,8 @@ import java.io.*; import java.net.URL; import java.util.*; +import org.apache.zeppelin.conf.ZeppelinConfiguration; + /** * Load helium visualization */ @@ -40,7 +42,6 @@ public class HeliumVisualizationFactory { Logger logger = LoggerFactory.getLogger(HeliumVisualizationFactory.class); private final String NODE_VERSION = "v6.9.1"; private final String NPM_VERSION = "3.10.8"; - private final String DEFAULT_NPM_REGISTRY_URL = "http://registry.npmjs.org/";; private final int FETCH_RETRY_COUNT = 2; private final int FETCH_RETRY_FACTOR_COUNT = 1; // Milliseconds @@ -48,8 +49,10 @@ public class HeliumVisualizationFactory { private final FrontendPluginFactory frontEndPluginFactory; private final File workingDirectory; + private ZeppelinConfiguration conf; private File tabledataModulePath; private File visualizationModulePath; + private String defaultNpmRegistryUrl; private Gson gson; private boolean nodeAndNpmInstalled = false; @@ -59,16 +62,21 @@ public class HeliumVisualizationFactory { ByteArrayOutputStream out = new ByteArrayOutputStream(); public HeliumVisualizationFactory( + ZeppelinConfiguration conf, File moduleDownloadPath, File tabledataModulePath, File visualizationModulePath) throws TaskRunnerException { - this(moduleDownloadPath); + this(conf, moduleDownloadPath); this.tabledataModulePath = tabledataModulePath; this.visualizationModulePath = visualizationModulePath; } - public HeliumVisualizationFactory(File moduleDownloadPath) throws TaskRunnerException { + public HeliumVisualizationFactory( + ZeppelinConfiguration conf, + File moduleDownloadPath) throws TaskRunnerException { this.workingDirectory = new File(moduleDownloadPath, "vis"); + this.conf = conf; + this.defaultNpmRegistryUrl = conf.getHeliumNpmRegistry(); File installDirectory = workingDirectory; frontEndPluginFactory = new FrontendPluginFactory( @@ -372,7 +380,7 @@ public class HeliumVisualizationFactory { private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException { installNodeAndNpm(); - NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), DEFAULT_NPM_REGISTRY_URL); + NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), defaultNpmRegistryUrl); npm.execute(args, env); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2aea45e6/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumVisualizationFactoryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumVisualizationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumVisualizationFactoryTest.java index 4f2ee9f..9932211 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumVisualizationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumVisualizationFactoryTest.java @@ -30,10 +30,13 @@ import java.net.URL; import java.util.LinkedList; import java.util.List; +import org.apache.zeppelin.conf.ZeppelinConfiguration; + import static org.junit.Assert.*; public class HeliumVisualizationFactoryTest { private File tmpDir; + private ZeppelinConfiguration conf; private HeliumVisualizationFactory hvf; @Before @@ -46,7 +49,10 @@ public class HeliumVisualizationFactoryTest { String resDir = new File(res.getFile()).getParent(); File moduleDir = new File(resDir + "/../../../../zeppelin-web/src/app/"); - hvf = new HeliumVisualizationFactory(tmpDir, + conf = new ZeppelinConfiguration(); + + hvf = new HeliumVisualizationFactory(conf, + tmpDir, new File(moduleDir, "tabledata"), new File(moduleDir, "visualization")); }
