Added: zeppelin/site/docs/0.9.0/setup/deployment/flink_and_spark_cluster.html URL: http://svn.apache.org/viewvc/zeppelin/site/docs/0.9.0/setup/deployment/flink_and_spark_cluster.html?rev=1884775&view=auto ============================================================================== --- zeppelin/site/docs/0.9.0/setup/deployment/flink_and_spark_cluster.html (added) +++ zeppelin/site/docs/0.9.0/setup/deployment/flink_and_spark_cluster.html Thu Dec 24 14:36:01 2020 @@ -0,0 +1,620 @@ + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Apache Zeppelin 0.9.0 Documentation: Install Zeppelin with Flink and Spark in cluster mode</title> + <meta name="description" content=""> + <meta name="author" content="The Apache Software Foundation"> + + <!-- Enable responsive viewport --> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <!-- Le HTML5 shim, for IE6-8 support of HTML elements --> + <!--[if lt IE 9]> + <script src="http://html5shim.googlecode.com/svn/trunk/html5.js";></script> + <![endif]--> + + <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet"> + + <!-- Le styles --> + <link href="/docs/0.9.0/assets/themes/zeppelin/bootstrap/css/bootstrap.css" rel="stylesheet"> + <link href="/docs/0.9.0/assets/themes/zeppelin/css/style.css?body=1" rel="stylesheet" type="text/css"> + <link href="/docs/0.9.0/assets/themes/zeppelin/css/syntax.css" rel="stylesheet" type="text/css" media="screen" /> + <!-- Le fav and touch icons --> + <!-- Update these with your own images + <link rel="shortcut icon" href="images/favicon.ico"> + <link rel="apple-touch-icon" href="images/apple-touch-icon.png"> + <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png"> + <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png"> + --> + + <!-- Js --> + <script src="https://code.jquery.com/jquery-1.10.2.min.js";></script> + <script src="/docs/0.9.0/assets/themes/zeppelin/bootstrap/js/bootstrap.min.js"></script> + <script src="/docs/0.9.0/assets/themes/zeppelin/js/docs.js"></script> + <script src="/docs/0.9.0/assets/themes/zeppelin/js/anchor.min.js"></script> + <script src="/docs/0.9.0/assets/themes/zeppelin/js/toc.js"></script> + <script src="/docs/0.9.0/assets/themes/zeppelin/js/lunr.min.js"></script> + <script src="/docs/0.9.0/assets/themes/zeppelin/js/search.js"></script> + + <!-- atom & rss feed --> + <link href="/docs/0.9.0/atom.xml" type="application/atom+xml" rel="alternate" title="Sitewide ATOM Feed"> + <link href="/docs/0.9.0/rss.xml" type="application/rss+xml" rel="alternate" title="Sitewide RSS Feed"> + </head> + + <body> + + <div id="menu" class="navbar navbar-inverse navbar-fixed-top" role="navigation"> + <div class="container navbar-container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-brand"> + <a class="navbar-brand-main" href="http://zeppelin.apache.org";> + <img src="/docs/0.9.0/assets/themes/zeppelin/img/zeppelin_logo.png" width="50" + style="margin-top: -2px;" alt="I'm zeppelin"> + <span style="margin-left: 5px; font-size: 27px;">Zeppelin</span> + <a class="navbar-brand-version" href="/docs/0.9.0" + style="font-size: 15px; color: white;"> 0.9.0 + </a> + </a> + </div> + </div> + <nav class="navbar-collapse collapse" role="navigation"> + <ul class="nav navbar-nav"> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Quick Start <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li class="title"><span>Getting Started</span></li> + <li><a href="/docs/0.9.0/quickstart/install.html">Install</a></li> + <li><a href="/docs/0.9.0/quickstart/explore_ui.html">Explore UI</a></li> + <li><a href="/docs/0.9.0/quickstart/tutorial.html">Tutorial</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Run Mode</span></li> + <li><a href="/docs/0.9.0/quickstart/kubernetes.html">Kubernetes</a></li> + <li><a href="/docs/0.9.0/quickstart/docker.html">Docker</a></li> + <li><a href="/docs/0.9.0/quickstart/yarn.html">Yarn</a></li> + <li role="separator" class="divider"></li> + <li><a href="/docs/0.9.0/quickstart/spark_with_zeppelin.html">Spark with Zeppelin</a></li> + <li><a href="/docs/0.9.0/quickstart/sql_with_zeppelin.html">SQL with Zeppelin</a></li> + <li><a href="/docs/0.9.0/quickstart/python_with_zeppelin.html">Python with Zeppelin</a></li> + </ul> + </li> + + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Usage<b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu"> + <li class="title"><span>Dynamic Form</span></li> + <li><a href="/docs/0.9.0/usage/dynamic_form/intro.html">What is Dynamic Form?</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Display System</span></li> + <li><a href="/docs/0.9.0/usage/display_system/basic.html#text">Text Display</a></li> + <li><a href="/docs/0.9.0/usage/display_system/basic.html#html">HTML Display</a></li> + <li><a href="/docs/0.9.0/usage/display_system/basic.html#table">Table Display</a></li> + <li><a href="/docs/0.9.0/usage/display_system/basic.html#network">Network Display</a></li> + <li><a href="/docs/0.9.0/usage/display_system/angular_backend.html">Angular Display using Backend API</a></li> + <li><a href="/docs/0.9.0/usage/display_system/angular_frontend.html">Angular Display using Frontend API</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Interpreter</span></li> + <li><a href="/docs/0.9.0/usage/interpreter/overview.html">Overview</a></li> + <li><a href="/docs/0.9.0/usage/interpreter/interpreter_binding_mode.html">Interpreter Binding Mode</a></li> + <li><a href="/docs/0.9.0/usage/interpreter/user_impersonation.html">User Impersonation</a></li> + <li><a href="/docs/0.9.0/usage/interpreter/dependency_management.html">Dependency Management</a></li> + <li><a href="/docs/0.9.0/usage/interpreter/installation.html">Installing Interpreters</a></li> + <!--<li><a href="/docs/0.9.0/usage/interpreter/dynamic_loading.html">Dynamic Interpreter Loading (Experimental)</a></li>--> + <li><a href="/docs/0.9.0/usage/interpreter/execution_hooks.html">Execution Hooks (Experimental)</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Other Features</span></li> + <li><a href="/docs/0.9.0/usage/other_features/publishing_paragraphs.html">Publishing Paragraphs</a></li> + <li><a href="/docs/0.9.0/usage/other_features/personalized_mode.html">Personalized Mode</a></li> + <li><a href="/docs/0.9.0/usage/other_features/customizing_homepage.html">Customizing Zeppelin Homepage</a></li> + <li><a href="/docs/0.9.0/usage/other_features/notebook_actions.html">Notebook Actions</a></li> + <li><a href="/docs/0.9.0/usage/other_features/cron_scheduler.html">Cron Scheduler</a></li> + <li><a href="/docs/0.9.0/usage/other_features/zeppelin_context.html">Zeppelin Context</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>REST API</span></li> + <li><a href="/docs/0.9.0/usage/rest_api/interpreter.html">Interpreter API</a></li> + <li><a href="/docs/0.9.0/usage/rest_api/zeppelin_server.html">Zeppelin Server API</a></li> + <li><a href="/docs/0.9.0/usage/rest_api/notebook.html">Notebook API</a></li> + <li><a href="/docs/0.9.0/usage/rest_api/notebook_repository.html">Notebook Repository API</a></li> + <li><a href="/docs/0.9.0/usage/rest_api/configuration.html">Configuration API</a></li> + <li><a href="/docs/0.9.0/usage/rest_api/credential.html">Credential API</a></li> + <li><a href="/docs/0.9.0/usage/rest_api/helium.html">Helium API</a></li> + </ul> + </li> + + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Setup<b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu"> + <li class="title"><span>Basics</span></li> + <li><a href="/docs/0.9.0/setup/basics/how_to_build.html">How to Build Zeppelin</a></li> + <li><a href="/docs/0.9.0/setup/basics/hadoop_integration.html">Hadoop Integration</a></li> + <li><a href="/docs/0.9.0/setup/basics/multi_user_support.html">Multi-user Support</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Deployment</span></li> + <!--<li><a href="/docs/0.9.0/setup/deployment/docker.html">Docker Image for Zeppelin</a></li>--> + <li><a href="/docs/0.9.0/setup/deployment/spark_cluster_mode.html#spark-standalone-mode">Spark Cluster Mode: Standalone</a></li> + <li><a href="/docs/0.9.0/setup/deployment/spark_cluster_mode.html#spark-on-yarn-mode">Spark Cluster Mode: YARN</a></li> + <li><a href="/docs/0.9.0/setup/deployment/spark_cluster_mode.html#spark-on-mesos-mode">Spark Cluster Mode: Mesos</a></li> + <li><a href="/docs/0.9.0/setup/deployment/flink_and_spark_cluster.html">Zeppelin with Flink, Spark Cluster</a></li> + <li><a href="/docs/0.9.0/setup/deployment/cdh.html">Zeppelin on CDH</a></li> + <li><a href="/docs/0.9.0/setup/deployment/virtual_machine.html">Zeppelin on VM: Vagrant</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Security</span></li> + <li><a href="/docs/0.9.0/setup/security/authentication_nginx.html">HTTP Basic Auth using NGINX</a></li> + <li><a href="/docs/0.9.0/setup/security/shiro_authentication.html">Shiro Authentication</a></li> + <li><a href="/docs/0.9.0/setup/security/notebook_authorization.html">Notebook Authorization</a></li> + <li><a href="/docs/0.9.0/setup/security/datasource_authorization.html">Data Source Authorization</a></li> + <li><a href="/docs/0.9.0/setup/security/http_security_headers.html">HTTP Security Headers</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Notebook Storage</span></li> + <li><a href="/docs/0.9.0/setup/storage/storage.html#notebook-storage-in-local-git-repository">Git Storage</a></li> + <li><a href="/docs/0.9.0/setup/storage/storage.html#notebook-storage-in-s3">S3 Storage</a></li> + <li><a href="/docs/0.9.0/setup/storage/storage.html#notebook-storage-in-azure">Azure Storage</a></li> + <li><a href="/docs/0.9.0/setup/storage/storage.html#notebook-storage-in-oss">OSS Storage</a></li> + <li><a href="/docs/0.9.0/setup/storage/storage.html#notebook-storage-in-zeppelinhub">ZeppelinHub Storage</a></li> + <li><a href="/docs/0.9.0/setup/storage/storage.html#notebook-storage-in-mongodb">MongoDB Storage</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Operation</span></li> + <li><a href="/docs/0.9.0/setup/operation/configuration.html">Configuration</a></li> + <li><a href="/docs/0.9.0/setup/operation/proxy_setting.html">Proxy Setting</a></li> + <li><a href="/docs/0.9.0/setup/operation/upgrading.html">Upgrading</a></li> + <li><a href="/docs/0.9.0/setup/operation/trouble_shooting.html">Trouble Shooting</a></li> + </ul> + </li> + + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Interpreter <b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu"> + <li class="title"><span>Interpreters</span></li> + <li><a href="/docs/0.9.0/usage/interpreter/overview.html">Overview</a></li> + <li role="separator" class="divider"></li> + <li><a href="/docs/0.9.0/interpreter/spark.html">Spark</a></li> + <li><a href="/docs/0.9.0/interpreter/jdbc.html">JDBC</a></li> + <li><a href="/docs/0.9.0/interpreter/python.html">Python</a></li> + <li><a href="/docs/0.9.0/interpreter/r.html">R</a></li> + <li role="separator" class="divider"></li> + <li><a href="/docs/0.9.0/interpreter/alluxio.html">Alluxio</a></li> + <li><a href="/docs/0.9.0/interpreter/beam.html">Beam</a></li> + <li><a href="/docs/0.9.0/interpreter/bigquery.html">BigQuery</a></li> + <li><a href="/docs/0.9.0/interpreter/cassandra.html">Cassandra</a></li> + <li><a href="/docs/0.9.0/interpreter/elasticsearch.html">Elasticsearch</a></li> + <li><a href="/docs/0.9.0/interpreter/flink.html">Flink</a></li> + <li><a href="/docs/0.9.0/interpreter/geode.html">Geode</a></li> + <li><a href="/docs/0.9.0/interpreter/groovy.html">Groovy</a></li> + <li><a href="/docs/0.9.0/interpreter/hazelcastjet.html">Hazelcast Jet</a></li> + <li><a href="/docs/0.9.0/interpreter/hbase.html">HBase</a></li> + <li><a href="/docs/0.9.0/interpreter/hdfs.html">HDFS</a></li> + <li><a href="/docs/0.9.0/interpreter/hive.html">Hive</a></li> + <li><a href="/docs/0.9.0/interpreter/ignite.html">Ignite</a></li> + <li><a href="/docs/0.9.0/interpreter/java.html">Java</a></li> + <li><a href="/docs/0.9.0/interpreter/jupyter.html">Jupyter</a></li> + <li><a href="/docs/0.9.0/interpreter/kotlin.html">Kotlin</a></li> + <li><a href="/docs/0.9.0/interpreter/kylin.html">Kylin</a></li> + <li><a href="/docs/0.9.0/interpreter/lens.html">Lens</a></li> + <li><a href="/docs/0.9.0/interpreter/livy.html">Livy</a></li> + <li><a href="/docs/0.9.0/interpreter/markdown.html">Markdown</a></li> + <li><a href="/docs/0.9.0/interpreter/mongodb.html">MongoDB</a></li> + <li><a href="/docs/0.9.0/interpreter/neo4j.html">Neo4j</a></li> + <li><a href="/docs/0.9.0/interpreter/pig.html">Pig</a></li> + <li><a href="/docs/0.9.0/interpreter/postgresql.html">Postgresql, HAWQ</a></li> + <li><a href="/docs/0.9.0/interpreter/scalding.html">Scalding</a></li> + <li><a href="/docs/0.9.0/interpreter/scio.html">Scio</a></li> + <li><a href="/docs/0.9.0/interpreter/shell.html">Shell</a></li> + <li><a href="/docs/0.9.0/interpreter/sparql.html">Sparql</a></li> + <li><a href="/docs/0.9.0/interpreter/submarine.html">Submarine</a></li> + </ul> + </li> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">More<b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu" style="right: 0; left: auto;"> + <li class="title"><span>Extending Zeppelin</span></li> + <li><a href="/docs/0.9.0/development/writing_zeppelin_interpreter.html">Writing Zeppelin Interpreter</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Helium (Experimental)</span></li> + <li><a href="/docs/0.9.0/development/helium/overview.html">Overview</a></li> + <li><a href="/docs/0.9.0/development/helium/writing_application.html">Writing Helium Application</a></li> + <li><a href="/docs/0.9.0/development/helium/writing_spell.html">Writing Helium Spell</a></li> + <li><a href="/docs/0.9.0/development/helium/writing_visualization_basic.html">Writing Helium Visualization: Basics</a></li> + <li><a href="/docs/0.9.0/development/helium/writing_visualization_transformation.html">Writing Helium Visualization: Transformation</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Contributing to Zeppelin</span></li> + <li><a href="/docs/0.9.0/setup/basics/how_to_build.html">How to Build Zeppelin</a></li> + <li><a href="/docs/0.9.0/development/contribution/useful_developer_tools.html">Useful Developer Tools</a></li> + <li><a href="/docs/0.9.0/development/contribution/how_to_contribute_code.html">How to Contribute (code)</a></li> + <li><a href="/docs/0.9.0/development/contribution/how_to_contribute_website.html">How to Contribute (website)</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>External Resources</span></li> + <li><a target="_blank" href="https://zeppelin.apache.org/community.html";>Mailing List</a></li> + <li><a target="_blank" href="https://cwiki.apache.org/confluence/display/ZEPPELIN/Zeppelin+Home";>Apache Zeppelin Wiki</a></li> + <li><a target="_blank" href="http://stackoverflow.com/questions/tagged/apache-zeppelin";>Stackoverflow Questions about Zeppelin</a></li> + </ul> + </li> + <li> + <a href="/docs/0.9.0/search.html" class="nav-search-link"> + <span class="fa fa-search nav-search-icon"></span> + </a> + </li> + </ul> + </nav><!--/.navbar-collapse --> + </div> + </div> + + + + <div class="content"> + +<!--<div class="hero-unit Install Zeppelin with Flink and Spark in cluster mode"> + <h1></h1> +</div> +--> + +<div class="row"> + <div class="col-md-12"> + <!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +<h1>Install with Flink and Spark cluster</h1> + +<div id="toc"></div> + +<p>This tutorial is extremely entry-level. It assumes no prior knowledge of Linux, git, or other tools. If you carefully type what I tell you when I tell you, you should be able to get Zeppelin running.</p> + +<h2>Installing Zeppelin with Flink and Spark in cluster mode</h2> + +<p>This tutorial assumes the user has a machine (real or <a href="https://www.virtualbox.org/wiki/Downloads";>virtual</a> with a fresh, minimal installation of <a href="http://www.ubuntu.com/download/server";>Ubuntu 14.04.3 Server</a>.</p> + +<p><strong>Note:</strong> On the size requirements of the Virtual Machine, some users reported trouble when using the default virtual machine sizes, specifically that the hard drive needed to be at least 16GB- other users did not have this issue.</p> + +<p>There are many good tutorials on how to install Ubuntu Server on a virtual box, <a href="http://ilearnstack.com/2013/04/13/setting-ubuntu-vm-in-virtualbox/";>here is one of them</a></p> + +<h3>Required Programs</h3> + +<p>Assuming the minimal install, there are several programs that we will need to install before Zeppelin, Flink, and Spark.</p> + +<ul> +<li>git</li> +<li>openssh-server</li> +<li>OpenJDK 7</li> +<li>Maven 3.1+</li> +</ul> + +<p>For git, openssh-server, and OpenJDK 7 we will be using the apt package manager.</p> + +<h5>git</h5> + +<p>From the command prompt:</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">sudo apt-get install git +</code></pre></div> +<h5>openssh-server</h5> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">sudo apt-get install openssh-server +</code></pre></div> +<h5>OpenJDK 7</h5> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">sudo apt-get install openjdk-7-jdk openjdk-7-jre-lib +</code></pre></div> +<p><em>A note for those using Ubuntu 16.04</em>: To install <code>openjdk-7</code> on Ubuntu 16.04, one must add a repository. <a href="http://askubuntu.com/questions/761127/ubuntu-16-04-and-openjdk-7";>Source</a></p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">sudo add-apt-repository ppa:openjdk-r/ppa +sudo apt-get update +sudo apt-get install openjdk-7-jdk openjdk-7-jre-lib +</code></pre></div> +<h5>Maven 3.1+</h5> + +<p>Zeppelin requires maven version 3.x. The version available in the repositories at the time of writing is 2.x, so maven must be installed manually.</p> + +<p>Purge any existing versions of maven.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">sudo apt-get purge maven maven2 +</code></pre></div> +<p>Download the maven 3.3.9 binary.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">wget <span class="s2">"http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz"</span> +</code></pre></div> +<p>Unarchive the binary and move to the <code>/usr/local</code> directory.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">tar -zxvf apache-maven-3.3.9-bin.tar.gz +sudo mv ./apache-maven-3.3.9 /usr/local +</code></pre></div> +<p>Create symbolic links in <code>/usr/bin</code>.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">sudo ln -s /usr/local/apache-maven-3.3.9/bin/mvn /usr/bin/mvn +</code></pre></div> +<h3>Installing Zeppelin</h3> + +<p>This provides a quick overview of Zeppelin installation from source, however the reader is encouraged to review the <a href="../../quickstart/install.html">Zeppelin Installation Guide</a></p> + +<p>From the command prompt: +Clone Zeppelin.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">git clone https://github.com/apache/zeppelin.git +</code></pre></div> +<p>Enter the Zeppelin root directory.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash"><span class="nb">cd </span>zeppelin +</code></pre></div> +<p>Package Zeppelin.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">mvn clean package -DskipTests -Pspark-1.6 -Dflink.version<span class="o">=</span>1.1.3 -Pscala-2.10 +</code></pre></div> +<p><code>-DskipTests</code> skips build tests- you're not developing (yet), so you don't need to do tests, the clone version <em>should</em> build.</p> + +<p><code>-Pspark-1.6</code> tells maven to build a Zeppelin with Spark 1.6. This is important because Zeppelin has its own Spark interpreter and the versions must be the same.</p> + +<p><code>-Dflink.version=1.1.3</code> tells maven specifically to build Zeppelin with Flink version 1.1.3.</p> + +<p>-<code>-Pscala-2.10</code> tells maven to build with Scala v2.10.</p> + +<p><strong>Note:</strong> You can build against any version of Spark that has a Zeppelin build profile available. The key is to make sure you check out the matching version of Spark to build. At the time of this writing, Spark 1.6 was the most recent Spark version available.</p> + +<p><strong>Note:</strong> On build failures. Having installed Zeppelin close to 30 times now, I will tell you that sometimes the build fails for seemingly no reason. +As long as you didn't edit any code, it is unlikely the build is failing because of something you did. What does tend to happen, is some dependency that maven is trying to download is unreachable. If your build fails on this step here are some tips:</p> + +<ul> +<li>Don't get discouraged.</li> +<li>Scroll up and read through the logs. There will be clues there.</li> +<li>Retry (that is, run the <code>mvn clean package -DskipTests -Pspark-1.6</code> again)</li> +<li>If there were clues that a dependency couldn't be downloaded wait a few hours or even days and retry again. Open source software when compiling is trying to download all of the dependencies it needs, if a server is off-line there is nothing you can do but wait for it to come back.</li> +<li>Make sure you followed all of the steps carefully.</li> +<li>Ask the community to help you. Go <a href="http://zeppelin.apache.org/community.html";>here</a> and join the user mailing list. People are there to help you. Make sure to copy and paste the build output (everything that happened in the console) and include that in your message.</li> +</ul> + +<p>Start the Zeppelin daemon.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">bin/zeppelin-daemon.sh start +</code></pre></div> +<p>Use <code>ifconfig</code> to determine the host machine's IP address. If you are not familiar with how to do this, a fairly comprehensive post can be found <a href="http://www.cyberciti.biz/faq/how-to-find-out-the-ip-address-assigned-to-eth0-and-display-ip-only/";>here</a>.</p> + +<p>Open a web-browser on a machine connected to the same network as the host (or in the host operating system if using a virtual machine). Navigate to http://<code>yourip</code>:8080, where yourip is the IP address you found in <code>ifconfig</code>.</p> + +<p>See the <a href="../../quickstart/tutorial.html">Zeppelin tutorial</a> for basic Zeppelin usage. It is also advised that you take a moment to check out the tutorial notebook that is included with each Zeppelin install, and to familiarize yourself with basic notebook functionality.</p> + +<h5>Flink Test</h5> + +<p>Create a new notebook named "Flink Test" and copy and paste the following code.</p> +<div class="highlight"><pre><code class="scala language-scala" data-lang="scala"><span class="o">%</span><span class="n">flink</span> <span class="c1">// let Zeppelin know what interpreter to use.</span> + +<span class="k">val</span> <span class="n">text</span> <span class="k">=</span> <span class="n">benv</span><span class="o">.</span><span class="n">fromElements</span><span class="o">(</span><span class="s">"In the time of chimpanzees, I was a monkey"</span><span class="o">,</span> <span class="c1">// some lines of text to analyze</span> +<span class="s">"Butane in my veins and I'm out to cut the junkie"</span><span class="o">,</span> +<span class="s">"With the plastic eyeballs, spray paint the vegetables"</span><span class="o">,</span> +<span class="s">"Dog food stalls with the beefcake pantyhose"</span><span class="o">,</span> +<span class="s">"Kill the headlights and put it in neutral"</span><span class="o">,</span> +<span class="s">"Stock car flamin' with a loser in the cruise control"</span><span class="o">,</span> +<span class="s">"Baby's in Reno with the Vitamin D"</span><span class="o">,</span> +<span class="s">"Got a couple of couches, sleep on the love seat"</span><span class="o">,</span> +<span class="s">"Someone came in sayin' I'm insane to complain"</span><span class="o">,</span> +<span class="s">"About a shotgun wedding and a stain on my shirt"</span><span class="o">,</span> +<span class="s">"Don't believe everything that you breathe"</span><span class="o">,</span> +<span class="s">"You get a parking violation and a maggot on your sleeve"</span><span class="o">,</span> +<span class="s">"So shave your face with some mace in the dark"</span><span class="o">,</span> +<span class="s">"Savin' all your food stamps and burnin' down the trailer park"</span><span class="o">,</span> +<span class="s">"Yo, cut it"</span><span class="o">)</span> + +<span class="cm">/* The meat and potatoes:</span> +<span class="cm"> this tells Flink to iterate through the elements, in this case strings,</span> +<span class="cm"> transform the string to lower case and split the string at white space into individual words</span> +<span class="cm"> then finally aggregate the occurrence of each word.</span> + +<span class="cm"> This creates the count variable which is a list of tuples of the form (word, occurances)</span> + +<span class="cm">counts.collect().foreach(println(_)) // execute the script and print each element in the counts list</span> + +<span class="cm">*/</span> +<span class="k">val</span> <span class="n">counts</span> <span class="k">=</span> <span class="n">text</span><span class="o">.</span><span class="n">flatMap</span><span class="o">{</span> <span class="k">_</span><span class="o">.</span><span class="n">toLowerCase</span><span class="o">.</span><span class="n">split</span><span class="o">(</span><span class="s">"\\W+"</span><span class="o">)</span> <span class="o">}.</span><span class="n">map</span> <span class="o">{</span> <span class="o">(</span><span class="k">_</span><span class="o">,</span><span class="mi">1</span><span class="o">)</span> <span class="o">}.</span><span class="n">groupBy</span><span class="o">(</span><span class="mi">0</span><span class="o">).</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span> + +<span class="n">counts</span><span class="o">.</span><span class="n">collect</span><span class="o">().</span><span class="n">foreach</span><span class="o">(</span><span class="n">println</span><span class="o">(</span><span class="k">_</span><span class="o">))</span> <span class="c1">// execute the script and print each element in the counts list</span> +</code></pre></div> +<p>Run the code to make sure the built-in Zeppelin Flink interpreter is working properly.</p> + +<h5>Spark Test</h5> + +<p>Create a new notebook named "Spark Test" and copy and paste the following code.</p> +<div class="highlight"><pre><code class="scala language-scala" data-lang="scala"><span class="o">%</span><span class="n">spark</span> <span class="c1">// let Zeppelin know what interpreter to use.</span> + +<span class="k">val</span> <span class="n">text</span> <span class="k">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="o">(</span><span class="nc">List</span><span class="o">(</span><span class="s">"In the time of chimpanzees, I was a monkey"</span><span class="o">,</span> <span class="c1">// some lines of text to analyze</span> +<span class="s">"Butane in my veins and I'm out to cut the junkie"</span><span class="o">,</span> +<span class="s">"With the plastic eyeballs, spray paint the vegetables"</span><span class="o">,</span> +<span class="s">"Dog food stalls with the beefcake pantyhose"</span><span class="o">,</span> +<span class="s">"Kill the headlights and put it in neutral"</span><span class="o">,</span> +<span class="s">"Stock car flamin' with a loser in the cruise control"</span><span class="o">,</span> +<span class="s">"Baby's in Reno with the Vitamin D"</span><span class="o">,</span> +<span class="s">"Got a couple of couches, sleep on the love seat"</span><span class="o">,</span> +<span class="s">"Someone came in sayin' I'm insane to complain"</span><span class="o">,</span> +<span class="s">"About a shotgun wedding and a stain on my shirt"</span><span class="o">,</span> +<span class="s">"Don't believe everything that you breathe"</span><span class="o">,</span> +<span class="s">"You get a parking violation and a maggot on your sleeve"</span><span class="o">,</span> +<span class="s">"So shave your face with some mace in the dark"</span><span class="o">,</span> +<span class="s">"Savin' all your food stamps and burnin' down the trailer park"</span><span class="o">,</span> +<span class="s">"Yo, cut it"</span><span class="o">))</span> + + +<span class="cm">/* The meat and potatoes:</span> +<span class="cm"> this tells spark to iterate through the elements, in this case strings,</span> +<span class="cm"> transform the string to lower case and split the string at white space into individual words</span> +<span class="cm"> then finally aggregate the occurrence of each word.</span> + +<span class="cm"> This creates the count variable which is a list of tuples of the form (word, occurances)</span> +<span class="cm">*/</span> +<span class="k">val</span> <span class="n">counts</span> <span class="k">=</span> <span class="n">text</span><span class="o">.</span><span class="n">flatMap</span> <span class="o">{</span> <span class="k">_</span><span class="o">.</span><span class="n">toLowerCase</span><span class="o">.</span><span class="n">split</span><span class="o">(</span><span class="s">"\\W+"</span><span class="o">)</span> <span class="o">}</span> + <span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="o">(</span><span class="k">_</span><span class="o">,</span><span class="mi">1</span><span class="o">)</span> <span class="o">}</span> + <span class="o">.</span><span class="n">reduceByKey</span><span class="o">(</span><span class="k">_</span> <span class="o">+</span> <span class="k">_</span><span class="o">)</span> + +<span class="n">counts</span><span class="o">.</span><span class="n">collect</span><span class="o">().</span><span class="n">foreach</span><span class="o">(</span><span class="n">println</span><span class="o">(</span><span class="k">_</span><span class="o">))</span> <span class="c1">// execute the script and print each element in the counts list</span> +</code></pre></div> +<p>Run the code to make sure the built-in Zeppelin Flink interpreter is working properly.</p> + +<p>Finally, stop the Zeppelin daemon. From the command prompt run:</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">bin/zeppelin-daemon.sh stop +</code></pre></div> +<h3>Installing Clusters</h3> + +<h5>Flink Cluster</h5> + +<h6>Download Binaries</h6> + +<p>Building from source is recommended where possible, for simplicity in this tutorial we will download Flink and Spark Binaries.</p> + +<p>To download the Flink Binary use <code>wget</code></p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">wget <span class="s2">"http://mirror.cogentco.com/pub/apache/flink/flink-1.1.3/flink-1.1.3-bin-hadoop24-scala_2.10.tgz"</span> +tar -xzvf flink-1.1.3-bin-hadoop24-scala_2.10.tgz +</code></pre></div> +<p>This will download Flink 1.1.3, compatible with Hadoop 2.4. You do not have to install Hadoop for this binary to work, but if you are using Hadoop, please change <code>24</code> to your appropriate version.</p> + +<p>Start the Flink Cluster.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">flink-1.1.3/bin/start-cluster.sh +</code></pre></div> +<h6>Building From source</h6> + +<p>If you wish to build Flink from source, the following will be instructive. Note that if you have downloaded and used the binary version this should be skipped. The changing nature of build tools and versions across platforms makes this section somewhat precarious. For example, Java8 and Maven 3.0.3 are recommended for building Flink, which are not recommended for Zeppelin at the time of writing. If the user wishes to attempt to build from source, this section will provide some reference. If errors are encountered, please contact the Apache Flink community.</p> + +<p>See the <a href="https://github.com/apache/flink/blob/master/README.md";>Flink Installation guide</a> for more detailed instructions.</p> + +<p>Return to the directory where you have been downloading, this tutorial assumes that is <code>$HOME</code>. Clone Flink, check out release-1.1.3-rc2, and build.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash"><span class="nb">cd</span> <span class="nv">$HOME</span> +git clone https://github.com/apache/flink.git +<span class="nb">cd </span>flink +git checkout release-1.1.3-rc2 +mvn clean install -DskipTests +</code></pre></div> +<p>Start the Flink Cluster in stand-alone mode</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">build-target/bin/start-cluster.sh +</code></pre></div> +<h6>Ensure the cluster is up</h6> + +<p>In a browser, navigate to http://<code>yourip</code>:8082 to see the Flink Web-UI. Click on 'Task Managers' in the left navigation bar. Ensure there is at least one Task Manager present.</p> + +<p><center><img src="/docs/0.9.0/assets/themes/zeppelin/img/screenshots/flink-webui.png" alt="alt text" title="The Flink Web-UI"></center></p> + +<p>If no task managers are present, restart the Flink cluster with the following commands:</p> + +<p>(if binaries)</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">flink-1.1.3/bin/stop-cluster.sh +flink-1.1.3/bin/start-cluster.sh +</code></pre></div> +<p>(if built from source)</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">build-target/bin/stop-cluster.sh +build-target/bin/start-cluster.sh +</code></pre></div> +<h5>Spark 1.6 Cluster</h5> + +<h6>Download Binaries</h6> + +<p>Building from source is recommended where possible, for simplicity in this tutorial we will download Flink and Spark Binaries.</p> + +<p>Using binaries is also</p> + +<p>To download the Spark Binary use <code>wget</code></p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">wget <span class="s2">"http://d3kbcqa49mib13.cloudfront.net/spark-1.6.3-bin-hadoop2.6.tgz"</span> +tar -xzvf spark-1.6.3-bin-hadoop2.6.tgz +mv spark-1.6.3-bin-hadoop2.6 spark +</code></pre></div> +<p>This will download Spark 1.6.3, compatible with Hadoop 2.6. You do not have to install Hadoop for this binary to work, but if you are using Hadoop, please change <code>2.6</code> to your appropriate version.</p> + +<h6>Building From source</h6> + +<p>Spark is an extraordinarily large project, which takes considerable time to download and build. It is also prone to build failures for similar reasons listed in the Flink section. If the user wishes to attempt to build from source, this section will provide some reference. If errors are encountered, please contact the Apache Spark community.</p> + +<p>See the <a href="https://github.com/apache/spark/blob/master/README.md";>Spark Installation</a> guide for more detailed instructions.</p> + +<p>Return to the directory where you have been downloading, this tutorial assumes that is $HOME. Clone Spark, check out branch-1.6, and build. +<strong>Note:</strong> Recall, we're only checking out 1.6 because it is the most recent Spark for which a Zeppelin profile exists at + the time of writing. You are free to check out other version, just make sure you build Zeppelin against the correct version of Spark. However if you use Spark 2.0, the word count example will need to be changed as Spark 2.0 is not compatible with the following examples.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash"><span class="nb">cd</span> <span class="nv">$HOME</span> +</code></pre></div> +<p>Clone, check out, and build Spark version 1.6.x.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">git clone https://github.com/apache/spark.git +<span class="nb">cd </span>spark +git checkout branch-1.6 +mvn clean package -DskipTests +</code></pre></div> +<h6>Start the Spark cluster</h6> + +<p>Return to the <code>$HOME</code> directory.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash"><span class="nb">cd</span> <span class="nv">$HOME</span> +</code></pre></div> +<p>Start the Spark cluster in stand alone mode, specifying the webui-port as some port other than 8080 (the webui-port of Zeppelin).</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">spark/sbin/start-master.sh --webui-port 8082 +</code></pre></div> +<p><strong>Note:</strong> Why <code>--webui-port 8082</code>? There is a digression toward the end of this document that explains this.</p> + +<p>Open a browser and navigate to http://<code>yourip</code>:8082 to ensure the Spark master is running.</p> + +<p><center><img src="/docs/0.9.0/assets/themes/zeppelin/img/screenshots/spark-master-webui1.png" alt="alt text" title="It should look like this..."></center></p> + +<p>Toward the top of the page there will be a <em>URL</em>: spark://<code>yourhost</code>:7077. Note this URL, the Spark Master URI, it will be needed in subsequent steps.</p> + +<p>Start the slave using the URI from the Spark master WebUI:</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">spark/sbin/start-slave.sh spark://yourhostname:7077 +</code></pre></div> +<p>Return to the root directory and start the Zeppelin daemon.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash"><span class="nb">cd</span> <span class="nv">$HOME</span> + +zeppelin/bin/zeppelin-daemon.sh start +</code></pre></div> +<h5>Configure Interpreters</h5> + +<p>Open a web browser and go to the Zeppelin web-ui at http://yourip:8080.</p> + +<p>Now go back to the Zeppelin web-ui at http://<code>yourip</code>:8080 and this time click on <em>anonymous</em> at the top right, which will open a drop-down menu, select <em>Interpreters</em> to enter interpreter configuration.</p> + +<p>In the Spark section, click the edit button in the top right corner to make the property values editable (looks like a pencil). +The only field that needs to be edited in the Spark interpreter is the <code>spark.master</code> field. Change this value from <code>local[*]</code> to the URL you used to start the slave, mine was <code>spark://ubuntu:7077</code>.</p> + +<p>Click <em>Save</em> to update the parameters, and click <em>OK</em> when it asks you about restarting the interpreter.</p> + +<p>Now scroll down to the Flink section. Click the edit button and change the value of <em>host</em> from <code>local</code> to <code>localhost</code>. Click <em>Save</em> again.</p> + +<p>Reopen the examples and execute them again (I.e. you need to click the play button at the top of the screen, or the button on the paragraph .</p> + +<p>You should be able check the Flink and Spark webuis (at something like http://<code>yourip</code>:8081, http://<code>yourip</code>:8082, http://<code>yourip</code>:8083) and see that jobs have been run against the clusters.</p> + +<p><strong>Digression</strong> Sorry to be vague and use terms such as 'something like', but exactly what web-ui is at what port is going to depend on what order you started things. + What is really going on here is you are pointing your browser at specific ports, namely 8081, 8082, and 8083. Flink and Spark all want to put their web-ui on port 8080, but are + well behaved and will take the next port available. Since Zeppelin started first, it will get port 8080. When Flink starts (assuming you started Flink first), it will try to bind to + port 8080, see that it is already taken, and go to the next one available, hopefully 8081. Spark has a webui for the master and the slave, so when they start they will try to bind to 8080 + already taken by Zeppelin), then 8081 (already taken by Flink's webui), then 8082. If everything goes smoothy and you followed the directions precisely, the webuis should be 8081 and 8082. + It <em>is</em> possible to specify the port you want the webui to bind to (at the command line by passing the <code>--webui-port <port></code> flag when you start the Flink and Spark, where <code><port></code> is the port + you want to see that webui on. You can also set the default webui port of Spark and Flink (and Zeppelin) in the configuration files, but this is a tutorial for novices and slightly out of scope.</p> + +<h3>Next Steps</h3> + +<p>Check out the <a href="../../quickstart/tutorial.html">tutorial</a> for more cool things you can do with your new toy!</p> + +<p><a href="http://zeppelin.apache.org/community.html";>Join the community</a>, ask questions and contribute! Every little bit helps.</p> + + </div> +</div> + + + <hr> + <footer> + <!-- <p>© 2020 The Apache Software Foundation</p>--> + </footer> + </div> + + + + + <script type="text/javascript"> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-45176241-5', 'zeppelin.apache.org'); + ga('require', 'linkid', 'linkid.js'); + ga('send', 'pageview'); + +</script> + + + + </body> +</html> +
Added: zeppelin/site/docs/0.9.0/setup/deployment/spark_cluster_mode.html URL: http://svn.apache.org/viewvc/zeppelin/site/docs/0.9.0/setup/deployment/spark_cluster_mode.html?rev=1884775&view=auto ============================================================================== --- zeppelin/site/docs/0.9.0/setup/deployment/spark_cluster_mode.html (added) +++ zeppelin/site/docs/0.9.0/setup/deployment/spark_cluster_mode.html Thu Dec 24 14:36:01 2020 @@ -0,0 +1,488 @@ + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Apache Zeppelin 0.9.0 Documentation: Apache Zeppelin on Spark cluster mode</title> + <meta name="description" content="This document will guide you how you can build and configure the environment on 3 types of Spark cluster manager(Standalone, Hadoop Yarn, Apache Mesos) with Apache Zeppelin using docker scripts."> + <meta name="author" content="The Apache Software Foundation"> + + <!-- Enable responsive viewport --> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <!-- Le HTML5 shim, for IE6-8 support of HTML elements --> + <!--[if lt IE 9]> + <script src="http://html5shim.googlecode.com/svn/trunk/html5.js";></script> + <![endif]--> + + <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet"> + + <!-- Le styles --> + <link href="/docs/0.9.0/assets/themes/zeppelin/bootstrap/css/bootstrap.css" rel="stylesheet"> + <link href="/docs/0.9.0/assets/themes/zeppelin/css/style.css?body=1" rel="stylesheet" type="text/css"> + <link href="/docs/0.9.0/assets/themes/zeppelin/css/syntax.css" rel="stylesheet" type="text/css" media="screen" /> + <!-- Le fav and touch icons --> + <!-- Update these with your own images + <link rel="shortcut icon" href="images/favicon.ico"> + <link rel="apple-touch-icon" href="images/apple-touch-icon.png"> + <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png"> + <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png"> + --> + + <!-- Js --> + <script src="https://code.jquery.com/jquery-1.10.2.min.js";></script> + <script src="/docs/0.9.0/assets/themes/zeppelin/bootstrap/js/bootstrap.min.js"></script> + <script src="/docs/0.9.0/assets/themes/zeppelin/js/docs.js"></script> + <script src="/docs/0.9.0/assets/themes/zeppelin/js/anchor.min.js"></script> + <script src="/docs/0.9.0/assets/themes/zeppelin/js/toc.js"></script> + <script src="/docs/0.9.0/assets/themes/zeppelin/js/lunr.min.js"></script> + <script src="/docs/0.9.0/assets/themes/zeppelin/js/search.js"></script> + + <!-- atom & rss feed --> + <link href="/docs/0.9.0/atom.xml" type="application/atom+xml" rel="alternate" title="Sitewide ATOM Feed"> + <link href="/docs/0.9.0/rss.xml" type="application/rss+xml" rel="alternate" title="Sitewide RSS Feed"> + </head> + + <body> + + <div id="menu" class="navbar navbar-inverse navbar-fixed-top" role="navigation"> + <div class="container navbar-container"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <div class="navbar-brand"> + <a class="navbar-brand-main" href="http://zeppelin.apache.org";> + <img src="/docs/0.9.0/assets/themes/zeppelin/img/zeppelin_logo.png" width="50" + style="margin-top: -2px;" alt="I'm zeppelin"> + <span style="margin-left: 5px; font-size: 27px;">Zeppelin</span> + <a class="navbar-brand-version" href="/docs/0.9.0" + style="font-size: 15px; color: white;"> 0.9.0 + </a> + </a> + </div> + </div> + <nav class="navbar-collapse collapse" role="navigation"> + <ul class="nav navbar-nav"> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Quick Start <b class="caret"></b></a> + <ul class="dropdown-menu"> + <li class="title"><span>Getting Started</span></li> + <li><a href="/docs/0.9.0/quickstart/install.html">Install</a></li> + <li><a href="/docs/0.9.0/quickstart/explore_ui.html">Explore UI</a></li> + <li><a href="/docs/0.9.0/quickstart/tutorial.html">Tutorial</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Run Mode</span></li> + <li><a href="/docs/0.9.0/quickstart/kubernetes.html">Kubernetes</a></li> + <li><a href="/docs/0.9.0/quickstart/docker.html">Docker</a></li> + <li><a href="/docs/0.9.0/quickstart/yarn.html">Yarn</a></li> + <li role="separator" class="divider"></li> + <li><a href="/docs/0.9.0/quickstart/spark_with_zeppelin.html">Spark with Zeppelin</a></li> + <li><a href="/docs/0.9.0/quickstart/sql_with_zeppelin.html">SQL with Zeppelin</a></li> + <li><a href="/docs/0.9.0/quickstart/python_with_zeppelin.html">Python with Zeppelin</a></li> + </ul> + </li> + + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Usage<b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu"> + <li class="title"><span>Dynamic Form</span></li> + <li><a href="/docs/0.9.0/usage/dynamic_form/intro.html">What is Dynamic Form?</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Display System</span></li> + <li><a href="/docs/0.9.0/usage/display_system/basic.html#text">Text Display</a></li> + <li><a href="/docs/0.9.0/usage/display_system/basic.html#html">HTML Display</a></li> + <li><a href="/docs/0.9.0/usage/display_system/basic.html#table">Table Display</a></li> + <li><a href="/docs/0.9.0/usage/display_system/basic.html#network">Network Display</a></li> + <li><a href="/docs/0.9.0/usage/display_system/angular_backend.html">Angular Display using Backend API</a></li> + <li><a href="/docs/0.9.0/usage/display_system/angular_frontend.html">Angular Display using Frontend API</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Interpreter</span></li> + <li><a href="/docs/0.9.0/usage/interpreter/overview.html">Overview</a></li> + <li><a href="/docs/0.9.0/usage/interpreter/interpreter_binding_mode.html">Interpreter Binding Mode</a></li> + <li><a href="/docs/0.9.0/usage/interpreter/user_impersonation.html">User Impersonation</a></li> + <li><a href="/docs/0.9.0/usage/interpreter/dependency_management.html">Dependency Management</a></li> + <li><a href="/docs/0.9.0/usage/interpreter/installation.html">Installing Interpreters</a></li> + <!--<li><a href="/docs/0.9.0/usage/interpreter/dynamic_loading.html">Dynamic Interpreter Loading (Experimental)</a></li>--> + <li><a href="/docs/0.9.0/usage/interpreter/execution_hooks.html">Execution Hooks (Experimental)</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Other Features</span></li> + <li><a href="/docs/0.9.0/usage/other_features/publishing_paragraphs.html">Publishing Paragraphs</a></li> + <li><a href="/docs/0.9.0/usage/other_features/personalized_mode.html">Personalized Mode</a></li> + <li><a href="/docs/0.9.0/usage/other_features/customizing_homepage.html">Customizing Zeppelin Homepage</a></li> + <li><a href="/docs/0.9.0/usage/other_features/notebook_actions.html">Notebook Actions</a></li> + <li><a href="/docs/0.9.0/usage/other_features/cron_scheduler.html">Cron Scheduler</a></li> + <li><a href="/docs/0.9.0/usage/other_features/zeppelin_context.html">Zeppelin Context</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>REST API</span></li> + <li><a href="/docs/0.9.0/usage/rest_api/interpreter.html">Interpreter API</a></li> + <li><a href="/docs/0.9.0/usage/rest_api/zeppelin_server.html">Zeppelin Server API</a></li> + <li><a href="/docs/0.9.0/usage/rest_api/notebook.html">Notebook API</a></li> + <li><a href="/docs/0.9.0/usage/rest_api/notebook_repository.html">Notebook Repository API</a></li> + <li><a href="/docs/0.9.0/usage/rest_api/configuration.html">Configuration API</a></li> + <li><a href="/docs/0.9.0/usage/rest_api/credential.html">Credential API</a></li> + <li><a href="/docs/0.9.0/usage/rest_api/helium.html">Helium API</a></li> + </ul> + </li> + + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Setup<b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu"> + <li class="title"><span>Basics</span></li> + <li><a href="/docs/0.9.0/setup/basics/how_to_build.html">How to Build Zeppelin</a></li> + <li><a href="/docs/0.9.0/setup/basics/hadoop_integration.html">Hadoop Integration</a></li> + <li><a href="/docs/0.9.0/setup/basics/multi_user_support.html">Multi-user Support</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Deployment</span></li> + <!--<li><a href="/docs/0.9.0/setup/deployment/docker.html">Docker Image for Zeppelin</a></li>--> + <li><a href="/docs/0.9.0/setup/deployment/spark_cluster_mode.html#spark-standalone-mode">Spark Cluster Mode: Standalone</a></li> + <li><a href="/docs/0.9.0/setup/deployment/spark_cluster_mode.html#spark-on-yarn-mode">Spark Cluster Mode: YARN</a></li> + <li><a href="/docs/0.9.0/setup/deployment/spark_cluster_mode.html#spark-on-mesos-mode">Spark Cluster Mode: Mesos</a></li> + <li><a href="/docs/0.9.0/setup/deployment/flink_and_spark_cluster.html">Zeppelin with Flink, Spark Cluster</a></li> + <li><a href="/docs/0.9.0/setup/deployment/cdh.html">Zeppelin on CDH</a></li> + <li><a href="/docs/0.9.0/setup/deployment/virtual_machine.html">Zeppelin on VM: Vagrant</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Security</span></li> + <li><a href="/docs/0.9.0/setup/security/authentication_nginx.html">HTTP Basic Auth using NGINX</a></li> + <li><a href="/docs/0.9.0/setup/security/shiro_authentication.html">Shiro Authentication</a></li> + <li><a href="/docs/0.9.0/setup/security/notebook_authorization.html">Notebook Authorization</a></li> + <li><a href="/docs/0.9.0/setup/security/datasource_authorization.html">Data Source Authorization</a></li> + <li><a href="/docs/0.9.0/setup/security/http_security_headers.html">HTTP Security Headers</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Notebook Storage</span></li> + <li><a href="/docs/0.9.0/setup/storage/storage.html#notebook-storage-in-local-git-repository">Git Storage</a></li> + <li><a href="/docs/0.9.0/setup/storage/storage.html#notebook-storage-in-s3">S3 Storage</a></li> + <li><a href="/docs/0.9.0/setup/storage/storage.html#notebook-storage-in-azure">Azure Storage</a></li> + <li><a href="/docs/0.9.0/setup/storage/storage.html#notebook-storage-in-oss">OSS Storage</a></li> + <li><a href="/docs/0.9.0/setup/storage/storage.html#notebook-storage-in-zeppelinhub">ZeppelinHub Storage</a></li> + <li><a href="/docs/0.9.0/setup/storage/storage.html#notebook-storage-in-mongodb">MongoDB Storage</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Operation</span></li> + <li><a href="/docs/0.9.0/setup/operation/configuration.html">Configuration</a></li> + <li><a href="/docs/0.9.0/setup/operation/proxy_setting.html">Proxy Setting</a></li> + <li><a href="/docs/0.9.0/setup/operation/upgrading.html">Upgrading</a></li> + <li><a href="/docs/0.9.0/setup/operation/trouble_shooting.html">Trouble Shooting</a></li> + </ul> + </li> + + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Interpreter <b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu"> + <li class="title"><span>Interpreters</span></li> + <li><a href="/docs/0.9.0/usage/interpreter/overview.html">Overview</a></li> + <li role="separator" class="divider"></li> + <li><a href="/docs/0.9.0/interpreter/spark.html">Spark</a></li> + <li><a href="/docs/0.9.0/interpreter/jdbc.html">JDBC</a></li> + <li><a href="/docs/0.9.0/interpreter/python.html">Python</a></li> + <li><a href="/docs/0.9.0/interpreter/r.html">R</a></li> + <li role="separator" class="divider"></li> + <li><a href="/docs/0.9.0/interpreter/alluxio.html">Alluxio</a></li> + <li><a href="/docs/0.9.0/interpreter/beam.html">Beam</a></li> + <li><a href="/docs/0.9.0/interpreter/bigquery.html">BigQuery</a></li> + <li><a href="/docs/0.9.0/interpreter/cassandra.html">Cassandra</a></li> + <li><a href="/docs/0.9.0/interpreter/elasticsearch.html">Elasticsearch</a></li> + <li><a href="/docs/0.9.0/interpreter/flink.html">Flink</a></li> + <li><a href="/docs/0.9.0/interpreter/geode.html">Geode</a></li> + <li><a href="/docs/0.9.0/interpreter/groovy.html">Groovy</a></li> + <li><a href="/docs/0.9.0/interpreter/hazelcastjet.html">Hazelcast Jet</a></li> + <li><a href="/docs/0.9.0/interpreter/hbase.html">HBase</a></li> + <li><a href="/docs/0.9.0/interpreter/hdfs.html">HDFS</a></li> + <li><a href="/docs/0.9.0/interpreter/hive.html">Hive</a></li> + <li><a href="/docs/0.9.0/interpreter/ignite.html">Ignite</a></li> + <li><a href="/docs/0.9.0/interpreter/java.html">Java</a></li> + <li><a href="/docs/0.9.0/interpreter/jupyter.html">Jupyter</a></li> + <li><a href="/docs/0.9.0/interpreter/kotlin.html">Kotlin</a></li> + <li><a href="/docs/0.9.0/interpreter/kylin.html">Kylin</a></li> + <li><a href="/docs/0.9.0/interpreter/lens.html">Lens</a></li> + <li><a href="/docs/0.9.0/interpreter/livy.html">Livy</a></li> + <li><a href="/docs/0.9.0/interpreter/markdown.html">Markdown</a></li> + <li><a href="/docs/0.9.0/interpreter/mongodb.html">MongoDB</a></li> + <li><a href="/docs/0.9.0/interpreter/neo4j.html">Neo4j</a></li> + <li><a href="/docs/0.9.0/interpreter/pig.html">Pig</a></li> + <li><a href="/docs/0.9.0/interpreter/postgresql.html">Postgresql, HAWQ</a></li> + <li><a href="/docs/0.9.0/interpreter/scalding.html">Scalding</a></li> + <li><a href="/docs/0.9.0/interpreter/scio.html">Scio</a></li> + <li><a href="/docs/0.9.0/interpreter/shell.html">Shell</a></li> + <li><a href="/docs/0.9.0/interpreter/sparql.html">Sparql</a></li> + <li><a href="/docs/0.9.0/interpreter/submarine.html">Submarine</a></li> + </ul> + </li> + <li> + <a href="#" data-toggle="dropdown" class="dropdown-toggle">More<b class="caret"></b></a> + <ul class="dropdown-menu scrollable-menu" style="right: 0; left: auto;"> + <li class="title"><span>Extending Zeppelin</span></li> + <li><a href="/docs/0.9.0/development/writing_zeppelin_interpreter.html">Writing Zeppelin Interpreter</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Helium (Experimental)</span></li> + <li><a href="/docs/0.9.0/development/helium/overview.html">Overview</a></li> + <li><a href="/docs/0.9.0/development/helium/writing_application.html">Writing Helium Application</a></li> + <li><a href="/docs/0.9.0/development/helium/writing_spell.html">Writing Helium Spell</a></li> + <li><a href="/docs/0.9.0/development/helium/writing_visualization_basic.html">Writing Helium Visualization: Basics</a></li> + <li><a href="/docs/0.9.0/development/helium/writing_visualization_transformation.html">Writing Helium Visualization: Transformation</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>Contributing to Zeppelin</span></li> + <li><a href="/docs/0.9.0/setup/basics/how_to_build.html">How to Build Zeppelin</a></li> + <li><a href="/docs/0.9.0/development/contribution/useful_developer_tools.html">Useful Developer Tools</a></li> + <li><a href="/docs/0.9.0/development/contribution/how_to_contribute_code.html">How to Contribute (code)</a></li> + <li><a href="/docs/0.9.0/development/contribution/how_to_contribute_website.html">How to Contribute (website)</a></li> + <li role="separator" class="divider"></li> + <li class="title"><span>External Resources</span></li> + <li><a target="_blank" href="https://zeppelin.apache.org/community.html";>Mailing List</a></li> + <li><a target="_blank" href="https://cwiki.apache.org/confluence/display/ZEPPELIN/Zeppelin+Home";>Apache Zeppelin Wiki</a></li> + <li><a target="_blank" href="http://stackoverflow.com/questions/tagged/apache-zeppelin";>Stackoverflow Questions about Zeppelin</a></li> + </ul> + </li> + <li> + <a href="/docs/0.9.0/search.html" class="nav-search-link"> + <span class="fa fa-search nav-search-icon"></span> + </a> + </li> + </ul> + </nav><!--/.navbar-collapse --> + </div> + </div> + + + + <div class="content"> + +<!--<div class="hero-unit Apache Zeppelin on Spark cluster mode"> + <h1></h1> +</div> +--> + +<div class="row"> + <div class="col-md-12"> + <!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +<h1>Apache Zeppelin on Spark Cluster Mode</h1> + +<div id="toc"></div> + +<h2>Overview</h2> + +<p><a href="http://spark.apache.org/";>Apache Spark</a> has supported three cluster manager types(<a href="http://spark.apache.org/docs/latest/spark-standalone.html";>Standalone</a>, <a href="http://spark.apache.org/docs/latest/running-on-mesos.html";>Apache Mesos</a> and <a href="http://spark.apache.org/docs/latest/running-on-yarn.html";>Hadoop YARN</a>) so far. +This document will guide you how you can build and configure the environment on 3 types of Spark cluster manager with Apache Zeppelin using <a href="https://www.docker.com/";>Docker</a> scripts. +So <a href="https://docs.docker.com/engine/installation/";>install docker</a> on the machine first.</p> + +<h2>Spark standalone mode</h2> + +<p><a href="http://spark.apache.org/docs/latest/spark-standalone.html";>Spark standalone</a> is a simple cluster manager included with Spark that makes it easy to set up a cluster. +You can simply set up Spark standalone environment with below steps.</p> + +<blockquote> +<p><strong>Note :</strong> Since Apache Zeppelin and Spark use same <code>8080</code> port for their web UI, you might need to change <code>zeppelin.server.port</code> in <code>conf/zeppelin-site.xml</code>.</p> +</blockquote> + +<h3>1. Build Docker file</h3> + +<p>You can find docker script files under <code>scripts/docker/spark-cluster-managers</code>.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash"><span class="nb">cd</span> <span class="nv">$ZEPPELIN_HOME</span>/scripts/docker/spark-cluster-managers/spark_standalone +docker build -t <span class="s2">"spark_standalone"</span> . +</code></pre></div> +<h3>2. Run docker</h3> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">docker run -it <span class="se">\</span> +-p 8080:8080 <span class="se">\</span> +-p 7077:7077 <span class="se">\</span> +-p 8888:8888 <span class="se">\</span> +-p 8081:8081 <span class="se">\</span> +-h sparkmaster <span class="se">\</span> +--name spark_standalone <span class="se">\</span> +spark_standalone bash<span class="p">;</span> +</code></pre></div> +<p>Note that <code>sparkmaster</code> hostname used here to run docker container should be defined in your <code>/etc/hosts</code>.</p> + +<h3>3. Configure Spark interpreter in Zeppelin</h3> + +<p>Set Spark master as <code>spark://<hostname>:7077</code> in Zeppelin <strong>Interpreters</strong> setting page.</p> + +<p><img src="/docs/0.9.0/assets/themes/zeppelin/img/docs-img/standalone_conf.png" /></p> + +<h3>4. Run Zeppelin with Spark interpreter</h3> + +<p>After running single paragraph with Spark interpreter in Zeppelin, browse <code>https://<hostname>:8080</code> and check whether Spark cluster is running well or not.</p> + +<p><img src="/docs/0.9.0/assets/themes/zeppelin/img/docs-img/spark_ui.png" /></p> + +<p>You can also simply verify that Spark is running well in Docker with below command.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">ps -ef <span class="p">|</span> grep spark +</code></pre></div> +<h2>Spark on YARN mode</h2> + +<p>You can simply set up <a href="http://spark.apache.org/docs/latest/running-on-yarn.html";>Spark on YARN</a> docker environment with below steps.</p> + +<blockquote> +<p><strong>Note :</strong> Since Apache Zeppelin and Spark use same <code>8080</code> port for their web UI, you might need to change <code>zeppelin.server.port</code> in <code>conf/zeppelin-site.xml</code>.</p> +</blockquote> + +<h3>1. Build Docker file</h3> + +<p>You can find docker script files under <code>scripts/docker/spark-cluster-managers</code>.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash"><span class="nb">cd</span> <span class="nv">$ZEPPELIN_HOME</span>/scripts/docker/spark-cluster-managers/spark_yarn_cluster +docker build -t <span class="s2">"spark_yarn"</span> . +</code></pre></div> +<h3>2. Run docker</h3> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">docker run -it <span class="se">\</span> + -p 5000:5000 <span class="se">\</span> + -p 9000:9000 <span class="se">\</span> + -p 9001:9001 <span class="se">\</span> + -p 8088:8088 <span class="se">\</span> + -p 8042:8042 <span class="se">\</span> + -p 8030:8030 <span class="se">\</span> + -p 8031:8031 <span class="se">\</span> + -p 8032:8032 <span class="se">\</span> + -p 8033:8033 <span class="se">\</span> + -p 8080:8080 <span class="se">\</span> + -p 7077:7077 <span class="se">\</span> + -p 8888:8888 <span class="se">\</span> + -p 8081:8081 <span class="se">\</span> + -p 50010:50010 <span class="se">\</span> + -p 50075:50075 <span class="se">\</span> + -p 50020:50020 <span class="se">\</span> + -p 50070:50070 <span class="se">\</span> + --name spark_yarn <span class="se">\</span> + -h sparkmaster <span class="se">\</span> + spark_yarn bash<span class="p">;</span> +</code></pre></div> +<p>Note that <code>sparkmaster</code> hostname used here to run docker container should be defined in your <code>/etc/hosts</code>.</p> + +<h3>3. Verify running Spark on YARN.</h3> + +<p>You can simply verify the processes of Spark and YARN are running well in Docker with below command.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">ps -ef +</code></pre></div> +<p>You can also check each application web UI for HDFS on <code>http://<hostname>:50070/</code>, YARN on <code>http://<hostname>:8088/cluster</code> and Spark on <code>http://<hostname>:8080/</code>.</p> + +<h3>4. Configure Spark interpreter in Zeppelin</h3> + +<p>Set following configurations to <code>conf/zeppelin-env.sh</code>.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash"><span class="nb">export </span><span class="nv">HADOOP_CONF_DIR</span><span class="o">=[</span>your_hadoop_conf_path<span class="o">]</span> +<span class="nb">export </span><span class="nv">SPARK_HOME</span><span class="o">=[</span>your_spark_home_path<span class="o">]</span> +</code></pre></div> +<p><code>HADOOP_CONF_DIR</code>(Hadoop configuration path) is defined in <code>/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf</code>.</p> + +<p>Don't forget to set Spark <code>spark.master</code> as <code>yarn-client</code> in Zeppelin <strong>Interpreters</strong> setting page like below.</p> + +<p><img src="/docs/0.9.0/assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png" /></p> + +<h3>5. Run Zeppelin with Spark interpreter</h3> + +<p>After running a single paragraph with Spark interpreter in Zeppelin, browse <code>http://<hostname>:8088/cluster/apps</code> and check Zeppelin application is running well or not.</p> + +<p><img src="/docs/0.9.0/assets/themes/zeppelin/img/docs-img/yarn_applications.png" /></p> + +<h2>Spark on Mesos mode</h2> + +<p>You can simply set up <a href="http://spark.apache.org/docs/latest/running-on-mesos.html";>Spark on Mesos</a> docker environment with below steps.</p> + +<h3>1. Build Docker file</h3> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash"><span class="nb">cd</span> <span class="nv">$ZEPPELIN_HOME</span>/scripts/docker/spark-cluster-managers/spark_mesos +docker build -t <span class="s2">"spark_mesos"</span> . +</code></pre></div> +<h3>2. Run docker</h3> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">docker run --net<span class="o">=</span>host -it <span class="se">\</span> +-p 8080:8080 <span class="se">\</span> +-p 7077:7077 <span class="se">\</span> +-p 8888:8888 <span class="se">\</span> +-p 8081:8081 <span class="se">\</span> +-p 8082:8082 <span class="se">\</span> +-p 5050:5050 <span class="se">\</span> +-p 5051:5051 <span class="se">\</span> +-p 4040:4040 <span class="se">\</span> +-h sparkmaster <span class="se">\</span> +--name spark_mesos <span class="se">\</span> +spark_mesos bash<span class="p">;</span> +</code></pre></div> +<p>Note that <code>sparkmaster</code> hostname used here to run docker container should be defined in your <code>/etc/hosts</code>.</p> + +<h3>3. Verify running Spark on Mesos.</h3> + +<p>You can simply verify the processes of Spark and Mesos are running well in Docker with below command.</p> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash">ps -ef +</code></pre></div> +<p>You can also check each application web UI for Mesos on <code>http://<hostname>:5050/cluster</code> and Spark on <code>http://<hostname>:8080/</code>.</p> + +<h3>4. Configure Spark interpreter in Zeppelin</h3> +<div class="highlight"><pre><code class="bash language-bash" data-lang="bash"><span class="nb">export </span><span class="nv">MESOS_NATIVE_JAVA_LIBRARY</span><span class="o">=[</span>PATH OF libmesos.so<span class="o">]</span> +<span class="nb">export </span><span class="nv">SPARK_HOME</span><span class="o">=[</span>PATH OF SPARK HOME<span class="o">]</span> +</code></pre></div> +<p>Don't forget to set Spark <code>spark.master</code> as <code>mesos://127.0.1.1:5050</code> in Zeppelin <strong>Interpreters</strong> setting page like below.</p> + +<p><img src="/docs/0.9.0/assets/themes/zeppelin/img/docs-img/zeppelin_mesos_conf.png" /></p> + +<h3>5. Run Zeppelin with Spark interpreter</h3> + +<p>After running a single paragraph with Spark interpreter in Zeppelin, browse <code>http://<hostname>:5050/#/frameworks</code> and check Zeppelin application is running well or not.</p> + +<p><img src="/docs/0.9.0/assets/themes/zeppelin/img/docs-img/mesos_frameworks.png" /></p> + +<h3>Troubleshooting for Spark on Mesos</h3> + +<ul> +<li>If you have problem with hostname, use <code>--add-host</code> option when executing <code>dockerrun</code></li> +</ul> +<div class="highlight"><pre><code class="text language-text" data-lang="text">## use `--add-host=moby:127.0.0.1` option to resolve +## since docker container couldn't resolve `moby` + +: java.net.UnknownHostException: moby: moby: Name or service not known + at java.net.InetAddress.getLocalHost(InetAddress.java:1496) + at org.apache.spark.util.Utils$.findLocalInetAddress(Utils.scala:789) + at org.apache.spark.util.Utils$.org$apache$spark$util$Utils$$localIpAddress$lzycompute(Utils.scala:782) + at org.apache.spark.util.Utils$.org$apache$spark$util$Utils$$localIpAddress(Utils.scala:782) +</code></pre></div> +<ul> +<li>If you have problem with mesos master, try <code>mesos://127.0.0.1</code> instead of <code>mesos://127.0.1.1</code></li> +</ul> +<div class="highlight"><pre><code class="text language-text" data-lang="text">I0103 20:17:22.329269 340 sched.cpp:330] New master detected at master@127.0.1.1:5050 +I0103 20:17:22.330749 340 sched.cpp:341] No credentials provided. Attempting to register without authentication +W0103 20:17:22.333531 340 sched.cpp:736] Ignoring framework registered message because it was sentfrom 'master@127.0.0.1:5050' instead of the leading master 'master@127.0.1.1:5050' +W0103 20:17:24.040252 339 sched.cpp:736] Ignoring framework registered message because it was sentfrom 'master@127.0.0.1:5050' instead of the leading master 'master@127.0.1.1:5050' +W0103 20:17:26.150250 339 sched.cpp:736] Ignoring framework registered message because it was sentfrom 'master@127.0.0.1:5050' instead of the leading master 'master@127.0.1.1:5050' +W0103 20:17:26.737604 339 sched.cpp:736] Ignoring framework registered message because it was sentfrom 'master@127.0.0.1:5050' instead of the leading master 'master@127.0.1.1:5050' +W0103 20:17:35.241714 336 sched.cpp:736] Ignoring framework registered message because it was sentfrom 'master@127.0.0.1:5050' instead of the leading master 'master@127.0.1.1:5050' +</code></pre></div> + </div> +</div> + + + <hr> + <footer> + <!-- <p>© 2020 The Apache Software Foundation</p>--> + </footer> + </div> + + + + + <script type="text/javascript"> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-45176241-5', 'zeppelin.apache.org'); + ga('require', 'linkid', 'linkid.js'); + ga('send', 'pageview'); + +</script> + + + + </body> +</html> +
