This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch branch-0.8
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/branch-0.8 by this push:
new b3c9632 [ZEPPELIN-4030] spark-scala-2.10 won't be built under some
profile
b3c9632 is described below
commit b3c963299ef2e662d8f739f357d6210d24d42c33
Author: Jeff Zhang <zjf...@apache.org>
AuthorDate: Wed Mar 6 15:24:36 2019 +0800
[ZEPPELIN-4030] spark-scala-2.10 won't be built under some profile
Before this PR, module spark-scala-2.10 is only built under spark profile
which support both scala 2.10 and 2.11. But it should not be depend on which
version of spark to build with. The spark profile should only affect the
embedded spark distribution of spark interpreter, it should support multiple
spark version no matter which profile it is built against.
[Bug Fix]
* [ ] - Task
* https://jira.apache.org/jira/browse/ZEPPELIN-4030
* CI pass
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zjf...@apache.org>
Closes #3325 from zjffdu/ZEPPELIN-4030 and squashes the following commits:
aec33d24c [Jeff Zhang] [ZEPPELIN-4030] spark-scala-2.10 won't be built
under some profile
(cherry picked from commit d4a2f6403aa06950b0786362f2aba88cb4df7ad0)
Signed-off-by: Jeff Zhang <zjf...@apache.org>
---
spark/interpreter/pom.xml | 77 ++--------------
spark/pom.xml | 23 +----
spark/scala-2.10/pom.xml | 1 +
spark/scala-2.11/pom.xml | 1 +
spark/spark-dependencies/pom.xml | 189 +--------------------------------------
spark/spark-scala-parent/pom.xml | 100 +++++++++++++++++----
spark/spark-shims/pom.xml | 7 --
spark/spark1-shims/pom.xml | 7 --
spark/spark2-shims/pom.xml | 7 --
9 files changed, 101 insertions(+), 311 deletions(-)
diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml
index 30440e4..321192d 100644
--- a/spark/interpreter/pom.xml
+++ b/spark/interpreter/pom.xml
@@ -52,8 +52,6 @@
<!-- settings -->
<pyspark.test.exclude>**/PySparkInterpreterMatplotlibTest.java</pyspark.test.exclude>
<pyspark.test.include>**/*Test.*</pyspark.test.include>
-
-
</properties>
<dependencies>
@@ -65,6 +63,12 @@
<dependency>
<groupId>org.apache.zeppelin</groupId>
+ <artifactId>spark-scala-2.10</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.zeppelin</groupId>
<artifactId>spark-scala-2.11</artifactId>
<version>${project.version}</version>
</dependency>
@@ -132,6 +136,7 @@
</exclusion>
</exclusions>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
@@ -144,18 +149,8 @@
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- </exclusions>
</dependency>
-
+
<!-- Aether :: maven dependency resolution -->
<dependency>
<groupId>org.apache.maven</groupId>
@@ -579,60 +574,4 @@
</plugins>
</build>
-
- <profiles>
-
- <profile>
- <id>spark-2.4</id>
- </profile>
-
- <profile>
- <id>spark-2.3</id>
- </profile>
-
- <profile>
- <id>spark-2.2</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.zeppelin</groupId>
- <artifactId>spark-scala-2.10</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>spark-2.1</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.zeppelin</groupId>
- <artifactId>spark-scala-2.10</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>spark-2.0</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.zeppelin</groupId>
- <artifactId>spark-scala-2.10</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>spark-1.6</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.zeppelin</groupId>
- <artifactId>spark-scala-2.10</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
-
- </profiles>
</project>
diff --git a/spark/pom.xml b/spark/pom.xml
index 10f5a1d..b489df7 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -60,6 +60,7 @@
<modules>
<module>interpreter</module>
<module>spark-scala-parent</module>
+ <module>scala-2.10</module>
<module>scala-2.11</module>
<module>spark-dependencies</module>
<module>spark-shims</module>
@@ -163,8 +164,6 @@
</executions>
<configuration>
<scalaVersion>${scala.compile.version}</scalaVersion>
- <!--<recompileMode>incremental</recompileMode>-->
- <!--<useZincServer>true</useZincServer>-->
<args>
<arg>-unchecked</arg>
<arg>-deprecation</arg>
@@ -189,7 +188,6 @@
</plugins>
</build>
-
<profiles>
<profile>
@@ -199,6 +197,9 @@
<protobuf.version>2.5.0</protobuf.version>
<py4j.version>0.10.7</py4j.version>
</properties>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
</profile>
<profile>
@@ -216,12 +217,6 @@
<spark.version>2.2.1</spark.version>
<py4j.version>0.10.4</py4j.version>
</properties>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <modules>
- <module>scala-2.10</module>
- </modules>
</profile>
<profile>
@@ -230,9 +225,6 @@
<spark.version>2.1.2</spark.version>
<py4j.version>0.10.4</py4j.version>
</properties>
- <modules>
- <module>scala-2.10</module>
- </modules>
</profile>
<profile>
@@ -241,9 +233,6 @@
<spark.version>2.0.2</spark.version>
<py4j.version>0.10.3</py4j.version>
</properties>
- <modules>
- <module>scala-2.10</module>
- </modules>
</profile>
<profile>
@@ -252,10 +241,6 @@
<spark.version>1.6.3</spark.version>
<py4j.version>0.9</py4j.version>
</properties>
- <modules>
- <module>scala-2.10</module>
- </modules>
</profile>
-
</profiles>
</project>
diff --git a/spark/scala-2.10/pom.xml b/spark/scala-2.10/pom.xml
index 229d473..c49b3e2 100644
--- a/spark/scala-2.10/pom.xml
+++ b/spark/scala-2.10/pom.xml
@@ -33,6 +33,7 @@
<name>Zeppelin: Spark Interpreter Scala_2.10</name>
<properties>
+ <spark.version>2.2.0</spark.version>
<scala.version>2.10.5</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<scala.compile.version>${scala.version}</scala.compile.version>
diff --git a/spark/scala-2.11/pom.xml b/spark/scala-2.11/pom.xml
index 453ce5c..298cc31 100644
--- a/spark/scala-2.11/pom.xml
+++ b/spark/scala-2.11/pom.xml
@@ -33,6 +33,7 @@
<name>Zeppelin: Spark Interpreter Scala_2.11</name>
<properties>
+ <spark.version>2.4.0</spark.version>
<scala.version>2.11.8</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<scala.compile.version>${scala.version}</scala.compile.version>
diff --git a/spark/spark-dependencies/pom.xml b/spark/spark-dependencies/pom.xml
index 9b591f0..a60cc8a 100644
--- a/spark/spark-dependencies/pom.xml
+++ b/spark/spark-dependencies/pom.xml
@@ -53,197 +53,10 @@
<akka.group>org.spark-project.akka</akka.group>
<akka.version>2.3.4-spark</akka.version>
-
<!--plugin versions-->
<plugin.shade.version>2.3</plugin.shade.version>
</properties>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <version>${avro.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-ipc</artifactId>
- <version>${avro.version}</version>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.velocity</groupId>
- <artifactId>velocity</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-mapred</artifactId>
- <version>${avro.version}</version>
- <classifier>${avro.mapred.classifier}</classifier>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.velocity</groupId>
- <artifactId>velocity</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- See SPARK-1556 for info on this dependency: -->
- <dependency>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- <version>${jets3t.version}</version>
- <scope>runtime</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <version>${yarn.version}</version>
- <exclusions>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- <version>${yarn.version}</version>
- <exclusions>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-web-proxy</artifactId>
- <version>${yarn.version}</version>
- <exclusions>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <version>${yarn.version}</version>
- <exclusions>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
<dependencies>
<!-- Spark -->
<dependency>
@@ -541,4 +354,6 @@
</plugin>
</plugins>
</build>
+
+
</project>
diff --git a/spark/spark-scala-parent/pom.xml b/spark/spark-scala-parent/pom.xml
index 76bd0b8..42e118c 100644
--- a/spark/spark-scala-parent/pom.xml
+++ b/spark/spark-scala-parent/pom.xml
@@ -21,10 +21,10 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
<parent>
- <artifactId>spark-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
+ <artifactId>interpreter-parent</artifactId>
<version>0.8.2-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
+ <relativePath>../../interpreter-parent/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -34,7 +34,20 @@
<packaging>pom</packaging>
<name>Zeppelin: Spark Scala Parent</name>
+ <properties>
+ <spark.version>2.4.0</spark.version>
+ <scala.binary.version>2.11</scala.binary.version>
+ <scala.version>2.11.8</scala.version>
+ <scala.compile.version>${scala.binary.version}</scala.compile.version>
+ </properties>
+
<dependencies>
+ <dependency>
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>zeppelin-interpreter</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.zeppelin</groupId>
@@ -84,10 +97,38 @@
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
</dependencies>
<build>
<plugins>
+
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-interpreter-setting</id>
+ <phase>none</phase>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
@@ -148,23 +189,52 @@
</plugin>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.2.2</version>
<executions>
<execution>
- <id>copy-interpreter-setting</id>
- <phase>none</phase>
- <configuration>
- <skip>true</skip>
- </configuration>
+ <id>eclipse-add-source</id>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>scala-compile-first</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>scala-test-compile-first</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
</execution>
</executions>
+ <configuration>
+ <scalaVersion>${scala.compile.version}</scalaVersion>
+ <args>
+ <arg>-unchecked</arg>
+ <arg>-deprecation</arg>
+ <arg>-feature</arg>
+ </args>
+ <jvmArgs>
+ <jvmArg>-Xms1024m</jvmArg>
+ <jvmArg>-Xmx1024m</jvmArg>
+ <jvmArg>-XX:PermSize=${PermGen}</jvmArg>
+ <jvmArg>-XX:MaxPermSize=${MaxPermGen}</jvmArg>
+ </jvmArgs>
+ <javacArgs>
+ <javacArg>-source</javacArg>
+ <javacArg>${java.version}</javacArg>
+ <javacArg>-target</javacArg>
+ <javacArg>${java.version}</javacArg>
+ <javacArg>-Xlint:all,-serial,-path,-options</javacArg>
+ </javacArgs>
+ </configuration>
</plugin>
</plugins>
diff --git a/spark/spark-shims/pom.xml b/spark/spark-shims/pom.xml
index 186427f..22458fc 100644
--- a/spark/spark-shims/pom.xml
+++ b/spark/spark-shims/pom.xml
@@ -57,13 +57,6 @@
<build>
<plugins>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
diff --git a/spark/spark1-shims/pom.xml b/spark/spark1-shims/pom.xml
index 547fe0f..ca24400 100644
--- a/spark/spark1-shims/pom.xml
+++ b/spark/spark1-shims/pom.xml
@@ -65,13 +65,6 @@
<build>
<plugins>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
diff --git a/spark/spark2-shims/pom.xml b/spark/spark2-shims/pom.xml
index 2f4731c..7e90b46 100644
--- a/spark/spark2-shims/pom.xml
+++ b/spark/spark2-shims/pom.xml
@@ -64,13 +64,6 @@
<build>
<plugins>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
