Repository: spark Updated Branches: refs/heads/master 0bb15f22d -> f240fe390
[WIP] [SPARK-3996]: Shade Jetty in Spark deliverables This patch piggy-back's on vanzin's work to simplify the Guava shading, and adds Jetty as a shaded library in Spark. Other than adding Jetty, it consilidates the \<artifactSet\>'s into the root pom. I found it was a bit easier to follow that way, since you don't need to look into child pom's to find out specific artifact sets included in shading. Author: Patrick Wendell <[email protected]> Closes #4252 from pwendell/jetty and squashes the following commits: 19f0710 [Patrick Wendell] More code review feedback 961452d [Patrick Wendell] Responding to feedback from Marcello 6df25ca [Patrick Wendell] [WIP] [SPARK-3996]: Shade Jetty in Spark deliverables Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f240fe39 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f240fe39 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f240fe39 Branch: refs/heads/master Commit: f240fe390b46b6e9859ce74108c5a5fba5c5f8b3 Parents: 0bb15f2 Author: Patrick Wendell <[email protected]> Authored: Thu Jan 29 16:31:19 2015 -0800 Committer: Patrick Wendell <[email protected]> Committed: Thu Jan 29 16:31:19 2015 -0800 ---------------------------------------------------------------------- bin/compute-classpath.sh | 4 +++- core/pom.xml | 22 ++++++++++++++++++++-- network/common/pom.xml | 12 ------------ pom.xml | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/f240fe39/bin/compute-classpath.sh ---------------------------------------------------------------------- diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index 9e8d0b7..a8c344b 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -50,8 +50,8 @@ fi if [ -n "$SPARK_PREPEND_CLASSES" ]; then echo "NOTE: SPARK_PREPEND_CLASSES is set, placing locally compiled Spark"\ "classes ahead of assembly." >&2 + # Spark classes CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/classes" - CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*" CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SPARK_SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SPARK_SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SPARK_SCALA_VERSION/classes" @@ -63,6 +63,8 @@ if [ -n "$SPARK_PREPEND_CLASSES" ]; then CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/sql/hive-thriftserver/target/scala-$SPARK_SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SPARK_SCALA_VERSION/classes" + # Jars for shaded deps in their original form (copied here during build) + CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*" fi # Use spark-assembly jar from either RELEASE or assembly directory http://git-wip-us.apache.org/repos/asf/spark/blob/f240fe39/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index 31e919a..d91f4ee 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -94,23 +94,36 @@ <groupId>org.apache.curator</groupId> <artifactId>curator-recipes</artifactId> </dependency> + + <!-- Jetty dependencies promoted to compile here so they are shaded + and inlined into spark-core jar --> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-plus</artifactId> + <scope>compile</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-security</artifactId> + <scope>compile</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-util</artifactId> + <scope>compile</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> + <scope>compile</scope> </dependency> <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + <scope>compile</scope> + </dependency> + + <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> </dependency> @@ -348,19 +361,24 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> <executions> + <!-- When using SPARK_PREPEND_CLASSES Spark classes compiled locally don't use + shaded deps. So here we store jars in their original form which are added + when the classpath is computed. --> <execution> <id>copy-dependencies</id> <phase>package</phase> <goals> <goal>copy-dependencies</goal> </goals> - <configuration> + <configuration> <outputDirectory>${project.build.directory}</outputDirectory> <overWriteReleases>false</overWriteReleases> <overWriteSnapshots>false</overWriteSnapshots> <overWriteIfNewer>true</overWriteIfNewer> <useSubDirectoryPerType>true</useSubDirectoryPerType> - <includeArtifactIds>guava</includeArtifactIds> + <includeArtifactIds> + guava,jetty-io,jetty-http,jetty-plus,jetty-util,jetty-server + </includeArtifactIds> <silent>true</silent> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/spark/blob/f240fe39/network/common/pom.xml ---------------------------------------------------------------------- diff --git a/network/common/pom.xml b/network/common/pom.xml index 5a9bbe1..8f7c924 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -101,18 +101,6 @@ </execution> </executions> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <shadedArtifactAttached>false</shadedArtifactAttached> - <artifactSet> - <includes> - <include>com.google.guava:guava</include> - </includes> - </artifactSet> - </configuration> - </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/spark/blob/f240fe39/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 4adfdf3..63c0a2a 100644 --- a/pom.xml +++ b/pom.xml @@ -337,25 +337,39 @@ </exclusion> </exclusions> </dependency> + + <!-- Shaded deps marked as provided. These are promoted to compile scope + in the modules where we want the shaded classes to appear in the + associated jar. --> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + <version>${jetty.version}</version> + <scope>provided</scope> + </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-util</artifactId> <version>${jetty.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-security</artifactId> <version>${jetty.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-plus</artifactId> <version>${jetty.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> <version>${jetty.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -363,6 +377,8 @@ <version>14.0.1</version> <scope>provided</scope> </dependency> + <!-- End of shaded deps --> + <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> @@ -1276,11 +1292,27 @@ <shadedArtifactAttached>false</shadedArtifactAttached> <artifactSet> <includes> + <!-- At a minimum we must include this to force effective pom generation --> <include>org.spark-project.spark:unused</include> + + <include>org.eclipse.jetty:jetty-io</include> + <include>org.eclipse.jetty:jetty-http</include> + <include>org.eclipse.jetty:jetty-plus</include> + <include>org.eclipse.jetty:jetty-security</include> + <include>org.eclipse.jetty:jetty-util</include> + <include>org.eclipse.jetty:jetty-server</include> + <include>com.google.guava:guava</include> </includes> </artifactSet> <relocations> <relocation> + <pattern>org.eclipse.jetty</pattern> + <shadedPattern>org.spark-project.jetty</shadedPattern> + <includes> + <include>org.eclipse.jetty.**</include> + </includes> + </relocation> + <relocation> <pattern>com.google.common</pattern> <shadedPattern>org.spark-project.guava</shadedPattern> <excludes> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
