Repository: spark Updated Branches: refs/heads/master 26d31d15f -> 0734d0932
HOTFIX: Clean up build in network module. This is currently breaking the package build for some people (including me). This patch does some general clean-up which also fixes the current issue. - Uses consistent artifact naming - Adds sbt support for this module - Changes tests to use scalatest (fixes the original issue[1]) One thing to note, it turns out that scalatest when invoked in the Maven build doesn't succesfully detect JUnit Java tests. This is a long standing issue, I noticed it applies to all of our current test suites as well. I've created SPARK-4159 to fix this. [1] The original issue is that we need to allocate extra memory for the tests, happens by default in our scalatest configuration. Author: Patrick Wendell <[email protected]> Closes #3025 from pwendell/hotfix and squashes the following commits: faa9053 [Patrick Wendell] HOTFIX: Clean up build in network module. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0734d093 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0734d093 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0734d093 Branch: refs/heads/master Commit: 0734d09320fe37edd3a02718511cda0bda852478 Parents: 26d31d1 Author: Patrick Wendell <[email protected]> Authored: Thu Oct 30 20:15:36 2014 -0700 Committer: Aaron Davidson <[email protected]> Committed: Thu Oct 30 20:15:36 2014 -0700 ---------------------------------------------------------------------- core/pom.xml | 2 +- network/common/pom.xml | 34 +++++++++++++++++----------------- project/SparkBuild.scala | 8 +++++--- 3 files changed, 23 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/0734d093/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index 8020a2d..6963ce4 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -46,7 +46,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>network</artifactId> + <artifactId>spark-network-common_2.10</artifactId> <version>${project.version}</version> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/spark/blob/0734d093/network/common/pom.xml ---------------------------------------------------------------------- diff --git a/network/common/pom.xml b/network/common/pom.xml index e3b7e32..a33e44b 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -27,12 +27,12 @@ </parent> <groupId>org.apache.spark</groupId> - <artifactId>network</artifactId> + <artifactId>spark-network-common_2.10</artifactId> <packaging>jar</packaging> - <name>Shuffle Streaming Service</name> + <name>Spark Project Common Network Code</name> <url>http://spark.apache.org/</url> <properties> - <sbt.project.name>network</sbt.project.name> + <sbt.project.name>network-common</sbt.project.name> </properties> <dependencies> @@ -60,6 +60,11 @@ <scope>test</scope> </dependency> <dependency> + <groupId>com.novocode</groupId> + <artifactId>junit-interface</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <scope>test</scope> @@ -69,25 +74,20 @@ <artifactId>mockito-all</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.scalatest</groupId> + <artifactId>scalatest_${scala.binary.version}</artifactId> + <scope>test</scope> + </dependency> </dependencies> - <build> - <outputDirectory>target/java/classes</outputDirectory> - <testOutputDirectory>target/java/test-classes</testOutputDirectory> + <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> + <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> <plugins> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>2.17</version> - <configuration> - <skipTests>false</skipTests> - <includes> - <include>**/Test*.java</include> - <include>**/*Test.java</include> - <include>**/*Suite.java</include> - </includes> - </configuration> + <groupId>org.scalatest</groupId> + <artifactId>scalatest-maven-plugin</artifactId> </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/spark/blob/0734d093/project/SparkBuild.scala ---------------------------------------------------------------------- diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 6d5eb68..7708351 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -31,10 +31,10 @@ object BuildCommons { private val buildLocation = file(".").getAbsoluteFile.getParentFile val allProjects@Seq(bagel, catalyst, core, graphx, hive, hiveThriftServer, mllib, repl, - sql, streaming, streamingFlumeSink, streamingFlume, streamingKafka, streamingMqtt, + sql, networkCommon, streaming, streamingFlumeSink, streamingFlume, streamingKafka, streamingMqtt, streamingTwitter, streamingZeromq) = Seq("bagel", "catalyst", "core", "graphx", "hive", "hive-thriftserver", "mllib", "repl", - "sql", "streaming", "streaming-flume-sink", "streaming-flume", "streaming-kafka", + "sql", "network-common", "streaming", "streaming-flume-sink", "streaming-flume", "streaming-kafka", "streaming-mqtt", "streaming-twitter", "streaming-zeromq").map(ProjectRef(buildLocation, _)) val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, java8Tests, sparkGangliaLgpl, sparkKinesisAsl) = @@ -142,7 +142,9 @@ object SparkBuild extends PomBuild { // TODO: Add Sql to mima checks allProjects.filterNot(x => Seq(spark, sql, hive, hiveThriftServer, catalyst, repl, - streamingFlumeSink).contains(x)).foreach(x => enable(MimaBuild.mimaSettings(sparkHome, x))(x)) + streamingFlumeSink, networkCommon).contains(x)).foreach { + x => enable(MimaBuild.mimaSettings(sparkHome, x))(x) + } /* Enable Assembly for all assembly projects */ assemblyProjects.foreach(enable(Assembly.settings)) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
