From 0734d09320fe37edd3a02718511cda0bda852478 Mon Sep 17 00:00:00 2001
From: Patrick Wendell <pwendell@gmail.com>
Date: Thu, 30 Oct 2014 20:15:36 -0700
Subject: [PATCH] HOTFIX: Clean up build in network module.
This is currently breaking the package build for some people (including me).
This patch does some general clean-up which also fixes the current issue.
- Uses consistent artifact naming
- Adds sbt support for this module
- Changes tests to use scalatest (fixes the original issue[1])
One thing to note, it turns out that scalatest when invoked in the
Maven build doesn't succesfully detect JUnit Java tests. This is
a long standing issue, I noticed it applies to all of our current
test suites as well. I've created SPARK-4159 to fix this.
[1] The original issue is that we need to allocate extra memory
for the tests, happens by default in our scalatest configuration.
Author: Patrick Wendell <pwendell@gmail.com>
Closes #3025 from pwendell/hotfix and squashes the following commits:
faa9053 [Patrick Wendell] HOTFIX: Clean up build in network module.
---
core/pom.xml | 2 +-
network/common/pom.xml | 34 +++++++++++++++++-----------------
project/SparkBuild.scala | 8 +++++---
3 files changed, 23 insertions(+), 21 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 8020a2daf8..6963ce4777 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -46,7 +46,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>network</artifactId>
+ <artifactId>spark-network-common_2.10</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
diff --git a/network/common/pom.xml b/network/common/pom.xml
index e3b7e32870..a33e44b63d 100644
--- a/network/common/pom.xml
+++ b/network/common/pom.xml
@@ -27,12 +27,12 @@
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>network</artifactId>
+ <artifactId>spark-network-common_2.10</artifactId>
<packaging>jar</packaging>
- <name>Shuffle Streaming Service</name>
+ <name>Spark Project Common Network Code</name>
<url>http://spark.apache.org/</url>
<properties>
- <sbt.project.name>network</sbt.project.name>
+ <sbt.project.name>network-common</sbt.project.name>
</properties>
<dependencies>
@@ -59,6 +59,11 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.novocode</groupId>
+ <artifactId>junit-interface</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
@@ -69,25 +74,20 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.binary.version}</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
-
<build>
- <outputDirectory>target/java/classes</outputDirectory>
- <testOutputDirectory>target/java/test-classes</testOutputDirectory>
+ <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
+ <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.17</version>
- <configuration>
- <skipTests>false</skipTests>
- <includes>
- <include>**/Test*.java</include>
- <include>**/*Test.java</include>
- <include>**/*Suite.java</include>
- </includes>
- </configuration>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 6d5eb681c6..77083518bb 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -31,10 +31,10 @@ object BuildCommons {
private val buildLocation = file(".").getAbsoluteFile.getParentFile
val allProjects@Seq(bagel, catalyst, core, graphx, hive, hiveThriftServer, mllib, repl,
- sql, streaming, streamingFlumeSink, streamingFlume, streamingKafka, streamingMqtt,
+ sql, networkCommon, streaming, streamingFlumeSink, streamingFlume, streamingKafka, streamingMqtt,
streamingTwitter, streamingZeromq) =
Seq("bagel", "catalyst", "core", "graphx", "hive", "hive-thriftserver", "mllib", "repl",
- "sql", "streaming", "streaming-flume-sink", "streaming-flume", "streaming-kafka",
+ "sql", "network-common", "streaming", "streaming-flume-sink", "streaming-flume", "streaming-kafka",
"streaming-mqtt", "streaming-twitter", "streaming-zeromq").map(ProjectRef(buildLocation, _))
val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, java8Tests, sparkGangliaLgpl, sparkKinesisAsl) =
@@ -142,7 +142,9 @@ object SparkBuild extends PomBuild {
// TODO: Add Sql to mima checks
allProjects.filterNot(x => Seq(spark, sql, hive, hiveThriftServer, catalyst, repl,
- streamingFlumeSink).contains(x)).foreach(x => enable(MimaBuild.mimaSettings(sparkHome, x))(x))
+ streamingFlumeSink, networkCommon).contains(x)).foreach {
+ x => enable(MimaBuild.mimaSettings(sparkHome, x))(x)
+ }
/* Enable Assembly for all assembly projects */
assemblyProjects.foreach(enable(Assembly.settings))
--
GitLab