diff --git a/make-distribution.sh b/make-distribution.sh index d46edbc50d152f79b3acba41e9ae84e175871dd9..55cbdc14ac0559eeb081e0373d8bb7ae9a04b5cc 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -181,6 +181,7 @@ echo "Spark $VERSION$GITREVSTRING built for Hadoop $SPARK_HADOOP_VERSION" > "$DI # Copy jars cp "$FWDIR"/assembly/target/scala*/*assembly*hadoop*.jar "$DISTDIR/lib/" cp "$FWDIR"/examples/target/scala*/spark-examples*.jar "$DISTDIR/lib/" +cp "$FWDIR"/network/yarn/target/scala*/spark-*-yarn-shuffle.jar "$DISTDIR/lib/" # Copy example sources (needed for python and SQL) mkdir -p "$DISTDIR/examples/src/main" diff --git a/network/common/pom.xml b/network/common/pom.xml index 8b24ebf1ba1f2f53ea84a6fa0b90d924321671f8..a6bee7ed09ad5097d136a5b772bad3a5f72f122c 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -41,12 +41,13 @@ <groupId>io.netty</groupId> <artifactId>netty-all</artifactId> </dependency> + + <!-- Provided dependencies --> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> + <scope>provided</scope> </dependency> - - <!-- Provided dependencies --> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml index a180a5e5f926e2694406d1fb4bde00925693b304..be78331ea9b633d9e4a7cd5fc03359e33074b028 100644 --- a/network/shuffle/pom.xml +++ b/network/shuffle/pom.xml @@ -42,12 +42,13 @@ <artifactId>spark-network-common_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> + + <!-- Provided dependencies --> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> + <scope>provided</scope> </dependency> - - <!-- Provided dependencies --> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index 85960eb85b48256d2ff12aea411d1cb32d6aecf7..7845011ec3200d281accf993dc7aea87ed099e58 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -54,5 +54,38 @@ <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <shadedArtifactAttached>false</shadedArtifactAttached> + <outputFile>${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar</outputFile> + <artifactSet> + <includes> + <include>*:*</include> + </includes> + </artifactSet> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> </build> </project> diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 5eb3ed439cde9e7e3274676a16bf7e45fc7b2b1f..d95d50a1d8124b5da7c9e194386fe483b449e8e4 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -38,12 +38,12 @@ object BuildCommons { "streaming-flume", "streaming-kafka", "streaming-mqtt", "streaming-twitter", "streaming-zeromq").map(ProjectRef(buildLocation, _)) - val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, networkYarn, java8Tests, - sparkGangliaLgpl, sparkKinesisAsl) = Seq("yarn", "yarn-stable", "yarn-alpha", "network-yarn", + val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, java8Tests, + sparkGangliaLgpl, sparkKinesisAsl) = Seq("yarn", "yarn-stable", "yarn-alpha", "java8-tests", "ganglia-lgpl", "kinesis-asl").map(ProjectRef(buildLocation, _)) - val assemblyProjects@Seq(assembly, examples) = Seq("assembly", "examples") - .map(ProjectRef(buildLocation, _)) + val assemblyProjects@Seq(assembly, examples, networkYarn) = + Seq("assembly", "examples", "network-yarn").map(ProjectRef(buildLocation, _)) val tools = ProjectRef(buildLocation, "tools") // Root project. @@ -289,8 +289,15 @@ object Assembly { lazy val settings = assemblySettings ++ Seq( test in assembly := {}, - jarName in assembly <<= (version, moduleName) map { (v, mName) => mName + "-"+v + "-hadoop" + - Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar" }, + jarName in assembly <<= (version, moduleName) map { (v, mName) => + if (mName.contains("network-yarn")) { + // This must match the same name used in maven (see network/yarn/pom.xml) + "spark-" + v + "-yarn-shuffle.jar" + } else { + mName + "-" + v + "-hadoop" + + Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar" + } + }, mergeStrategy in assembly := { case PathList("org", "datanucleus", xs @ _*) => MergeStrategy.discard case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard