Skip to content
Snippets Groups Projects
Commit 856c50f5 authored by Sean Owen's avatar Sean Owen Committed by Patrick Wendell
Browse files

SPARK-1387. Update build plugins, avoid plugin version warning, centralize versions 

Another handful of small build changes to organize and standardize a bit, and avoid warnings:

- Update Maven plugin versions for good measure
- Since plugins need maven 3.0.4 already, require it explicitly (<3.0.4 had some bugs anyway)
- Use variables to define versions across dependencies where they should move in lock step
- ... and make this consistent between Maven/SBT

OK, I also updated the JIRA URL while I was at it here.

Author: Sean Owen <sowen@cloudera.com>

Closes #291 from srowen/SPARK-1387 and squashes the following commits:

461eca1 [Sean Owen] Couldn't resist also updating JIRA location to new one
c2d5cc5 [Sean Owen] Update plugins and Maven version; use variables consistently across Maven/SBT to define dependency versions that should stay in step.
parent e258e504
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
assembly/pom.xml
+ 1
− 1
View file @ 856c50f5
......@@ -208,7 +208,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<version>1.1</version>
<version>1.2</version>
<executions>
<execution>
<phase>validate</phase>
......
core/pom.xml
+ 0
− 2
View file @ 856c50f5
......@@ -117,12 +117,10 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>chill_${scala.binary.version}</artifactId>
<version>0.3.1</version>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>chill-java</artifactId>
<version>0.3.1</version>
</dependency>
<dependency>
<groupId>commons-net</groupId>
......
dev/audit-release/maven_app_core/pom.xml
+ 1
− 1
View file @ 856c50f5
......@@ -49,7 +49,7 @@
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<version>3.1</version>
</plugin>
</plugins>
</build>
......
docs/building-with-maven.md
+ 1
− 1
View file @ 856c50f5
......@@ -6,7 +6,7 @@ title: Building Spark with Maven
* This will become a table of contents (this text will be scraped).
{:toc}
Building Spark using Maven Requires Maven 3 (the build process is tested with Maven 3.0.4) and Java 1.6 or newer.
Building Spark using Maven requires Maven 3.0.4 or newer and Java 1.6 or newer.
## Setting up Maven's Memory Usage ##
......
examples/pom.xml
+ 1
− 1
View file @ 856c50f5
......@@ -110,7 +110,7 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
<version>0.94.6</version>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
......
graphx/pom.xml
+ 1
− 1
View file @ 856c50f5
......@@ -54,7 +54,7 @@
<dependency>
<groupId>org.jblas</groupId>
<artifactId>jblas</artifactId>
<version>1.2.3</version>
<version>${jblas.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
......
mllib/pom.xml
+ 1
− 1
View file @ 856c50f5
......@@ -58,7 +58,7 @@
<dependency>
<groupId>org.jblas</groupId>
<artifactId>jblas</artifactId>
<version>1.2.3</version>
<version>${jblas.version}</version>
</dependency>
<dependency>
<groupId>org.scalanlp</groupId>
......
pom.xml
+ 23
− 20
View file @ 856c50f5
......@@ -54,11 +54,11 @@
</developers>
<issueManagement>
<system>JIRA</system>
<url>https://spark-project.atlassian.net/browse/SPARK</url>
<url>https://issues.apache.org/jira/browse/SPARK</url>
</issueManagement>
<prerequisites>
<maven>3.0.0</maven>
<maven>3.0.4</maven>
</prerequisites>
<mailingLists>
......@@ -123,6 +123,10 @@
<hbase.version>0.94.6</hbase.version>
<hive.version>0.12.0</hive.version>
<parquet.version>1.3.2</parquet.version>
<jblas.version>1.2.3</jblas.version>
<jetty.version>8.1.14.v20131031</jetty.version>
<chill.version>0.3.1</chill.version>
<codahale.metrics.version>3.0.0</codahale.metrics.version>
<PermGen>64m</PermGen>
<MaxPermGen>512m</MaxPermGen>
......@@ -192,22 +196,22 @@
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
<version>8.1.14.v20131031</version>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-security</artifactId>
<version>8.1.14.v20131031</version>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-plus</artifactId>
<version>8.1.14.v20131031</version>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>8.1.14.v20131031</version>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
......@@ -273,7 +277,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>chill_${scala.binary.version}</artifactId>
<version>0.3.1</version>
<version>${chill.version}</version>
<exclusions>
<exclusion>
<groupId>org.ow2.asm</groupId>
......@@ -288,7 +292,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>chill-java</artifactId>
<version>0.3.1</version>
<version>${chill.version}</version>
<exclusions>
<exclusion>
<groupId>org.ow2.asm</groupId>
......@@ -392,27 +396,27 @@
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>3.0.0</version>
<version>${codahale.metrics.version}</version>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-jvm</artifactId>
<version>3.0.0</version>
<version>${codahale.metrics.version}</version>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-json</artifactId>
<version>3.0.0</version>
<version>${codahale.metrics.version}</version>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-ganglia</artifactId>
<version>3.0.0</version>
<version>${codahale.metrics.version}</version>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-graphite</artifactId>
<version>3.0.0</version>
<version>${codahale.metrics.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
......@@ -585,7 +589,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.1.1</version>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce-versions</id>
......@@ -595,7 +599,7 @@
<configuration>
<rules>
<requireMavenVersion>
<version>3.0.0</version>
<version>3.0.4</version>
</requireMavenVersion>
<requireJavaVersion>
<version>${java.version}</version>
......@@ -608,12 +612,12 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>1.7</version>
<version>1.8</version>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.1.5</version>
<version>3.1.6</version>
<executions>
<execution>
<id>scala-compile-first</id>
......@@ -674,7 +678,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.12.4</version>
<version>2.17</version>
<configuration>
<!-- Uses scalatest instead -->
<skipTests>true</skipTests>
......@@ -713,7 +717,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.0</version>
<version>2.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
......@@ -810,7 +814,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<goals>
......
project/SparkBuild.scala
+ 28
− 21
View file @ 856c50f5
......@@ -248,10 +248,10 @@ object SparkBuild extends Build {
libraryDependencies ++= Seq(
"io.netty" % "netty-all" % "4.0.17.Final",
"org.eclipse.jetty" % "jetty-server" % "8.1.14.v20131031",
"org.eclipse.jetty" % "jetty-util" % "8.1.14.v20131031",
"org.eclipse.jetty" % "jetty-plus" % "8.1.14.v20131031",
"org.eclipse.jetty" % "jetty-security" % "8.1.14.v20131031",
"org.eclipse.jetty" % "jetty-server" % jettyVersion,
"org.eclipse.jetty" % "jetty-util" % jettyVersion,
"org.eclipse.jetty" % "jetty-plus" % jettyVersion,
"org.eclipse.jetty" % "jetty-security" % jettyVersion,
/** Workaround for SPARK-959. Dependency used by org.eclipse.jetty. Fixed in ivy 2.3.0. */
"org.eclipse.jetty.orbit" % "javax.servlet" % "3.0.0.v201112011016" artifacts Artifact("javax.servlet", "jar", "jar"),
"org.scalatest" %% "scalatest" % "1.9.1" % "test",
......@@ -276,6 +276,13 @@ object SparkBuild extends Build {
publishLocalBoth <<= Seq(publishLocal in MavenCompile, publishLocal).dependOn
) ++ net.virtualvoid.sbt.graph.Plugin.graphSettings ++ ScalaStyleSettings
val akkaVersion = "2.2.3-shaded-protobuf"
val chillVersion = "0.3.1"
val codahaleMetricsVersion = "3.0.0"
val jblasVersion = "1.2.3"
val jettyVersion = "8.1.14.v20131031"
val hiveVersion = "0.12.0"
val parquetVersion = "1.3.2"
val slf4jVersion = "1.7.5"
val excludeNetty = ExclusionRule(organization = "org.jboss.netty")
......@@ -309,9 +316,9 @@ object SparkBuild extends Build {
"commons-daemon" % "commons-daemon" % "1.0.10", // workaround for bug HADOOP-9407
"com.ning" % "compress-lzf" % "1.0.0",
"org.xerial.snappy" % "snappy-java" % "1.0.5",
"org.spark-project.akka" %% "akka-remote" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
"org.spark-project.akka" %% "akka-slf4j" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty),
"org.spark-project.akka" %% "akka-testkit" % "2.2.3-shaded-protobuf" % "test",
"org.spark-project.akka" %% "akka-remote" % akkaVersion excludeAll(excludeNetty),
"org.spark-project.akka" %% "akka-slf4j" % akkaVersion excludeAll(excludeNetty),
"org.spark-project.akka" %% "akka-testkit" % akkaVersion % "test",
"org.json4s" %% "json4s-jackson" % "3.2.6" excludeAll(excludeScalap),
"it.unimi.dsi" % "fastutil" % "6.4.4",
"colt" % "colt" % "1.2.0",
......@@ -321,12 +328,12 @@ object SparkBuild extends Build {
"org.apache.derby" % "derby" % "10.4.2.0" % "test",
"org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J, excludeOldAsm),
"org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeNetty),
"com.codahale.metrics" % "metrics-core" % "3.0.0",
"com.codahale.metrics" % "metrics-jvm" % "3.0.0",
"com.codahale.metrics" % "metrics-json" % "3.0.0",
"com.codahale.metrics" % "metrics-graphite" % "3.0.0",
"com.twitter" %% "chill" % "0.3.1" excludeAll(excludeAsm),
"com.twitter" % "chill-java" % "0.3.1" excludeAll(excludeAsm),
"com.codahale.metrics" % "metrics-core" % codahaleMetricsVersion,
"com.codahale.metrics" % "metrics-jvm" % codahaleMetricsVersion,
"com.codahale.metrics" % "metrics-json" % codahaleMetricsVersion,
"com.codahale.metrics" % "metrics-graphite" % codahaleMetricsVersion,
"com.twitter" %% "chill" % chillVersion excludeAll(excludeAsm),
"com.twitter" % "chill-java" % chillVersion excludeAll(excludeAsm),
"org.tachyonproject" % "tachyon" % "0.4.1-thrift" excludeAll(excludeHadoop, excludeCurator, excludeEclipseJetty, excludePowermock),
"com.clearspring.analytics" % "stream" % "2.5.1"
),
......@@ -370,7 +377,7 @@ object SparkBuild extends Build {
name := "spark-graphx",
previousArtifact := sparkPreviousArtifact("spark-graphx"),
libraryDependencies ++= Seq(
"org.jblas" % "jblas" % "1.2.3"
"org.jblas" % "jblas" % jblasVersion
)
)
......@@ -383,7 +390,7 @@ object SparkBuild extends Build {
name := "spark-mllib",
previousArtifact := sparkPreviousArtifact("spark-mllib"),
libraryDependencies ++= Seq(
"org.jblas" % "jblas" % "1.2.3",
"org.jblas" % "jblas" % jblasVersion,
"org.scalanlp" %% "breeze" % "0.7"
)
)
......@@ -403,8 +410,8 @@ object SparkBuild extends Build {
def sqlCoreSettings = sharedSettings ++ Seq(
name := "spark-sql",
libraryDependencies ++= Seq(
"com.twitter" % "parquet-column" % "1.3.2",
"com.twitter" % "parquet-hadoop" % "1.3.2"
"com.twitter" % "parquet-column" % parquetVersion,
"com.twitter" % "parquet-hadoop" % parquetVersion
)
)
......@@ -416,9 +423,9 @@ object SparkBuild extends Build {
jarName in packageDependency <<= version map { v => "spark-hive-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" },
javaOptions += "-XX:MaxPermSize=1g",
libraryDependencies ++= Seq(
"org.apache.hive" % "hive-metastore" % "0.12.0",
"org.apache.hive" % "hive-exec" % "0.12.0",
"org.apache.hive" % "hive-serde" % "0.12.0"
"org.apache.hive" % "hive-metastore" % hiveVersion,
"org.apache.hive" % "hive-exec" % hiveVersion,
"org.apache.hive" % "hive-serde" % hiveVersion
),
// Multiple queries rely on the TestHive singleton. See comments there for more details.
parallelExecution in Test := false,
......@@ -549,7 +556,7 @@ object SparkBuild extends Build {
name := "spark-streaming-zeromq",
previousArtifact := sparkPreviousArtifact("spark-streaming-zeromq"),
libraryDependencies ++= Seq(
"org.spark-project.akka" %% "akka-zeromq" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty)
"org.spark-project.akka" %% "akka-zeromq" % akkaVersion excludeAll(excludeNetty)
)
)
......
streaming/pom.xml
+ 0
− 1
View file @ 856c50f5
......@@ -96,7 +96,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.2</version>
<executions>
<execution>
<goals>
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment