From cd30db566a327ddf63cd242c758e46ce2d9479df Mon Sep 17 00:00:00 2001
From: Sean Owen <sowen@cloudera.com>
Date: Mon, 25 Aug 2014 13:29:07 -0700
Subject: [PATCH] SPARK-2798 [BUILD] Correct several small errors in Flume
module pom.xml files
(EDIT) Since the scalatest issue was since resolved, this is now about a few small problems in the Flume Sink `pom.xml`
- `scalatest` is not declared as a test-scope dependency
- Its Avro version doesn't match the rest of the build
- Its Flume version is not synced with the other Flume module
- The other Flume module declares its dependency on Flume Sink slightly incorrectly, hard-coding the Scala 2.10 version
- It depends on Scala Lang directly, which it shouldn't
Author: Sean Owen <sowen@cloudera.com>
Closes #1726 from srowen/SPARK-2798 and squashes the following commits:
a46e2c6 [Sean Owen] scalatest to test scope, harmonize Avro and Flume versions, remove direct Scala dependency, fix '2.10' in Flume dependency
---
external/flume-sink/pom.xml | 15 ++++++---------
external/flume/pom.xml | 12 ++++++------
pom.xml | 1 +
3 files changed, 13 insertions(+), 15 deletions(-)
diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml
index 19192e40a7..c1e8e65464 100644
--- a/external/flume-sink/pom.xml
+++ b/external/flume-sink/pom.xml
@@ -25,19 +25,20 @@
<relativePath>../../pom.xml</relativePath>
</parent>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-flume-sink_2.10</artifactId>
<properties>
<sbt.project.name>streaming-flume-sink</sbt.project.name>
</properties>
-
<packaging>jar</packaging>
<name>Spark Project External Flume Sink</name>
<url>http://spark.apache.org/</url>
+
<dependencies>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-sdk</artifactId>
- <version>1.4.0</version>
+ <version>${flume.version}</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
@@ -52,7 +53,7 @@
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
- <version>1.4.0</version>
+ <version>${flume.version}</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
@@ -62,11 +63,7 @@
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
</exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
@@ -97,7 +94,7 @@
<plugin>
<groupId>org.apache.avro</groupId>
<artifactId>avro-maven-plugin</artifactId>
- <version>1.7.3</version>
+ <version>${avro.version}</version>
<configuration>
<!-- Generate the output in the same directory as the sbt-avro-plugin -->
<outputDirectory>${project.basedir}/target/scala-${scala.binary.version}/src_managed/main/compiled_avro</outputDirectory>
diff --git a/external/flume/pom.xml b/external/flume/pom.xml
index c532705f39..f71f6b6c4f 100644
--- a/external/flume/pom.xml
+++ b/external/flume/pom.xml
@@ -40,6 +40,11 @@
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-streaming-flume-sink_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
@@ -50,7 +55,7 @@
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-sdk</artifactId>
- <version>1.4.0</version>
+ <version>${flume.version}</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
@@ -82,11 +87,6 @@
<artifactId>junit-interface</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-flume-sink_2.10</artifactId>
- <version>${project.version}</version>
- </dependency>
</dependencies>
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
diff --git a/pom.xml b/pom.xml
index 9cbf3ea599..556b9da3d6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -125,6 +125,7 @@
<protobuf.version>2.4.1</protobuf.version>
<yarn.version>${hadoop.version}</yarn.version>
<hbase.version>0.94.6</hbase.version>
+ <flume.version>1.4.0</flume.version>
<zookeeper.version>3.4.5</zookeeper.version>
<hive.version>0.12.0</hive.version>
<parquet.version>1.4.3</parquet.version>
--
GitLab