From c243b21a8ba2610266702e00d7d4b5443cb1f687 Mon Sep 17 00:00:00 2001
From: Bertrand Bossy <bertrandbossy@gmail.com>
Date: Sun, 14 Sep 2014 21:10:17 -0700
Subject: [PATCH] SPARK-3039: Allow spark to be built using avro-mapred for
hadoop2
SPARK-3039: Adds the maven property "avro.mapred.classifier" to build spark-assembly with avro-mapred with support for the new Hadoop API. Sets this property to hadoop2 for Hadoop 2 profiles.
I am not very familiar with maven, nor do I know whether this potentially breaks something in the hive part of spark. There might be a more elegant way of doing this.
Author: Bertrand Bossy <bertrandbossy@gmail.com>
Closes #1945 from bbossy/SPARK-3039 and squashes the following commits:
c32ce59 [Bertrand Bossy] SPARK-3039: Allow spark to be built using avro-mapred for hadoop2
---
pom.xml | 5 +++++
sql/hive/pom.xml | 9 +++++++++
2 files changed, 14 insertions(+)
diff --git a/pom.xml b/pom.xml
index 28763476f8..520aed3806 100644
--- a/pom.xml
+++ b/pom.xml
@@ -134,6 +134,7 @@
<chill.version>0.3.6</chill.version>
<codahale.metrics.version>3.0.0</codahale.metrics.version>
<avro.version>1.7.6</avro.version>
+ <avro.mapred.classifier></avro.mapred.classifier>
<jets3t.version>0.7.1</jets3t.version>
<aws.java.sdk.version>1.8.3</aws.java.sdk.version>
<aws.kinesis.client.version>1.1.0</aws.kinesis.client.version>
@@ -621,6 +622,7 @@
<groupId>org.apache.avro</groupId>
<artifactId>avro-mapred</artifactId>
<version>${avro.version}</version>
+ <classifier>${avro.mapred.classifier}</classifier>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
@@ -1108,6 +1110,7 @@
<properties>
<hadoop.version>2.2.0</hadoop.version>
<protobuf.version>2.5.0</protobuf.version>
+ <avro.mapred.classifier>hadoop2</avro.mapred.classifier>
</properties>
</profile>
@@ -1117,6 +1120,7 @@
<hadoop.version>2.3.0</hadoop.version>
<protobuf.version>2.5.0</protobuf.version>
<jets3t.version>0.9.0</jets3t.version>
+ <avro.mapred.classifier>hadoop2</avro.mapred.classifier>
</properties>
</profile>
@@ -1126,6 +1130,7 @@
<hadoop.version>2.4.0</hadoop.version>
<protobuf.version>2.5.0</protobuf.version>
<jets3t.version>0.9.0</jets3t.version>
+ <avro.mapred.classifier>hadoop2</avro.mapred.classifier>
</properties>
</profile>
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 45a4c6dc98..9d7a02bf7b 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -95,6 +95,15 @@
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
+ <version>${avro.version}</version>
+ </dependency>
+ <!-- use the build matching the hadoop api of avro-mapred (i.e. no classifier for hadoop 1 API,
+ hadoop2 classifier for hadoop 2 API. avro-mapred is a dependency of org.spark-project.hive:hive-serde -->
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro-mapred</artifactId>
+ <version>${avro.version}</version>
+ <classifier>${avro.mapred.classifier}</classifier>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
--
GitLab