From 122302cbf5cbf1133067a5acdffd6ab96765dafe Mon Sep 17 00:00:00 2001
From: Sean Owen <sowen@cloudera.com>
Date: Tue, 17 May 2016 09:55:53 +0100
Subject: [PATCH] [SPARK-15290][BUILD] Move annotations, like @Since /
@DeveloperApi, into spark-tags
## What changes were proposed in this pull request?
(See https://github.com/apache/spark/pull/12416 where most of this was already reviewed and committed; this is just the module structure and move part. This change does not move the annotations into test scope, which was the apparently problem last time.)
Rename `spark-test-tags` -> `spark-tags`; move common annotations like `Since` to `spark-tags`
## How was this patch tested?
Jenkins tests.
Author: Sean Owen <sowen@cloudera.com>
Closes #13074 from srowen/SPARK-15290.
---
common/network-common/pom.xml | 2 +-
common/network-shuffle/pom.xml | 2 +-
common/network-yarn/pom.xml | 2 +-
common/sketch/pom.xml | 2 +-
common/tags/pom.xml | 6 +++---
.../spark/annotation/AlphaComponent.java | 0
.../apache/spark/annotation/DeveloperApi.java | 0
.../apache/spark/annotation/Experimental.java | 0
.../org/apache/spark/annotation/Private.java | 0
.../org/apache/spark/annotation/Since.scala | 0
.../apache/spark/annotation/package-info.java | 0
.../org/apache/spark/annotation/package.scala | 0
common/unsafe/pom.xml | 8 ++++----
core/pom.xml | 2 +-
dev/sparktestsupport/modules.py | 19 +++++++++++++------
external/docker-integration-tests/pom.xml | 2 +-
external/flume-sink/pom.xml | 2 +-
external/flume/pom.xml | 2 +-
external/java8-tests/pom.xml | 2 +-
external/kafka-0-8/pom.xml | 2 +-
external/kinesis-asl/pom.xml | 2 +-
graphx/pom.xml | 2 +-
launcher/pom.xml | 2 +-
mllib-local/pom.xml | 4 ++++
mllib/pom.xml | 2 +-
pom.xml | 3 +--
project/MimaExcludes.scala | 8 ++++++++
project/SparkBuild.scala | 10 +++++-----
repl/pom.xml | 2 +-
sql/catalyst/pom.xml | 2 +-
sql/core/pom.xml | 2 +-
sql/hive-thriftserver/pom.xml | 2 +-
sql/hive/pom.xml | 2 +-
streaming/pom.xml | 2 +-
yarn/pom.xml | 2 +-
35 files changed, 59 insertions(+), 41 deletions(-)
rename {core => common/tags}/src/main/java/org/apache/spark/annotation/AlphaComponent.java (100%)
rename {core => common/tags}/src/main/java/org/apache/spark/annotation/DeveloperApi.java (100%)
rename {core => common/tags}/src/main/java/org/apache/spark/annotation/Experimental.java (100%)
rename {core => common/tags}/src/main/java/org/apache/spark/annotation/Private.java (100%)
rename {core => common/tags}/src/main/scala/org/apache/spark/annotation/Since.scala (100%)
rename {core => common/tags}/src/main/scala/org/apache/spark/annotation/package-info.java (100%)
rename {core => common/tags}/src/main/scala/org/apache/spark/annotation/package.scala (100%)
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index bd507c2cb6..5444ae6d70 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -66,7 +66,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index 810ec10ca0..e736436aec 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -80,7 +80,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>log4j</groupId>
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index bc83ef24c3..1fd3af2e6e 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -48,7 +48,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
<!-- Provided dependencies -->
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 8bc1f52798..bbbb0bd5aa 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -38,7 +38,7 @@
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
</dependencies>
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 8e702b4fef..14e94eca93 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -27,12 +27,12 @@
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_2.11</artifactId>
+ <artifactId>spark-tags_2.11</artifactId>
<packaging>jar</packaging>
- <name>Spark Project Test Tags</name>
+ <name>Spark Project Tags</name>
<url>http://spark.apache.org/</url>
<properties>
- <sbt.project.name>test-tags</sbt.project.name>
+ <sbt.project.name>tags</sbt.project.name>
</properties>
<dependencies>
diff --git a/core/src/main/java/org/apache/spark/annotation/AlphaComponent.java b/common/tags/src/main/java/org/apache/spark/annotation/AlphaComponent.java
similarity index 100%
rename from core/src/main/java/org/apache/spark/annotation/AlphaComponent.java
rename to common/tags/src/main/java/org/apache/spark/annotation/AlphaComponent.java
diff --git a/core/src/main/java/org/apache/spark/annotation/DeveloperApi.java b/common/tags/src/main/java/org/apache/spark/annotation/DeveloperApi.java
similarity index 100%
rename from core/src/main/java/org/apache/spark/annotation/DeveloperApi.java
rename to common/tags/src/main/java/org/apache/spark/annotation/DeveloperApi.java
diff --git a/core/src/main/java/org/apache/spark/annotation/Experimental.java b/common/tags/src/main/java/org/apache/spark/annotation/Experimental.java
similarity index 100%
rename from core/src/main/java/org/apache/spark/annotation/Experimental.java
rename to common/tags/src/main/java/org/apache/spark/annotation/Experimental.java
diff --git a/core/src/main/java/org/apache/spark/annotation/Private.java b/common/tags/src/main/java/org/apache/spark/annotation/Private.java
similarity index 100%
rename from core/src/main/java/org/apache/spark/annotation/Private.java
rename to common/tags/src/main/java/org/apache/spark/annotation/Private.java
diff --git a/core/src/main/scala/org/apache/spark/annotation/Since.scala b/common/tags/src/main/scala/org/apache/spark/annotation/Since.scala
similarity index 100%
rename from core/src/main/scala/org/apache/spark/annotation/Since.scala
rename to common/tags/src/main/scala/org/apache/spark/annotation/Since.scala
diff --git a/core/src/main/scala/org/apache/spark/annotation/package-info.java b/common/tags/src/main/scala/org/apache/spark/annotation/package-info.java
similarity index 100%
rename from core/src/main/scala/org/apache/spark/annotation/package-info.java
rename to common/tags/src/main/scala/org/apache/spark/annotation/package-info.java
diff --git a/core/src/main/scala/org/apache/spark/annotation/package.scala b/common/tags/src/main/scala/org/apache/spark/annotation/package.scala
similarity index 100%
rename from core/src/main/scala/org/apache/spark/annotation/package.scala
rename to common/tags/src/main/scala/org/apache/spark/annotation/package.scala
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index 93b9580f26..d0d1da69ea 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -36,6 +36,10 @@
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
+ </dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>chill_${scala.binary.version}</artifactId>
@@ -59,10 +63,6 @@
</dependency>
<!-- Test dependencies -->
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
- </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
diff --git a/core/pom.xml b/core/pom.xml
index 4b8fb4e5fb..45f8bfcc05 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -331,7 +331,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
</dependencies>
<build>
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index f81db8e44a..d7eb44d7a5 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -92,10 +92,17 @@ class Module(object):
def __hash__(self):
return hash(self.name)
+tags = Module(
+ name="tags",
+ dependencies=[],
+ source_file_regexes=[
+ "common/tags/",
+ ]
+)
catalyst = Module(
name="catalyst",
- dependencies=[],
+ dependencies=[tags],
source_file_regexes=[
"sql/catalyst/",
],
@@ -165,7 +172,7 @@ hivecontext_compatibility = Module(
sketch = Module(
name="sketch",
- dependencies=[],
+ dependencies=[tags],
source_file_regexes=[
"common/sketch/",
],
@@ -177,7 +184,7 @@ sketch = Module(
graphx = Module(
name="graphx",
- dependencies=[],
+ dependencies=[tags],
source_file_regexes=[
"graphx/",
],
@@ -189,7 +196,7 @@ graphx = Module(
streaming = Module(
name="streaming",
- dependencies=[],
+ dependencies=[tags],
source_file_regexes=[
"streaming",
],
@@ -205,7 +212,7 @@ streaming = Module(
# fail other PRs.
streaming_kinesis_asl = Module(
name="streaming-kinesis-asl",
- dependencies=[],
+ dependencies=[tags],
source_file_regexes=[
"external/kinesis-asl/",
"external/kinesis-asl-assembly/",
@@ -270,7 +277,7 @@ streaming_flume_assembly = Module(
mllib_local = Module(
name="mllib-local",
- dependencies=[],
+ dependencies=[tags],
source_file_regexes=[
"mllib-local",
],
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index 3169a0ebae..21d40863b7 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -126,7 +126,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml
index e4effe158c..016af934bb 100644
--- a/external/flume-sink/pom.xml
+++ b/external/flume-sink/pom.xml
@@ -92,7 +92,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
</dependencies>
<build>
diff --git a/external/flume/pom.xml b/external/flume/pom.xml
index d650dd034d..f51d334de0 100644
--- a/external/flume/pom.xml
+++ b/external/flume/pom.xml
@@ -68,7 +68,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
</dependencies>
<build>
diff --git a/external/java8-tests/pom.xml b/external/java8-tests/pom.xml
index 1ea9196e9d..60e3ff60df 100644
--- a/external/java8-tests/pom.xml
+++ b/external/java8-tests/pom.xml
@@ -72,7 +72,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
</dependencies>
diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml
index cccfda3c61..4a20b78917 100644
--- a/external/kafka-0-8/pom.xml
+++ b/external/kafka-0-8/pom.xml
@@ -88,7 +88,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
</dependencies>
<build>
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 935155eb5d..b5f5ff2854 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -77,7 +77,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
</dependencies>
<build>
diff --git a/graphx/pom.xml b/graphx/pom.xml
index 1813f383cd..fc6c700dd1 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -72,7 +72,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
</dependencies>
<build>
diff --git a/launcher/pom.xml b/launcher/pom.xml
index ef73194882..e7303853e6 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -65,7 +65,7 @@
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
<!-- Not needed by the test code, but referenced by SparkSubmit which is used by the tests. -->
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index 68f15dd905..1c6ab2b62d 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -53,6 +53,10 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
+ </dependency>
</dependencies>
<profiles>
<profile>
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 24d8274e22..c324afb2a2 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -116,7 +116,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
</dependencies>
<profiles>
diff --git a/pom.xml b/pom.xml
index 864824d86c..9c13af17e4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -286,9 +286,8 @@
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
<version>${project.version}</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 69161e0d61..1a02f660fd 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -716,6 +716,14 @@ object MimaExcludes {
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.executor.ShuffleReadMetrics.localBlocksFetched"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.ShuffleReadMetrics.remoteBlocksFetched"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.ShuffleReadMetrics.localBlocksFetched")
+ ) ++ Seq(
+ // [SPARK-15290] Move annotations, like @Since / @DeveloperApi, into spark-tags
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.annotation.package$"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.annotation.package"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.annotation.Private"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.annotation.AlphaComponent"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.annotation.Experimental"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.annotation.DeveloperApi")
)
case v if v.startsWith("1.6") =>
Seq(
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index d83afa03f5..3ad9873f43 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -50,10 +50,10 @@ object BuildCommons {
).map(ProjectRef(buildLocation, _))
val allProjects@Seq(
- core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, testTags, sketch, _*
+ core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, tags, sketch, _*
) = Seq(
"core", "graphx", "mllib", "mllib-local", "repl", "network-common", "network-shuffle", "launcher", "unsafe",
- "test-tags", "sketch"
+ "tags", "sketch"
).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects
val optionallyEnabledProjects@Seq(yarn, java8Tests, sparkGangliaLgpl,
@@ -340,7 +340,7 @@ object SparkBuild extends PomBuild {
val mimaProjects = allProjects.filterNot { x =>
Seq(
spark, hive, hiveThriftServer, hiveCompatibility, catalyst, repl, networkCommon, networkShuffle, networkYarn,
- unsafe, testTags, sketch, mllibLocal
+ unsafe, tags, sketch, mllibLocal
).contains(x)
}
@@ -685,9 +685,9 @@ object Unidoc {
publish := {},
unidocProjectFilter in(ScalaUnidoc, unidoc) :=
- inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, streamingFlumeSink, yarn, testTags),
+ inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, streamingFlumeSink, yarn, tags),
unidocProjectFilter in(JavaUnidoc, unidoc) :=
- inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, streamingFlumeSink, yarn, testTags),
+ inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, streamingFlumeSink, yarn, tags),
// Skip actual catalyst, but include the subproject.
// Catalyst is not public API and contains quasiquotes which break scaladoc.
diff --git a/repl/pom.xml b/repl/pom.xml
index 0f396c9b80..c12d121c61 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -87,7 +87,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>org.apache.xbean</groupId>
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 1748fa2778..1923199f4b 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -55,7 +55,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 184fa2f6bd..2ea980bf20 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -73,7 +73,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 5cf16e42b9..809d36dc69 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -84,7 +84,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>net.sf.jpam</groupId>
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 177b6884fa..c8b20f0afc 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -60,7 +60,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
<!--
<dependency>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 2055cffe75..3f67745936 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -49,7 +49,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
<!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. -->
diff --git a/yarn/pom.xml b/yarn/pom.xml
index 11df2b3f4f..e07b93ab95 100644
--- a/yarn/pom.xml
+++ b/yarn/pom.xml
@@ -54,7 +54,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_${scala.binary.version}</artifactId>
+ <artifactId>spark-tags_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
--
GitLab