From 256704c771d301700af9ebf0d180c1ba7c4116c0 Mon Sep 17 00:00:00 2001
From: Sean Owen <sowen@cloudera.com>
Date: Wed, 9 Mar 2016 18:27:44 +0000
Subject: [PATCH] [SPARK-13595][BUILD] Move docker, extras modules into
external
## What changes were proposed in this pull request?
Move `docker` dirs out of top level into `external/`; move `extras/*` into `external/`
## How was this patch tested?
This is tested with Jenkins tests.
Author: Sean Owen <sowen@cloudera.com>
Closes #11523 from srowen/SPARK-13595.
---
dev/sparktestsupport/modules.py | 4 ++--
docs/streaming-kinesis-integration.md | 10 +++++-----
.../docker-integration-tests}/pom.xml | 2 +-
.../spark/sql/jdbc/DockerJDBCIntegrationSuite.scala | 0
.../apache/spark/sql/jdbc/MySQLIntegrationSuite.scala | 0
.../apache/spark/sql/jdbc/OracleIntegrationSuite.scala | 0
.../spark/sql/jdbc/PostgresIntegrationSuite.scala | 0
.../test/scala/org/apache/spark/util/DockerUtils.scala | 0
{docker => external/docker}/README.md | 0
{docker => external/docker}/build | 0
{docker => external/docker}/spark-mesos/Dockerfile | 0
{docker => external/docker}/spark-test/README.md | 0
{docker => external/docker}/spark-test/base/Dockerfile | 0
{docker => external/docker}/spark-test/build | 0
.../docker}/spark-test/master/Dockerfile | 0
.../docker}/spark-test/master/default_cmd | 0
.../docker}/spark-test/worker/Dockerfile | 0
.../docker}/spark-test/worker/default_cmd | 0
{extras => external}/java8-tests/README.md | 0
{extras => external}/java8-tests/pom.xml | 0
.../src/test/java/org/apache/spark/Java8APISuite.java | 0
.../java/org/apache/spark/streaming/Java8APISuite.java | 0
.../java8-tests/src/test/resources/log4j.properties | 0
.../test/scala/org/apache/spark/JDK8ScalaSuite.scala | 0
{extras => external}/kinesis-asl-assembly/pom.xml | 0
{extras => external}/kinesis-asl/pom.xml | 0
.../examples/streaming/JavaKinesisWordCountASL.java | 0
.../python/examples/streaming/kinesis_wordcount_asl.py | 4 ++--
.../kinesis-asl/src/main/resources/log4j.properties | 0
.../spark/examples/streaming/KinesisWordCountASL.scala | 0
.../streaming/kinesis/KinesisBackedBlockRDD.scala | 0
.../spark/streaming/kinesis/KinesisCheckpointer.scala | 0
.../spark/streaming/kinesis/KinesisInputDStream.scala | 0
.../spark/streaming/kinesis/KinesisReceiver.scala | 0
.../streaming/kinesis/KinesisRecordProcessor.scala | 0
.../spark/streaming/kinesis/KinesisTestUtils.scala | 0
.../apache/spark/streaming/kinesis/KinesisUtils.scala | 0
.../streaming/kinesis/JavaKinesisStreamSuite.java | 0
.../kinesis-asl/src/test/resources/log4j.properties | 0
.../streaming/kinesis/KPLBasedKinesisTestUtils.scala | 0
.../streaming/kinesis/KinesisBackedBlockRDDSuite.scala | 0
.../streaming/kinesis/KinesisCheckpointerSuite.scala | 0
.../spark/streaming/kinesis/KinesisFunSuite.scala | 0
.../spark/streaming/kinesis/KinesisReceiverSuite.scala | 0
.../spark/streaming/kinesis/KinesisStreamSuite.scala | 0
{extras => external}/spark-ganglia-lgpl/pom.xml | 0
.../org/apache/spark/metrics/sink/GangliaSink.scala | 0
extras/README.md | 1 -
pom.xml | 10 +++++-----
python/pyspark/streaming/tests.py | 2 +-
50 files changed, 16 insertions(+), 17 deletions(-)
rename {docker-integration-tests => external/docker-integration-tests}/pom.xml (99%)
rename {docker-integration-tests => external/docker-integration-tests}/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala (100%)
rename {docker-integration-tests => external/docker-integration-tests}/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala (100%)
rename {docker-integration-tests => external/docker-integration-tests}/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala (100%)
rename {docker-integration-tests => external/docker-integration-tests}/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala (100%)
rename {docker-integration-tests => external/docker-integration-tests}/src/test/scala/org/apache/spark/util/DockerUtils.scala (100%)
rename {docker => external/docker}/README.md (100%)
rename {docker => external/docker}/build (100%)
rename {docker => external/docker}/spark-mesos/Dockerfile (100%)
rename {docker => external/docker}/spark-test/README.md (100%)
rename {docker => external/docker}/spark-test/base/Dockerfile (100%)
rename {docker => external/docker}/spark-test/build (100%)
rename {docker => external/docker}/spark-test/master/Dockerfile (100%)
rename {docker => external/docker}/spark-test/master/default_cmd (100%)
rename {docker => external/docker}/spark-test/worker/Dockerfile (100%)
rename {docker => external/docker}/spark-test/worker/default_cmd (100%)
rename {extras => external}/java8-tests/README.md (100%)
rename {extras => external}/java8-tests/pom.xml (100%)
rename {extras => external}/java8-tests/src/test/java/org/apache/spark/Java8APISuite.java (100%)
rename {extras => external}/java8-tests/src/test/java/org/apache/spark/streaming/Java8APISuite.java (100%)
rename {extras => external}/java8-tests/src/test/resources/log4j.properties (100%)
rename {extras => external}/java8-tests/src/test/scala/org/apache/spark/JDK8ScalaSuite.scala (100%)
rename {extras => external}/kinesis-asl-assembly/pom.xml (100%)
rename {extras => external}/kinesis-asl/pom.xml (100%)
rename {extras => external}/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java (100%)
rename {extras => external}/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py (95%)
rename {extras => external}/kinesis-asl/src/main/resources/log4j.properties (100%)
rename {extras => external}/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala (100%)
rename {extras => external}/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala (100%)
rename {extras => external}/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala (100%)
rename {extras => external}/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala (100%)
rename {extras => external}/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala (100%)
rename {extras => external}/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala (100%)
rename {extras => external}/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala (100%)
rename {extras => external}/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtils.scala (100%)
rename {extras => external}/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java (100%)
rename {extras => external}/kinesis-asl/src/test/resources/log4j.properties (100%)
rename {extras => external}/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala (100%)
rename {extras => external}/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala (100%)
rename {extras => external}/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala (100%)
rename {extras => external}/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisFunSuite.scala (100%)
rename {extras => external}/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala (100%)
rename {extras => external}/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala (100%)
rename {extras => external}/spark-ganglia-lgpl/pom.xml (100%)
rename {extras => external}/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala (100%)
delete mode 100644 extras/README.md
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index e4f2edaf95..1781de4c65 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -195,8 +195,8 @@ streaming_kinesis_asl = Module(
name="streaming-kinesis-asl",
dependencies=[],
source_file_regexes=[
- "extras/kinesis-asl/",
- "extras/kinesis-asl-assembly/",
+ "external/kinesis-asl/",
+ "external/kinesis-asl-assembly/",
],
build_profile_flags=[
"-Pkinesis-asl",
diff --git a/docs/streaming-kinesis-integration.md b/docs/streaming-kinesis-integration.md
index 2a868e8bca..5b9a7554d2 100644
--- a/docs/streaming-kinesis-integration.md
+++ b/docs/streaming-kinesis-integration.md
@@ -37,7 +37,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
[region name], [initial position], [checkpoint interval], StorageLevel.MEMORY_AND_DISK_2)
See the [API docs](api/scala/index.html#org.apache.spark.streaming.kinesis.KinesisUtils$)
- and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala). Refer to the [Running the Example](#running-the-example) subsection for instructions on how to run the example.
+ and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala). Refer to the [Running the Example](#running-the-example) subsection for instructions on how to run the example.
</div>
<div data-lang="java" markdown="1">
@@ -50,7 +50,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
[region name], [initial position], [checkpoint interval], StorageLevel.MEMORY_AND_DISK_2);
See the [API docs](api/java/index.html?org/apache/spark/streaming/kinesis/KinesisUtils.html)
- and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/extras/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java). Refer to the [Running the Example](#running-the-example) subsection for instructions to run the example.
+ and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java). Refer to the [Running the Example](#running-the-example) subsection for instructions to run the example.
</div>
<div data-lang="python" markdown="1">
@@ -61,7 +61,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
[region name], [initial position], [checkpoint interval], StorageLevel.MEMORY_AND_DISK_2)
See the [API docs](api/python/pyspark.streaming.html#pyspark.streaming.kinesis.KinesisUtils)
- and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/extras/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py). Refer to the [Running the Example](#running-the-example) subsection for instructions to run the example.
+ and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py). Refer to the [Running the Example](#running-the-example) subsection for instructions to run the example.
</div>
</div>
@@ -190,9 +190,9 @@ To run the example,
</div>
<div data-lang="python" markdown="1">
- bin/spark-submit --jars extras/kinesis-asl/target/scala-*/\
+ bin/spark-submit --jars external/kinesis-asl/target/scala-*/\
spark-streaming-kinesis-asl-assembly_*.jar \
- extras/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \
+ external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \
[Kinesis app name] [Kinesis stream name] [endpoint URL] [region name]
</div>
diff --git a/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
similarity index 99%
rename from docker-integration-tests/pom.xml
rename to external/docker-integration-tests/pom.xml
index 048e58d888..1764aa9465 100644
--- a/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -23,7 +23,7 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
+ <relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>spark-docker-integration-tests_2.11</artifactId>
diff --git a/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
similarity index 100%
rename from docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
rename to external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
diff --git a/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
similarity index 100%
rename from docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
rename to external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
diff --git a/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
similarity index 100%
rename from docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
rename to external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
diff --git a/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
similarity index 100%
rename from docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
rename to external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
diff --git a/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala
similarity index 100%
rename from docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala
rename to external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala
diff --git a/docker/README.md b/external/docker/README.md
similarity index 100%
rename from docker/README.md
rename to external/docker/README.md
diff --git a/docker/build b/external/docker/build
similarity index 100%
rename from docker/build
rename to external/docker/build
diff --git a/docker/spark-mesos/Dockerfile b/external/docker/spark-mesos/Dockerfile
similarity index 100%
rename from docker/spark-mesos/Dockerfile
rename to external/docker/spark-mesos/Dockerfile
diff --git a/docker/spark-test/README.md b/external/docker/spark-test/README.md
similarity index 100%
rename from docker/spark-test/README.md
rename to external/docker/spark-test/README.md
diff --git a/docker/spark-test/base/Dockerfile b/external/docker/spark-test/base/Dockerfile
similarity index 100%
rename from docker/spark-test/base/Dockerfile
rename to external/docker/spark-test/base/Dockerfile
diff --git a/docker/spark-test/build b/external/docker/spark-test/build
similarity index 100%
rename from docker/spark-test/build
rename to external/docker/spark-test/build
diff --git a/docker/spark-test/master/Dockerfile b/external/docker/spark-test/master/Dockerfile
similarity index 100%
rename from docker/spark-test/master/Dockerfile
rename to external/docker/spark-test/master/Dockerfile
diff --git a/docker/spark-test/master/default_cmd b/external/docker/spark-test/master/default_cmd
similarity index 100%
rename from docker/spark-test/master/default_cmd
rename to external/docker/spark-test/master/default_cmd
diff --git a/docker/spark-test/worker/Dockerfile b/external/docker/spark-test/worker/Dockerfile
similarity index 100%
rename from docker/spark-test/worker/Dockerfile
rename to external/docker/spark-test/worker/Dockerfile
diff --git a/docker/spark-test/worker/default_cmd b/external/docker/spark-test/worker/default_cmd
similarity index 100%
rename from docker/spark-test/worker/default_cmd
rename to external/docker/spark-test/worker/default_cmd
diff --git a/extras/java8-tests/README.md b/external/java8-tests/README.md
similarity index 100%
rename from extras/java8-tests/README.md
rename to external/java8-tests/README.md
diff --git a/extras/java8-tests/pom.xml b/external/java8-tests/pom.xml
similarity index 100%
rename from extras/java8-tests/pom.xml
rename to external/java8-tests/pom.xml
diff --git a/extras/java8-tests/src/test/java/org/apache/spark/Java8APISuite.java b/external/java8-tests/src/test/java/org/apache/spark/Java8APISuite.java
similarity index 100%
rename from extras/java8-tests/src/test/java/org/apache/spark/Java8APISuite.java
rename to external/java8-tests/src/test/java/org/apache/spark/Java8APISuite.java
diff --git a/extras/java8-tests/src/test/java/org/apache/spark/streaming/Java8APISuite.java b/external/java8-tests/src/test/java/org/apache/spark/streaming/Java8APISuite.java
similarity index 100%
rename from extras/java8-tests/src/test/java/org/apache/spark/streaming/Java8APISuite.java
rename to external/java8-tests/src/test/java/org/apache/spark/streaming/Java8APISuite.java
diff --git a/extras/java8-tests/src/test/resources/log4j.properties b/external/java8-tests/src/test/resources/log4j.properties
similarity index 100%
rename from extras/java8-tests/src/test/resources/log4j.properties
rename to external/java8-tests/src/test/resources/log4j.properties
diff --git a/extras/java8-tests/src/test/scala/org/apache/spark/JDK8ScalaSuite.scala b/external/java8-tests/src/test/scala/org/apache/spark/JDK8ScalaSuite.scala
similarity index 100%
rename from extras/java8-tests/src/test/scala/org/apache/spark/JDK8ScalaSuite.scala
rename to external/java8-tests/src/test/scala/org/apache/spark/JDK8ScalaSuite.scala
diff --git a/extras/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
similarity index 100%
rename from extras/kinesis-asl-assembly/pom.xml
rename to external/kinesis-asl-assembly/pom.xml
diff --git a/extras/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
similarity index 100%
rename from extras/kinesis-asl/pom.xml
rename to external/kinesis-asl/pom.xml
diff --git a/extras/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java b/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
similarity index 100%
rename from extras/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
rename to external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
diff --git a/extras/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
similarity index 95%
rename from extras/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
rename to external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
index 51f8c5ca66..4d7fc9a549 100644
--- a/extras/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
+++ b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
@@ -34,9 +34,9 @@
$ export AWS_SECRET_KEY=<your-secret-key>
# run the example
- $ bin/spark-submit -jar extras/kinesis-asl/target/scala-*/\
+ $ bin/spark-submit -jar external/kinesis-asl/target/scala-*/\
spark-streaming-kinesis-asl-assembly_*.jar \
- extras/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \
+ external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \
myAppName mySparkStream https://kinesis.us-east-1.amazonaws.com
There is a companion helper class called KinesisWordProducerASL which puts dummy data
diff --git a/extras/kinesis-asl/src/main/resources/log4j.properties b/external/kinesis-asl/src/main/resources/log4j.properties
similarity index 100%
rename from extras/kinesis-asl/src/main/resources/log4j.properties
rename to external/kinesis-asl/src/main/resources/log4j.properties
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
similarity index 100%
rename from extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
rename to external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
similarity index 100%
rename from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
rename to external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
similarity index 100%
rename from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
rename to external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala
similarity index 100%
rename from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala
rename to external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala
similarity index 100%
rename from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala
rename to external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala
similarity index 100%
rename from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala
rename to external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
similarity index 100%
rename from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
rename to external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtils.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtils.scala
similarity index 100%
rename from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtils.scala
rename to external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtils.scala
diff --git a/extras/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java b/external/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java
similarity index 100%
rename from extras/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java
rename to external/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java
diff --git a/extras/kinesis-asl/src/test/resources/log4j.properties b/external/kinesis-asl/src/test/resources/log4j.properties
similarity index 100%
rename from extras/kinesis-asl/src/test/resources/log4j.properties
rename to external/kinesis-asl/src/test/resources/log4j.properties
diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala
similarity index 100%
rename from extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala
rename to external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala
diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala
similarity index 100%
rename from extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala
rename to external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala
diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala
similarity index 100%
rename from extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala
rename to external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala
diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisFunSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisFunSuite.scala
similarity index 100%
rename from extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisFunSuite.scala
rename to external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisFunSuite.scala
diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
similarity index 100%
rename from extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
rename to external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala
similarity index 100%
rename from extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala
rename to external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala
diff --git a/extras/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
similarity index 100%
rename from extras/spark-ganglia-lgpl/pom.xml
rename to external/spark-ganglia-lgpl/pom.xml
diff --git a/extras/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala b/external/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala
similarity index 100%
rename from extras/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala
rename to external/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala
diff --git a/extras/README.md b/extras/README.md
deleted file mode 100644
index 1b4174b7d5..0000000000
--- a/extras/README.md
+++ /dev/null
@@ -1 +0,0 @@
-This directory contains build components not included by default in Spark's build.
diff --git a/pom.xml b/pom.xml
index 90f46724e1..a3af20cb04 100644
--- a/pom.xml
+++ b/pom.xml
@@ -99,7 +99,7 @@
<module>sql/catalyst</module>
<module>sql/core</module>
<module>sql/hive</module>
- <module>docker-integration-tests</module>
+ <module>external/docker-integration-tests</module>
<module>assembly</module>
<module>external/twitter</module>
<module>external/flume</module>
@@ -2354,7 +2354,7 @@
<profile>
<id>spark-ganglia-lgpl</id>
<modules>
- <module>extras/spark-ganglia-lgpl</module>
+ <module>external/spark-ganglia-lgpl</module>
</modules>
</profile>
@@ -2362,8 +2362,8 @@
<profile>
<id>kinesis-asl</id>
<modules>
- <module>extras/kinesis-asl</module>
- <module>extras/kinesis-asl-assembly</module>
+ <module>external/kinesis-asl</module>
+ <module>external/kinesis-asl-assembly</module>
</modules>
</profile>
@@ -2387,7 +2387,7 @@
</build>
<modules>
- <module>extras/java8-tests</module>
+ <module>external/java8-tests</module>
</modules>
</profile>
diff --git a/python/pyspark/streaming/tests.py b/python/pyspark/streaming/tests.py
index b33e8252a7..469c068134 100644
--- a/python/pyspark/streaming/tests.py
+++ b/python/pyspark/streaming/tests.py
@@ -1619,7 +1619,7 @@ def search_mqtt_test_jar():
def search_kinesis_asl_assembly_jar():
SPARK_HOME = os.environ["SPARK_HOME"]
- kinesis_asl_assembly_dir = os.path.join(SPARK_HOME, "extras/kinesis-asl-assembly")
+ kinesis_asl_assembly_dir = os.path.join(SPARK_HOME, "external/kinesis-asl-assembly")
jars = search_jar(kinesis_asl_assembly_dir, "spark-streaming-kinesis-asl-assembly")
if not jars:
return None
--
GitLab