From 23ac3aaba4a33bc3d31d01f21e93c4681ef6de03 Mon Sep 17 00:00:00 2001
From: Bruce Robbins <bersprockets@gmail.com>
Date: Wed, 28 Feb 2018 09:25:02 +0900
Subject: [PATCH] [SPARK-23417][PYTHON] Fix the build instructions supplied by
exception messages in python streaming tests
## What changes were proposed in this pull request?
Fix the build instructions supplied by exception messages in python streaming tests.
I also added -DskipTests to the maven instructions to avoid the 170 minutes of scala tests that occurs each time one wants to add a jar to the assembly directory.
## How was this patch tested?
- clone branch
- run build/sbt package
- run python/run-tests --modules "pyspark-streaming" , expect error message
- follow instructions in error message. i.e., run build/sbt assembly/package streaming-kafka-0-8-assembly/assembly
- rerun python tests, expect error message
- follow instructions in error message. i.e run build/sbt -Pflume assembly/package streaming-flume-assembly/assembly
- rerun python tests, see success.
- repeated all of the above for mvn version of the process.
Author: Bruce Robbins <bersprockets@gmail.com>
Closes #20638 from bersprockets/SPARK-23417_propa.
---
python/pyspark/streaming/tests.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/python/pyspark/streaming/tests.py b/python/pyspark/streaming/tests.py
index 5b86c1cb2c..71f8101e34 100644
--- a/python/pyspark/streaming/tests.py
+++ b/python/pyspark/streaming/tests.py
@@ -1477,8 +1477,8 @@ def search_kafka_assembly_jar():
raise Exception(
("Failed to find Spark Streaming kafka assembly jar in %s. " % kafka_assembly_dir) +
"You need to build Spark with "
- "'build/sbt assembly/package streaming-kafka-0-8-assembly/assembly' or "
- "'build/mvn -Pkafka-0-8 package' before running this test.")
+ "'build/sbt -Pkafka-0-8 assembly/package streaming-kafka-0-8-assembly/assembly' or "
+ "'build/mvn -DskipTests -Pkafka-0-8 package' before running this test.")
elif len(jars) > 1:
raise Exception(("Found multiple Spark Streaming Kafka assembly JARs: %s; please "
"remove all but one") % (", ".join(jars)))
@@ -1494,8 +1494,8 @@ def search_flume_assembly_jar():
raise Exception(
("Failed to find Spark Streaming Flume assembly jar in %s. " % flume_assembly_dir) +
"You need to build Spark with "
- "'build/sbt assembly/assembly streaming-flume-assembly/assembly' or "
- "'build/mvn -Pflume package' before running this test.")
+ "'build/sbt -Pflume assembly/package streaming-flume-assembly/assembly' or "
+ "'build/mvn -DskipTests -Pflume package' before running this test.")
elif len(jars) > 1:
raise Exception(("Found multiple Spark Streaming Flume assembly JARs: %s; please "
"remove all but one") % (", ".join(jars)))
--
GitLab