From fb499be1ac935b6f91046ec8ff23ac1267c82342 Mon Sep 17 00:00:00 2001
From: Patrick Wendell <pwendell@gmail.com>
Date: Tue, 10 Jun 2014 13:13:17 -0700
Subject: [PATCH] HOTFIX: Fix Python tests on Jenkins.
Author: Patrick Wendell <pwendell@gmail.com>
Closes #1036 from pwendell/jenkins-test and squashes the following commits:
9c99856 [Patrick Wendell] Better output during tests
71e7b74 [Patrick Wendell] Removing incorrect python path
74984db [Patrick Wendell] HOTFIX: Allow PySpark tests to run on Jenkins.
---
bin/pyspark | 6 +++++-
dev/run-tests | 3 ---
python/run-tests | 11 +++++++----
3 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/bin/pyspark b/bin/pyspark
index d0fa56f319..114cbbc3a8 100755
--- a/bin/pyspark
+++ b/bin/pyspark
@@ -86,6 +86,10 @@ else
if [[ "$IPYTHON" = "1" ]]; then
exec ipython $IPYTHON_OPTS
else
- exec "$PYSPARK_PYTHON"
+ if [[ -n $SPARK_TESTING ]]; then
+ exec "$PYSPARK_PYTHON" -m doctest
+ else
+ exec "$PYSPARK_PYTHON"
+ fi
fi
fi
diff --git a/dev/run-tests b/dev/run-tests
index 93d6692f83..c82a47ebb6 100755
--- a/dev/run-tests
+++ b/dev/run-tests
@@ -73,9 +73,6 @@ fi
echo "========================================================================="
echo "Running PySpark tests"
echo "========================================================================="
-if [ -z "$PYSPARK_PYTHON" ]; then
- export PYSPARK_PYTHON=/usr/local/bin/python2.7
-fi
./python/run-tests
echo "========================================================================="
diff --git a/python/run-tests b/python/run-tests
index 36a96121cb..3b4501178c 100755
--- a/python/run-tests
+++ b/python/run-tests
@@ -32,7 +32,8 @@ rm -f unit-tests.log
rm -rf metastore warehouse
function run_test() {
- SPARK_TESTING=0 $FWDIR/bin/pyspark $1 2>&1 | tee -a > unit-tests.log
+ echo "Running test: $1"
+ SPARK_TESTING=1 $FWDIR/bin/pyspark $1 2>&1 | tee -a > unit-tests.log
FAILED=$((PIPESTATUS[0]||$FAILED))
# Fail and exit on the first test failure.
@@ -46,15 +47,17 @@ function run_test() {
}
+echo "Running PySpark tests. Output is in python/unit-tests.log."
+
run_test "pyspark/rdd.py"
run_test "pyspark/context.py"
run_test "pyspark/conf.py"
if [ -n "$_RUN_SQL_TESTS" ]; then
run_test "pyspark/sql.py"
fi
-run_test "-m doctest pyspark/broadcast.py"
-run_test "-m doctest pyspark/accumulators.py"
-run_test "-m doctest pyspark/serializers.py"
+run_test "pyspark/broadcast.py"
+run_test "pyspark/accumulators.py"
+run_test "pyspark/serializers.py"
run_test "pyspark/tests.py"
run_test "pyspark/mllib/_common.py"
run_test "pyspark/mllib/classification.py"
--
GitLab