diff --git a/bin/pyspark b/bin/pyspark index d0fa56f31913ff9f1127a790236ae0c18119e6f8..114cbbc3a8a8e2d5f0bf730afa3f1f134b45c611 100755 --- a/bin/pyspark +++ b/bin/pyspark @@ -86,6 +86,10 @@ else if [[ "$IPYTHON" = "1" ]]; then exec ipython $IPYTHON_OPTS else - exec "$PYSPARK_PYTHON" + if [[ -n $SPARK_TESTING ]]; then + exec "$PYSPARK_PYTHON" -m doctest + else + exec "$PYSPARK_PYTHON" + fi fi fi diff --git a/dev/run-tests b/dev/run-tests index 93d6692f83ca85418ecd745842d1635f19b6e977..c82a47ebb618bc1102eb397aa7515d651d84ea09 100755 --- a/dev/run-tests +++ b/dev/run-tests @@ -73,9 +73,6 @@ fi echo "=========================================================================" echo "Running PySpark tests" echo "=========================================================================" -if [ -z "$PYSPARK_PYTHON" ]; then - export PYSPARK_PYTHON=/usr/local/bin/python2.7 -fi ./python/run-tests echo "=========================================================================" diff --git a/python/run-tests b/python/run-tests index 36a96121cbc0d543fe2c72744522acddec5fb76c..3b4501178c89f314dfb7389648ac26084df8f437 100755 --- a/python/run-tests +++ b/python/run-tests @@ -32,7 +32,8 @@ rm -f unit-tests.log rm -rf metastore warehouse function run_test() { - SPARK_TESTING=0 $FWDIR/bin/pyspark $1 2>&1 | tee -a > unit-tests.log + echo "Running test: $1" + SPARK_TESTING=1 $FWDIR/bin/pyspark $1 2>&1 | tee -a > unit-tests.log FAILED=$((PIPESTATUS[0]||$FAILED)) # Fail and exit on the first test failure. @@ -46,15 +47,17 @@ function run_test() { } +echo "Running PySpark tests. Output is in python/unit-tests.log." + run_test "pyspark/rdd.py" run_test "pyspark/context.py" run_test "pyspark/conf.py" if [ -n "$_RUN_SQL_TESTS" ]; then run_test "pyspark/sql.py" fi -run_test "-m doctest pyspark/broadcast.py" -run_test "-m doctest pyspark/accumulators.py" -run_test "-m doctest pyspark/serializers.py" +run_test "pyspark/broadcast.py" +run_test "pyspark/accumulators.py" +run_test "pyspark/serializers.py" run_test "pyspark/tests.py" run_test "pyspark/mllib/_common.py" run_test "pyspark/mllib/classification.py"