From 6343f66557434ce889a25a7889d76d0d24188ced Mon Sep 17 00:00:00 2001 From: MechCoder <mks542@nyu.edu> Date: Thu, 7 Jul 2016 11:31:10 +0100 Subject: [PATCH] [SPARK-16399][PYSPARK] Force PYSPARK_PYTHON to python ## What changes were proposed in this pull request? I would like to change ```bash if hash python2.7 2>/dev/null; then # Attempt to use Python 2.7, if installed: DEFAULT_PYTHON="python2.7" else DEFAULT_PYTHON="python" fi ``` to just ```DEFAULT_PYTHON="python"``` I'm not sure if it is a great assumption that python2.7 is used by default, when python points to something else. ## How was this patch tested? (Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) (If this patch involves UI changes, please attach a screenshot; otherwise, remove this) Author: MechCoder <mks542@nyu.edu> Closes #14016 from MechCoder/followup. --- bin/pyspark | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/bin/pyspark b/bin/pyspark index ac8aa04dba..a0d7e22e8a 100755 --- a/bin/pyspark +++ b/bin/pyspark @@ -30,14 +30,6 @@ export _SPARK_CMD_USAGE="Usage: ./bin/pyspark [options]" # (e.g. PYSPARK_DRIVER_PYTHON_OPTS='notebook'). This supports full customization of the IPython # and executor Python executables. -# Determine the Python executable to use if PYSPARK_PYTHON or PYSPARK_DRIVER_PYTHON isn't set: -if hash python2.7 2>/dev/null; then - # Attempt to use Python 2.7, if installed: - DEFAULT_PYTHON="python2.7" -else - DEFAULT_PYTHON="python" -fi - # Fail noisily if removed options are set if [[ -n "$IPYTHON" || -n "$IPYTHON_OPTS" ]]; then echo "Error in pyspark startup:" @@ -47,10 +39,10 @@ fi # Default to standard python interpreter unless told otherwise if [[ -z "$PYSPARK_DRIVER_PYTHON" ]]; then - PYSPARK_DRIVER_PYTHON="${PYSPARK_PYTHON:-"$DEFAULT_PYTHON"}" + PYSPARK_DRIVER_PYTHON="${PYSPARK_PYTHON:-"python"}" fi -WORKS_WITH_IPYTHON=$($DEFAULT_PYTHON -c 'import sys; print(sys.version_info >= (2, 7, 0))') +WORKS_WITH_IPYTHON=$(python -c 'import sys; print(sys.version_info >= (2, 7, 0))') # Determine the Python executable to use for the executors: if [[ -z "$PYSPARK_PYTHON" ]]; then @@ -58,7 +50,7 @@ if [[ -z "$PYSPARK_PYTHON" ]]; then echo "IPython requires Python 2.7+; please install python2.7 or set PYSPARK_PYTHON" 1>&2 exit 1 else - PYSPARK_PYTHON="$DEFAULT_PYTHON" + PYSPARK_PYTHON=python fi fi export PYSPARK_PYTHON -- GitLab