Skip to content
Snippets Groups Projects
Commit f39f2b76 authored by Christoph Grothaus's avatar Christoph Grothaus
Browse files

Incorporate feedback from mateiz:

- we do not need getEnvOrEmpty
- Instead of saving SPARK_NONDAEMON_JAVA_OPTS, it would be better to modify the scripts to use a different variable name for the JAVA_OPTS they do eventually use
parent 85a35c68
No related branches found
No related tags found
No related merge requests found
......@@ -75,10 +75,10 @@ private[spark] class ExecutorRunner(
def buildCommandSeq(): Seq[String] = {
val command = appDesc.command
val runner = if (getEnvOrEmpty("JAVA_HOME") == "") {
val runner = if (System.getenv("JAVA_HOME") == null) {
"java"
} else {
getEnvOrEmpty("JAVA_HOME") + "/bin/java"
System.getenv("JAVA_HOME") + "/bin/java"
}
// SPARK-698: do not call the run.cmd script, as process.destroy()
// fails to kill a process tree on Windows
......@@ -91,29 +91,19 @@ private[spark] class ExecutorRunner(
* way the JAVA_OPTS are assembled there.
*/
def buildJavaOpts(): Seq[String] = {
val _javaLibPath = if (getEnvOrEmpty("SPARK_LIBRARY_PATH") == "") {
val _javaLibPath = if (System.getenv("SPARK_LIBRARY_PATH") == null) {
""
} else {
"-Djava.library.path=" + getEnvOrEmpty("SPARK_LIBRARY_PATH")
"-Djava.library.path=" + System.getenv("SPARK_LIBRARY_PATH")
}
Seq("-cp",
getEnvOrEmpty("CLASSPATH"),
// SPARK_JAVA_OPTS is overwritten with SPARK_DAEMON_JAVA_OPTS for running the worker
getEnvOrEmpty("SPARK_NONDAEMON_JAVA_OPTS"),
System.getenv("CLASSPATH"),
System.getenv("SPARK_JAVA_OPTS"),
_javaLibPath,
"-Xms" + memory.toString + "M",
"-Xmx" + memory.toString + "M")
.filter(_ != "")
}
def getEnvOrEmpty(key: String): String = {
val result = System.getenv(key)
if (result == null) {
""
} else {
result
}
.filter(_ != null)
}
/** Spawn a thread that will redirect a given stream to a file */
......
......@@ -22,9 +22,10 @@ fi
# values for that; it doesn't need a lot
if [ "$1" = "spark.deploy.master.Master" -o "$1" = "spark.deploy.worker.Worker" ]; then
SPARK_MEM=${SPARK_DAEMON_MEMORY:-512m}
# Backup current SPARK_JAVA_OPTS for use in ExecutorRunner.scala
SPARK_NONDAEMON_JAVA_OPTS=$SPARK_JAVA_OPTS
SPARK_JAVA_OPTS=$SPARK_DAEMON_JAVA_OPTS # Empty by default
# Do not overwrite SPARK_JAVA_OPTS environment variable in this script
OUR_JAVA_OPTS=$SPARK_DAEMON_JAVA_OPTS # Empty by default
else
OUR_JAVA_OPTS=$SPARK_JAVA_OPTS
fi
if [ "$SPARK_LAUNCH_WITH_SCALA" == "1" ]; then
......@@ -64,7 +65,7 @@ fi
export SPARK_MEM
# Set JAVA_OPTS to be able to load native libraries and to set heap size
JAVA_OPTS="$SPARK_JAVA_OPTS"
JAVA_OPTS="$OUR_JAVA_OPTS"
JAVA_OPTS+=" -Djava.library.path=$SPARK_LIBRARY_PATH"
JAVA_OPTS+=" -Xms$SPARK_MEM -Xmx$SPARK_MEM"
# Load extra JAVA_OPTS from conf/java-opts, if it exists
......
......@@ -22,9 +22,9 @@ if "%1"=="spark.deploy.master.Master" set RUNNING_DAEMON=1
if "%1"=="spark.deploy.worker.Worker" set RUNNING_DAEMON=1
if "x%SPARK_DAEMON_MEMORY%" == "x" set SPARK_DAEMON_MEMORY=512m
if "%RUNNING_DAEMON%"=="1" set SPARK_MEM=%SPARK_DAEMON_MEMORY%
rem Backup current SPARK_JAVA_OPTS for use in ExecutorRunner.scala
if "%RUNNING_DAEMON%"=="1" set SPARK_NONDAEMON_JAVA_OPTS=%SPARK_JAVA_OPTS%
if "%RUNNING_DAEMON%"=="1" set SPARK_JAVA_OPTS=%SPARK_DAEMON_JAVA_OPTS%
rem Do not overwrite SPARK_JAVA_OPTS environment variable in this script
if "%RUNNING_DAEMON%"=="0" set OUR_JAVA_OPTS=%SPARK_JAVA_OPTS%
if "%RUNNING_DAEMON%"=="1" set OUR_JAVA_OPTS=%SPARK_DAEMON_JAVA_OPTS%
rem Check that SCALA_HOME has been specified
if not "x%SCALA_HOME%"=="x" goto scala_exists
......@@ -41,7 +41,7 @@ rem variable so that our process sees it and can report it to Mesos
if "x%SPARK_MEM%"=="x" set SPARK_MEM=512m
rem Set JAVA_OPTS to be able to load native libraries and to set heap size
set JAVA_OPTS=%SPARK_JAVA_OPTS% -Djava.library.path=%SPARK_LIBRARY_PATH% -Xms%SPARK_MEM% -Xmx%SPARK_MEM%
set JAVA_OPTS=%OUR_JAVA_OPTS% -Djava.library.path=%SPARK_LIBRARY_PATH% -Xms%SPARK_MEM% -Xmx%SPARK_MEM%
rem Load extra JAVA_OPTS from conf/java-opts, if it exists
if exist "%FWDIR%conf\java-opts.cmd" call "%FWDIR%conf\java-opts.cmd"
rem Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in ExecutorRunner.scala!
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment