From f3d65cd0bf34d8c362da492472c61796ef059428 Mon Sep 17 00:00:00 2001
From: Daoyuan Wang <daoyuan.wang@intel.com>
Date: Sat, 23 Aug 2014 08:09:30 -0700
Subject: [PATCH] [SPARK-3068]remove MaxPermSize option for jvm 1.8
In JVM 1.8.0, MaxPermSize is no longer supported.
In spark `stderr` output, there would be a line of
Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=128m; support was removed in 8.0
Author: Daoyuan Wang <daoyuan.wang@intel.com>
Closes #2011 from adrian-wang/maxpermsize and squashes the following commits:
ef1d660 [Daoyuan Wang] direct get java version in runtime
37db9c1 [Daoyuan Wang] code refine
3c1d554 [Daoyuan Wang] remove MaxPermSize option for jvm 1.8
---
bin/spark-class | 7 ++++++-
bin/spark-class2.cmd | 8 +++++++-
.../org/apache/spark/deploy/worker/CommandUtils.scala | 4 ++--
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/bin/spark-class b/bin/spark-class
index 22acf92288..c6543545a5 100755
--- a/bin/spark-class
+++ b/bin/spark-class
@@ -105,9 +105,14 @@ else
exit 1
fi
fi
+JAVA_VERSION=$($RUNNER -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q')
# Set JAVA_OPTS to be able to load native libraries and to set heap size
-JAVA_OPTS="-XX:MaxPermSize=128m $OUR_JAVA_OPTS"
+if [ "$JAVA_VERSION" -ge 18 ]; then
+ JAVA_OPTS="$OUR_JAVA_OPTS"
+else
+ JAVA_OPTS="-XX:MaxPermSize=128m $OUR_JAVA_OPTS"
+fi
JAVA_OPTS="$JAVA_OPTS -Xms$OUR_JAVA_MEM -Xmx$OUR_JAVA_MEM"
# Load extra JAVA_OPTS from conf/java-opts, if it exists
diff --git a/bin/spark-class2.cmd b/bin/spark-class2.cmd
index e420eb409e..e2c5f9c385 100755
--- a/bin/spark-class2.cmd
+++ b/bin/spark-class2.cmd
@@ -77,7 +77,13 @@ rem All drivers use SPARK_JAVA_OPTS + SPARK_DRIVER_MEMORY. The repl also uses SP
)
rem Set JAVA_OPTS to be able to load native libraries and to set heap size
-set JAVA_OPTS=-XX:MaxPermSize=128m %OUR_JAVA_OPTS% -Djava.library.path=%SPARK_LIBRARY_PATH% -Xms%OUR_JAVA_MEM% -Xmx%OUR_JAVA_MEM%
+for /f "tokens=3" %%i in ('java -version 2^>^&1 ^| find "version"') do set jversion=%%i
+for /f "tokens=1 delims=_" %%i in ("%jversion:~1,-1%") do set jversion=%%i
+if "%jversion%" geq "1.8.0" (
+ set JAVA_OPTS=%OUR_JAVA_OPTS% -Djava.library.path=%SPARK_LIBRARY_PATH% -Xms%OUR_JAVA_MEM% -Xmx%OUR_JAVA_MEM%
+) else (
+ set JAVA_OPTS=-XX:MaxPermSize=128m %OUR_JAVA_OPTS% -Djava.library.path=%SPARK_LIBRARY_PATH% -Xms%OUR_JAVA_MEM% -Xmx%OUR_JAVA_MEM%
+)
rem Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in CommandUtils.scala!
rem Test whether the user has built Spark
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
index 687e492a0d..12e98fd40d 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
@@ -64,8 +64,6 @@ object CommandUtils extends Logging {
Seq()
}
- val permGenOpt = Seq("-XX:MaxPermSize=128m")
-
// Figure out our classpath with the external compute-classpath script
val ext = if (System.getProperty("os.name").startsWith("Windows")) ".cmd" else ".sh"
val classPath = Utils.executeAndGetOutput(
@@ -73,6 +71,8 @@ object CommandUtils extends Logging {
extraEnvironment = command.environment)
val userClassPath = command.classPathEntries ++ Seq(classPath)
+ val javaVersion = System.getProperty("java.version")
+ val permGenOpt = if (!javaVersion.startsWith("1.8")) Some("-XX:MaxPermSize=128m") else None
Seq("-cp", userClassPath.filterNot(_.isEmpty).mkString(File.pathSeparator)) ++
permGenOpt ++ libraryOpts ++ workerLocalOpts ++ command.javaOpts ++ memoryOpts
}
--
GitLab