Skip to content
Snippets Groups Projects
Commit 6f2db8c2 authored by DB Tsai's avatar DB Tsai Committed by Xiangrui Meng
Browse files

Make sure that empty string is filtered out when we get the secondary jars from conf

Author: DB Tsai <dbtsai@dbtsai.com>

Closes #1027 from dbtsai/dbtsai-classloader and squashes the following commits:

9ac6be3 [DB Tsai] Fixed line too long
c9c7ad7 [DB Tsai] Make sure that empty string is filtered out when we get the secondary jars from conf.
parent a9ec033c
No related branches found
No related tags found
No related merge requests found
......@@ -221,7 +221,7 @@ trait ClientBase extends Logging {
}
}
var cachedSecondaryJarLinks = ListBuffer.empty[String]
val cachedSecondaryJarLinks = ListBuffer.empty[String]
val fileLists = List( (args.addJars, LocalResourceType.FILE, true),
(args.files, LocalResourceType.FILE, false),
(args.archives, LocalResourceType.ARCHIVE, false) )
......@@ -502,12 +502,14 @@ object ClientBase extends Logging {
def addClasspathEntry(path: String) = YarnSparkHadoopUtil.addToEnvironment(env,
Environment.CLASSPATH.name, path, File.pathSeparator)
/** Add entry to the classpath. Interpreted as a path relative to the working directory. */
def addPwdClasspathEntry(entry: String) = addClasspathEntry(Environment.PWD.$() + Path.SEPARATOR + entry)
def addPwdClasspathEntry(entry: String) =
addClasspathEntry(Environment.PWD.$() + Path.SEPARATOR + entry)
extraClassPath.foreach(addClasspathEntry)
val cachedSecondaryJarLinks =
sparkConf.getOption(CONF_SPARK_YARN_SECONDARY_JARS).getOrElse("").split(",")
.filter(_.nonEmpty)
// Normally the users app.jar is last in case conflicts with spark jars
if (sparkConf.get("spark.yarn.user.classpath.first", "false").toBoolean) {
addPwdClasspathEntry(APP_JAR)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment