Skip to content
Snippets Groups Projects
Commit a613628c authored by Liang-Chi Hsieh's avatar Liang-Chi Hsieh
Browse files

Do not copy local jars given to SparkContext in yarn mode since the Context is...

Do not copy local jars given to SparkContext in yarn mode since the Context is not running on local. This bug causes failure when jars can not be found. Example codes (such as spark.examples.SparkPi) can not work without this fix under yarn mode.
parent 0d0a47cb
No related branches found
No related tags found
No related merge requests found
...@@ -102,6 +102,7 @@ class SparkContext( ...@@ -102,6 +102,7 @@ class SparkContext(
} }
private val isLocal = (master == "local" || master.startsWith("local[")) private val isLocal = (master == "local" || master.startsWith("local["))
private val isYarn = (master == "yarn-standalone")
// Create the Spark execution environment (cache, map output tracker, etc) // Create the Spark execution environment (cache, map output tracker, etc)
private[spark] val env = SparkEnv.createFromSystemProperties( private[spark] val env = SparkEnv.createFromSystemProperties(
...@@ -577,11 +578,18 @@ class SparkContext( ...@@ -577,11 +578,18 @@ class SparkContext(
} else { } else {
val uri = new URI(path) val uri = new URI(path)
val key = uri.getScheme match { val key = uri.getScheme match {
case null | "file" => env.httpFileServer.addJar(new File(uri.getPath)) case null | "file" =>
if (!isYarn)
env.httpFileServer.addJar(new File(uri.getPath))
else
null
case _ => path case _ => path
} }
addedJars(key) = System.currentTimeMillis
logInfo("Added JAR " + path + " at " + key + " with timestamp " + addedJars(key)) if (key != null) {
addedJars(key) = System.currentTimeMillis
logInfo("Added JAR " + path + " at " + key + " with timestamp " + addedJars(key))
}
} }
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment