Skip to content
Snippets Groups Projects
Commit 149472a0 authored by Marcelo Vanzin's avatar Marcelo Vanzin
Browse files

[SPARK-11023] [YARN] Avoid creating URIs from local paths directly.

The issue is that local paths on Windows, when provided with drive
letters or backslashes, are not valid URIs.

Instead of trying to figure out whether paths are URIs or not, use
Utils.resolveURI() which does that for us.

Author: Marcelo Vanzin <vanzin@cloudera.com>

Closes #9049 from vanzin/SPARK-11023 and squashes the following commits:

77021f2 [Marcelo Vanzin] [SPARK-11023] [yarn] Avoid creating URIs from local paths directly.
parent 64b1d00e
No related branches found
No related tags found
No related merge requests found
......@@ -358,7 +358,8 @@ private[spark] class Client(
destName: Option[String] = None,
targetDir: Option[String] = None,
appMasterOnly: Boolean = false): (Boolean, String) = {
val localURI = new URI(path.trim())
val trimmedPath = path.trim()
val localURI = Utils.resolveURI(trimmedPath)
if (localURI.getScheme != LOCAL_SCHEME) {
if (addDistributedUri(localURI)) {
val localPath = getQualifiedLocalPath(localURI, hadoopConf)
......@@ -374,7 +375,7 @@ private[spark] class Client(
(false, null)
}
} else {
(true, path.trim())
(true, trimmedPath)
}
}
......@@ -595,10 +596,10 @@ private[spark] class Client(
LOCALIZED_PYTHON_DIR)
}
(pySparkArchives ++ pyArchives).foreach { path =>
val uri = new URI(path)
val uri = Utils.resolveURI(path)
if (uri.getScheme != LOCAL_SCHEME) {
pythonPath += buildPath(YarnSparkHadoopUtil.expandEnvironment(Environment.PWD),
new Path(path).getName())
new Path(uri).getName())
} else {
pythonPath += uri.getPath()
}
......@@ -1229,7 +1230,7 @@ object Client extends Logging {
private def getMainJarUri(mainJar: Option[String]): Option[URI] = {
mainJar.flatMap { path =>
val uri = new URI(path)
val uri = Utils.resolveURI(path)
if (uri.getScheme == LOCAL_SCHEME) Some(uri) else None
}.orElse(Some(new URI(APP_JAR)))
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment