From 640afa49aa349c7ebe35d365eec3ef9bb7710b1d Mon Sep 17 00:00:00 2001
From: Li Yichao <lyc@zhihu.com>
Date: Thu, 1 Jun 2017 14:39:57 -0700
Subject: [PATCH] [SPARK-20365][YARN] Remove local scheme when add path to
ClassPath.
In Spark on YARN, when configuring "spark.yarn.jars" with local jars (jars started with "local" scheme), we will get inaccurate classpath for AM and containers. This is because we don't remove "local" scheme when concatenating classpath. It is OK to run because classpath is separated with ":" and java treat "local" as a separate jar. But we could improve it to remove the scheme.
Updated `ClientSuite` to check "local" is not in the classpath.
cc jerryshao
Author: Li Yichao <lyc@zhihu.com>
Author: Li Yichao <liyichao.good@gmail.com>
Closes #18129 from liyichao/SPARK-20365.
---
.../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 3 ++-
.../test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala | 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 9956071fd6..1fb7edf2a6 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -1275,7 +1275,8 @@ private object Client extends Logging {
if (sparkConf.get(SPARK_ARCHIVE).isEmpty) {
sparkConf.get(SPARK_JARS).foreach { jars =>
jars.filter(isLocalUri).foreach { jar =>
- addClasspathEntry(getClusterPath(sparkConf, jar), env)
+ val uri = new URI(jar)
+ addClasspathEntry(getClusterPath(sparkConf, uri.getPath()), env)
}
}
}
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
index 3a11787aa5..6cf6842792 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
@@ -122,6 +122,7 @@ class ClientSuite extends SparkFunSuite with Matchers with BeforeAndAfterAll
cp should not contain (uri.getPath())
}
})
+ cp should not contain ("local")
cp should contain(PWD)
cp should contain (s"$PWD${Path.SEPARATOR}${LOCALIZED_CONF_DIR}")
cp should not contain (APP_JAR)
--
GitLab