From c9b89a0a0921ce3d52864afd4feb7f37b90f7b46 Mon Sep 17 00:00:00 2001
From: Iulian Dragos <jaguarul@gmail.com>
Date: Mon, 1 Feb 2016 13:38:38 -0800
Subject: [PATCH] =?UTF-8?q?[SPARK-12979][MESOS]=20Don=E2=80=99t=20resolve?=
=?UTF-8?q?=20paths=20on=20the=20local=20file=20system=20in=20Mesos=20sche?=
=?UTF-8?q?duler?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The driver filesystem is likely different from where the executors will run, so resolving paths (and symlinks, etc.) will lead to invalid paths on executors.
Author: Iulian Dragos <jaguarul@gmail.com>
Closes #10923 from dragos/issue/canonical-paths.
---
.../scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala | 2 +-
.../spark/scheduler/cluster/mesos/MesosClusterScheduler.scala | 2 +-
.../spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
index 58c30e7d97..2f095b86c6 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
@@ -179,7 +179,7 @@ private[spark] class CoarseMesosSchedulerBackend(
.orElse(Option(System.getenv("SPARK_EXECUTOR_URI")))
if (uri.isEmpty) {
- val runScript = new File(executorSparkHome, "./bin/spark-class").getCanonicalPath
+ val runScript = new File(executorSparkHome, "./bin/spark-class").getPath
command.setValue(
"%s \"%s\" org.apache.spark.executor.CoarseGrainedExecutorBackend"
.format(prefixEnv, runScript) +
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
index e77d77208c..8cda4ff0eb 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
@@ -394,7 +394,7 @@ private[spark] class MesosClusterScheduler(
.getOrElse {
throw new SparkException("Executor Spark home `spark.mesos.executor.home` is not set!")
}
- val cmdExecutable = new File(executorSparkHome, "./bin/spark-submit").getCanonicalPath
+ val cmdExecutable = new File(executorSparkHome, "./bin/spark-submit").getPath
// Sandbox points to the current directory by default with Mesos.
(cmdExecutable, ".")
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
index a8bf79a78c..340f29bac9 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
@@ -125,7 +125,7 @@ private[spark] class MesosSchedulerBackend(
val executorBackendName = classOf[MesosExecutorBackend].getName
if (uri.isEmpty) {
- val executorPath = new File(executorSparkHome, "/bin/spark-class").getCanonicalPath
+ val executorPath = new File(executorSparkHome, "/bin/spark-class").getPath
command.setValue(s"$prefixEnv $executorPath $executorBackendName")
} else {
// Grab everything to the first '.'. We'll use that and '*' to
--
GitLab