From 4cca3917dc30ee907e6cbd6a569b6ac58af963f7 Mon Sep 17 00:00:00 2001
From: nemccarthy <nathan@nemccarthy.me>
Date: Tue, 17 Mar 2015 09:33:11 -0700
Subject: [PATCH] [SPARK-6313] Add config option to disable file
locks/fetchFile cache to ...
...support NFS mounts.
This is a work around for now with the goal to find a more permanent solution.
https://issues.apache.org/jira/browse/SPARK-6313
Author: nemccarthy <nathan@nemccarthy.me>
Closes #5036 from nemccarthy/master and squashes the following commits:
2eaaf42 [nemccarthy] [SPARK-6313] Update config wording doc for spark.files.useFetchCache
5de7eb4 [nemccarthy] [SPARK-6313] Add config option to disable file locks/fetchFile cache to support NFS mounts
---
.../src/main/scala/org/apache/spark/util/Utils.scala | 3 ++-
docs/configuration.md | 12 ++++++++++++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index d3dc1d09cb..af8a24553a 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -403,7 +403,8 @@ private[spark] object Utils extends Logging {
useCache: Boolean) {
val fileName = url.split("/").last
val targetFile = new File(targetDir, fileName)
- if (useCache) {
+ val fetchCacheEnabled = conf.getBoolean("spark.files.useFetchCache", defaultValue = true)
+ if (useCache && fetchCacheEnabled) {
val cachedFileName = s"${url.hashCode}${timestamp}_cache"
val lockFileName = s"${url.hashCode}${timestamp}_lock"
val localDir = new File(getLocalDir(conf))
diff --git a/docs/configuration.md b/docs/configuration.md
index 63fc99e7d3..7fe1147521 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -745,6 +745,18 @@ Apart from these, the following properties are also available, and may be useful
the driver, in seconds.
</td>
</tr>
+<tr>
+ <td><code>spark.files.useFetchCache</code></td>
+ <td>true</td>
+ <td>
+ If set to true (default), file fetching will use a local cache that is shared by executors
+ that belong to the same application, which can improve task launching performance when
+ running many executors on the same host. If set to false, these caching optimizations will
+ be disabled and all executors will fetch their own copies of files. This optimization may be
+ disabled in order to use Spark local directories that reside on NFS filesystems (see
+ <a href="https://issues.apache.org/jira/browse/SPARK-6313">SPARK-6313</a> for more details).
+ </td>
+</tr>
<tr>
<td><code>spark.files.overwrite</code></td>
<td>false</td>
--
GitLab