From 5d30eae56051c563a8427f330b09ef66db0a0d21 Mon Sep 17 00:00:00 2001
From: Sean Owen <sowen@cloudera.com>
Date: Mon, 29 Jun 2015 17:21:35 -0700
Subject: [PATCH] [SPARK-8437] [DOCS] Using directory path without wildcard for
filename slow for large number of files with wholeTextFiles and binaryFiles
Note that 'dir/*' can be more efficient in some Hadoop FS implementations that 'dir/'
Author: Sean Owen <sowen@cloudera.com>
Closes #7036 from srowen/SPARK-8437 and squashes the following commits:
0e813ae [Sean Owen] Note that 'dir/*' can be more efficient in some Hadoop FS implementations that 'dir/'
---
core/src/main/scala/org/apache/spark/SparkContext.scala | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index b3c3bf3746..cb7e24c374 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -831,6 +831,8 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* }}}
*
* @note Small files are preferred, large file is also allowable, but may cause bad performance.
+ * @note On some filesystems, `.../path/*` can be a more efficient way to read all files in a directory
+ * rather than `.../path/` or `.../path`
*
* @param minPartitions A suggestion value of the minimal splitting number for input data.
*/
@@ -878,9 +880,11 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* (a-hdfs-path/part-nnnnn, its content)
* }}}
*
- * @param minPartitions A suggestion value of the minimal splitting number for input data.
- *
* @note Small files are preferred; very large files may cause bad performance.
+ * @note On some filesystems, `.../path/*` can be a more efficient way to read all files in a directory
+ * rather than `.../path/` or `.../path`
+ *
+ * @param minPartitions A suggestion value of the minimal splitting number for input data.
*/
@Experimental
def binaryFiles(
--
GitLab