From 99f3f23efa193762a61736b52d89b742e077cfe9 Mon Sep 17 00:00:00 2001
From: Matei Zaharia <matei@eecs.berkeley.edu>
Date: Tue, 8 Feb 2011 17:03:03 -0800
Subject: [PATCH] Changed default shuffle to LocalFileShuffle because it's way
faster for small files
---
core/src/main/scala/spark/RDD.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core/src/main/scala/spark/RDD.scala b/core/src/main/scala/spark/RDD.scala
index bac59319a0..39f2dc4458 100644
--- a/core/src/main/scala/spark/RDD.scala
+++ b/core/src/main/scala/spark/RDD.scala
@@ -359,7 +359,7 @@ extends RDD[Pair[T, U]](sc) {
: RDD[(K, C)] =
{
val shufClass = Class.forName(System.getProperty(
- "spark.shuffle.class", "spark.DfsShuffle"))
+ "spark.shuffle.class", "spark.LocalFileShuffle"))
val shuf = shufClass.newInstance().asInstanceOf[Shuffle[K, V, C]]
shuf.compute(self, numSplits, createCombiner, mergeValue, mergeCombiners)
}
--
GitLab