From 540a41163f28678e42084835d846f8f0c495cbbc Mon Sep 17 00:00:00 2001
From: Mosharaf Chowdhury <mosharaf@mosharaf-ubuntu.(none)>
Date: Thu, 2 Dec 2010 19:56:17 -0800
Subject: [PATCH] UseHttpPipelining is 'true' by default.
---
conf/java-opts | 2 +-
src/scala/spark/LocalFileShuffle.scala | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/conf/java-opts b/conf/java-opts
index d753f59de4..02ea427760 100644
--- a/conf/java-opts
+++ b/conf/java-opts
@@ -1 +1 @@
--Dspark.shuffle.class=spark.LocalFileShuffle -Dspark.shuffle.UseHttpPipelining=false
+-Dspark.shuffle.class=spark.LocalFileShuffle -Dspark.shuffle.UseHttpPipelining=true
diff --git a/src/scala/spark/LocalFileShuffle.scala b/src/scala/spark/LocalFileShuffle.scala
index 7b829522bd..03b6931f0a 100644
--- a/src/scala/spark/LocalFileShuffle.scala
+++ b/src/scala/spark/LocalFileShuffle.scala
@@ -62,7 +62,7 @@ class LocalFileShuffle[K, V, C] extends Shuffle[K, V, C] with Logging {
// Load config option to decide whether or not to use HTTP pipelining
val UseHttpPipelining =
- System.getProperty("spark.shuffle.UseHttpPipelining", "false").toBoolean
+ System.getProperty("spark.shuffle.UseHttpPipelining", "true").toBoolean
// Build a traversable list of pairs of server URI and split. Needs to be
// of type TraversableOnce[(String, ArrayBuffer[Int])]
@@ -94,6 +94,7 @@ class LocalFileShuffle[K, V, C] extends Shuffle[K, V, C] with Logging {
val url = "%s/shuffle/%d/%d/%d".format(serverUri, shuffleId, i, myId)
val readStartTime = System.currentTimeMillis
logInfo ("BEGIN READ: " + url)
+ // TODO: Insert data transfer code before this place
val inputStream = new ObjectInputStream(new URL(url).openStream())
try {
while (true) {
--
GitLab