diff --git a/src/scala/spark/BasicLocalFileShuffle.scala b/src/scala/spark/BasicLocalFileShuffle.scala index aa83e5cf8ca1400ede880618fa3cb0dd04fb72fe..57e8abf9641ca7da254720e60415cd50e336016e 100644 --- a/src/scala/spark/BasicLocalFileShuffle.scala +++ b/src/scala/spark/BasicLocalFileShuffle.scala @@ -9,7 +9,7 @@ import scala.collection.mutable.{ArrayBuffer, HashMap} /** - * A simple implementation of shuffle using local files served through HTTP. + * A basic implementation of shuffle using local files served through HTTP. * * TODO: Add support for compression when spark.compress is set to true. */ diff --git a/src/scala/spark/ParallelLocalFileShuffle.scala b/src/scala/spark/ParallelLocalFileShuffle.scala index af461e48c0604cf2345d4dd398356882273bae50..7c957aba42ad2985510b1425562ff0ba8e38a492 100644 --- a/src/scala/spark/ParallelLocalFileShuffle.scala +++ b/src/scala/spark/ParallelLocalFileShuffle.scala @@ -10,7 +10,9 @@ import scala.collection.mutable.{ArrayBuffer, HashMap} /** - * A simple implementation of shuffle using local files served through HTTP. + * An implementation of shuffle using local files served through HTTP where + * receivers create simultaneous connections to multiple servers by setting the + * 'spark.parallelLocalFileShuffle.maxConnections' config option. * * TODO: Add support for compression when spark.compress is set to true. */ @@ -221,12 +223,12 @@ object ParallelLocalFileShuffle extends Logging { if (!initialized) { // Load config parameters MinKnockInterval_ = System.getProperty ( - "spark.parallelLocalFileShuffle.minKnockInterval", "1000").toInt + "spark.parallelLocalFileShuffle.minKnockInterval", "1000").toInt MaxKnockInterval_ = System.getProperty ( "spark.parallelLocalFileShuffle.maxKnockInterval", "5000").toInt MaxConnections_ = System.getProperty ( - "spark.parallelLocalFileShuffle.maxConnections", "4").toInt + "spark.parallelLocalFileShuffle.maxConnections", "4").toInt // TODO: localDir should be created by some mechanism common to Spark // so that it can be shared among shuffle, broadcast, etc