Skip to content
Snippets Groups Projects
Commit 6c6e47e3 authored by Ankur Dave's avatar Ankur Dave
Browse files

Use BufferedOutputStream in ShuffleMapTask

parent a161f006
No related branches found
No related tags found
No related merge requests found
package spark package spark
import java.io.BufferedOutputStream
import java.io.FileOutputStream import java.io.FileOutputStream
import java.io.ObjectOutputStream import java.io.ObjectOutputStream
import scala.collection.mutable.HashMap import scala.collection.mutable.HashMap
...@@ -26,7 +27,7 @@ extends DAGTask[String](stageId) with Logging { ...@@ -26,7 +27,7 @@ extends DAGTask[String](stageId) with Logging {
val ser = SparkEnv.get.serializer.newInstance() val ser = SparkEnv.get.serializer.newInstance()
for (i <- 0 until numOutputSplits) { for (i <- 0 until numOutputSplits) {
val file = LocalFileShuffle.getOutputFile(dep.shuffleId, partition, i) val file = LocalFileShuffle.getOutputFile(dep.shuffleId, partition, i)
val out = ser.outputStream(new FileOutputStream(file)) val out = ser.outputStream(new BufferedOutputStream(new FileOutputStream(file)))
buckets(i).foreach(pair => out.writeObject(pair)) buckets(i).foreach(pair => out.writeObject(pair))
// TODO: have some kind of EOF marker // TODO: have some kind of EOF marker
out.close() out.close()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment