Skip to content
Snippets Groups Projects
Commit d33b8a2a authored by Reynold Xin's avatar Reynold Xin
Browse files

Added comments on task closure serialization.

parent 5ed38b4d
No related branches found
No related tags found
No related merge requests found
......@@ -515,6 +515,8 @@ private[spark] class ClusterTaskSetManager(sched: ClusterScheduler, val taskSet:
}
// Serialize and return the task
val startTime = System.currentTimeMillis
// We rely on the DAGScheduler to catch non-serializable closures and RDDs, so in here
// we assume the task can be serialized without exceptions.
val serializedTask = Task.serializeWithDependencies(
task, sched.sc.addedFiles, sched.sc.addedJars, ser)
val timeTaken = System.currentTimeMillis - startTime
......
......@@ -114,6 +114,8 @@ private[spark] class LocalTaskSetManager(sched: LocalScheduler, val taskSet: Tas
val info = new TaskInfo(taskId, index, System.currentTimeMillis(), "local", "local:1",
TaskLocality.NODE_LOCAL)
taskInfos(taskId) = info
// We rely on the DAGScheduler to catch non-serializable closures and RDDs, so in here
// we assume the task can be serialized without exceptions.
val bytes = Task.serializeWithDependencies(
task, sched.sc.addedFiles, sched.sc.addedJars, ser)
logInfo("Size of task " + taskId + " is " + bytes.limit + " bytes")
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment