Skip to content
Snippets Groups Projects
Commit 44a24400 authored by BlackNiuza's avatar BlackNiuza
Browse files

Remove active job from idToActiveJob when job finished or aborted

parent e063e29a
No related branches found
No related tags found
No related merge requests found
...@@ -524,6 +524,7 @@ class DAGScheduler( ...@@ -524,6 +524,7 @@ class DAGScheduler(
job.numFinished += 1 job.numFinished += 1
// If the whole job has finished, remove it // If the whole job has finished, remove it
if (job.numFinished == job.numPartitions) { if (job.numFinished == job.numPartitions) {
idToActiveJob -= stage.priority
activeJobs -= job activeJobs -= job
resultStageToJob -= stage resultStageToJob -= stage
markStageAsFinished(stage) markStageAsFinished(stage)
...@@ -671,6 +672,7 @@ class DAGScheduler( ...@@ -671,6 +672,7 @@ class DAGScheduler(
val error = new SparkException("Job failed: " + reason) val error = new SparkException("Job failed: " + reason)
job.listener.jobFailed(error) job.listener.jobFailed(error)
sparkListeners.foreach(_.onJobEnd(SparkListenerJobEnd(job, JobFailed(error)))) sparkListeners.foreach(_.onJobEnd(SparkListenerJobEnd(job, JobFailed(error))))
idToActiveJob -= resultStage.priority
activeJobs -= job activeJobs -= job
resultStageToJob -= resultStage resultStageToJob -= resultStage
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment