Skip to content
Snippets Groups Projects
Commit 8a5eb506 authored by Andrew Ash's avatar Andrew Ash Committed by gatorsmile
Browse files

[SPARK-21941] Stop storing unused attemptId in SQLTaskMetrics

## What changes were proposed in this pull request?

In a driver heap dump containing 390,105 instances of SQLTaskMetrics this
would have saved me approximately 3.2MB of memory.

Since we're not getting any benefit from storing this unused value, let's
eliminate it until a future PR makes use of it.

## How was this patch tested?

Existing unit tests

Author: Andrew Ash <andrew@andrewash.com>

Closes #19153 from ash211/aash/trim-sql-listener.
parent 31c74fec
No related branches found
No related tags found
No related merge requests found
...@@ -255,10 +255,8 @@ class SQLListener(conf: SparkConf) extends SparkListener with Logging { ...@@ -255,10 +255,8 @@ class SQLListener(conf: SparkConf) extends SparkListener with Logging {
// heartbeat reports // heartbeat reports
} }
case None => case None =>
// TODO Now just set attemptId to 0. Should fix here when we can get the attempt
// id from SparkListenerExecutorMetricsUpdate
stageMetrics.taskIdToMetricUpdates(taskId) = new SQLTaskMetrics( stageMetrics.taskIdToMetricUpdates(taskId) = new SQLTaskMetrics(
attemptId = 0, finished = finishTask, accumulatorUpdates) finished = finishTask, accumulatorUpdates)
} }
} }
case None => case None =>
...@@ -478,10 +476,11 @@ private[ui] class SQLStageMetrics( ...@@ -478,10 +476,11 @@ private[ui] class SQLStageMetrics(
val stageAttemptId: Long, val stageAttemptId: Long,
val taskIdToMetricUpdates: mutable.HashMap[Long, SQLTaskMetrics] = mutable.HashMap.empty) val taskIdToMetricUpdates: mutable.HashMap[Long, SQLTaskMetrics] = mutable.HashMap.empty)
// TODO Should add attemptId here when we can get it from SparkListenerExecutorMetricsUpdate
/** /**
* Store all accumulatorUpdates for a Spark task. * Store all accumulatorUpdates for a Spark task.
*/ */
private[ui] class SQLTaskMetrics( private[ui] class SQLTaskMetrics(
val attemptId: Long, // TODO not used yet
var finished: Boolean, var finished: Boolean,
var accumulatorUpdates: Seq[(Long, Any)]) var accumulatorUpdates: Seq[(Long, Any)])
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment