From a63aa1adb2dfb19c8189167932ee8569840f96a0 Mon Sep 17 00:00:00 2001
From: CrazyJvm <crazyjvm@gmail.com>
Date: Sun, 15 Jun 2014 23:23:26 -0700
Subject: [PATCH] SPARK-1999: StorageLevel in storage tab and RDD Storage Info
never changes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
StorageLevel in 'storage tab' and 'RDD Storage Info' never changes even if you call rdd.unpersist() and then you give the rdd another different storage level.
Author: CrazyJvm <crazyjvm@gmail.com>
Closes #968 from CrazyJvm/ui-storagelevel and squashes the following commits:
62555fa [CrazyJvm] change RDDInfo constructor param 'storageLevel' to var, so there's need to add another variable _storageLevel。
9f1571e [CrazyJvm] JIRA https://issues.apache.org/jira/browse/SPARK-1999 UI : StorageLevel in storage tab and RDD Storage Info never changes
---
core/src/main/scala/org/apache/spark/storage/RDDInfo.scala | 6 +++---
.../main/scala/org/apache/spark/storage/StorageUtils.scala | 3 +++
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
index 023fd6e4d8..5a72e21687 100644
--- a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
+++ b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
@@ -26,7 +26,7 @@ class RDDInfo(
val id: Int,
val name: String,
val numPartitions: Int,
- val storageLevel: StorageLevel)
+ var storageLevel: StorageLevel)
extends Ordered[RDDInfo] {
var numCachedPartitions = 0
@@ -36,8 +36,8 @@ class RDDInfo(
override def toString = {
import Utils.bytesToString
- ("RDD \"%s\" (%d) Storage: %s; CachedPartitions: %d; TotalPartitions: %d; MemorySize: %s; " +
- "TachyonSize: %s; DiskSize: %s").format(
+ ("RDD \"%s\" (%d) StorageLevel: %s; CachedPartitions: %d; TotalPartitions: %d; " +
+ "MemorySize: %s; TachyonSize: %s; DiskSize: %s").format(
name, id, storageLevel.toString, numCachedPartitions, numPartitions,
bytesToString(memSize), bytesToString(tachyonSize), bytesToString(diskSize))
}
diff --git a/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala b/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala
index 6f3252a2f6..f3bde1df45 100644
--- a/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala
+++ b/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala
@@ -89,10 +89,13 @@ private[spark] object StorageUtils {
// Add up memory, disk and Tachyon sizes
val persistedBlocks =
blocks.filter { status => status.memSize + status.diskSize + status.tachyonSize > 0 }
+ val _storageLevel =
+ if (persistedBlocks.length > 0) persistedBlocks(0).storageLevel else StorageLevel.NONE
val memSize = persistedBlocks.map(_.memSize).reduceOption(_ + _).getOrElse(0L)
val diskSize = persistedBlocks.map(_.diskSize).reduceOption(_ + _).getOrElse(0L)
val tachyonSize = persistedBlocks.map(_.tachyonSize).reduceOption(_ + _).getOrElse(0L)
rddInfoMap.get(rddId).map { rddInfo =>
+ rddInfo.storageLevel = _storageLevel
rddInfo.numCachedPartitions = persistedBlocks.length
rddInfo.memSize = memSize
rddInfo.diskSize = diskSize
--
GitLab