From 1710966b7644641a8dbf19b66f9a3d4387eb6d3e Mon Sep 17 00:00:00 2001
From: Justin Loew <jloloew@gmail.com>
Date: Thu, 3 May 2018 22:17:01 -0500
Subject: [PATCH] Handle Option.None correctly when removing an RDD
---
.../org/apache/spark/storage/memory/MemoryStore.scala | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
index 7c2bb8d6fa..56d48432a9 100644
--- a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
+++ b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
@@ -396,8 +396,13 @@ private[spark] class MemoryStore(
def remove(blockId: BlockId): Boolean = memoryManager.synchronized {
val entry = entries.synchronized {
- val origEntry = entries.get(blockId)
- blockIdAndSizeSet -= new OurBlockIdAndSizeType(blockId, origEntry.size)
+ entries.get(blockId) match {
+ case Some(origEntry) =>
+ blockIdAndSizeSet -= new OurBlockIdAndSizeType(blockId, origEntry.size)
+ case None =>
+ // Do nothing
+ logInfo("origEntry was None after all")
+ }
entries.remove(blockId)
}
if (entry != null) {
--
GitLab