From bb322943623d14b85283705e74d913e31230387f Mon Sep 17 00:00:00 2001
From: Xing SHI <shi-kou@indetail.co.jp>
Date: Wed, 14 Sep 2016 13:46:46 -0700
Subject: [PATCH] [SPARK-17465][SPARK CORE] Inappropriate memory management in
`org.apache.spark.storage.MemoryStore` may lead to memory leak
The expression like `if (memoryMap(taskAttemptId) == 0) memoryMap.remove(taskAttemptId)` in method `releaseUnrollMemoryForThisTask` and `releasePendingUnrollMemoryForThisTask` should be called after release memory operation, whatever `memoryToRelease` is > 0 or not.
If the memory of a task has been set to 0 when calling a `releaseUnrollMemoryForThisTask` or a `releasePendingUnrollMemoryForThisTask` method, the key in the memory map corresponding to that task will never be removed from the hash map.
See the details in [SPARK-17465](https://issues.apache.org/jira/browse/SPARK-17465).
Author: Xing SHI <shi-kou@indetail.co.jp>
Closes #15022 from saturday-shi/SPARK-17465.
---
.../scala/org/apache/spark/storage/memory/MemoryStore.scala | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
index baa3fde2d0..ec1b0f7149 100644
--- a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
+++ b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
@@ -593,11 +593,11 @@ private[spark] class MemoryStore(
val memoryToRelease = math.min(memory, unrollMemoryMap(taskAttemptId))
if (memoryToRelease > 0) {
unrollMemoryMap(taskAttemptId) -= memoryToRelease
- if (unrollMemoryMap(taskAttemptId) == 0) {
- unrollMemoryMap.remove(taskAttemptId)
- }
memoryManager.releaseUnrollMemory(memoryToRelease, memoryMode)
}
+ if (unrollMemoryMap(taskAttemptId) == 0) {
+ unrollMemoryMap.remove(taskAttemptId)
+ }
}
}
}
--
GitLab