diff --git a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
index ef75d639331b8f1deb66d5695063f45bb7135d55..d2d6e7b1096694613ec61ee5e0e35257bc2ade3e 100644
--- a/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
+++ b/core/src/main/scala/org/apache/spark/storage/memory/MemoryStore.scala
@@ -160,7 +160,7 @@ private[spark] class MemoryStore(
       val entry = new SerializedMemoryEntry[T](bytes, memoryMode, implicitly[ClassTag[T]])
       entries.synchronized {
         entries.put(blockId, entry)
-        blockIdAndSizeSet.put(OurBlockIdAndSizeType(blockId, entry.size))
+        blockIdAndSizeSet += OurBlockIdAndSizeType(blockId, entry.size)
       }
       logInfo("Block %s stored as bytes in memory (estimated size %s, free %s)".format(
         blockId, Utils.bytesToString(size), Utils.bytesToString(maxMemory - blocksMemoryUsed)))
@@ -270,7 +270,7 @@ private[spark] class MemoryStore(
 
         entries.synchronized {
           entries.put(blockId, entry)
-          blockIdAndSizeSet.put(OurBlockIdAndSizeType(blockId, entry.size))
+          blockIdAndSizeSet += OurBlockIdAndSizeType(blockId, entry.size)
         }
 
         logInfo("Block %s stored as values in memory (estimated size %s, free %s)".format(blockId,
@@ -398,7 +398,7 @@ private[spark] class MemoryStore(
     val entry = entries.synchronized {
       val origEntry = entries.get(blockId)
       entries.remove(blockId)
-      blockIdAndSizeSet.remove(OurBlockIdAndSizeType(blockId, origEntry.size))
+      blockIdAndSizeSet -= OurBlockIdAndSizeType(blockId, origEntry.size)
     }
     if (entry != null) {
       entry match {