Skip to content
Snippets Groups Projects
Commit 8577523f authored by Shivaram Venkataraman's avatar Shivaram Venkataraman
Browse files

Add test to verify if RDD is computed even if block manager has insufficient

memory
parent 2cf40c5f
No related branches found
No related tags found
No related merge requests found
......@@ -27,6 +27,7 @@ class DistributedSuite extends FunSuite with ShouldMatchers with BeforeAndAfter
sc = null
}
System.clearProperty("spark.reducer.maxMbInFlight")
System.clearProperty("spark.storage.memoryFraction")
// To avoid Akka rebinding to the same port, since it doesn't unbind immediately on shutdown
System.clearProperty("spark.master.port")
}
......@@ -156,4 +157,13 @@ class DistributedSuite extends FunSuite with ShouldMatchers with BeforeAndAfter
assert(data.count() === 1000)
assert(data.count() === 1000)
}
test("compute without caching with low memory") {
System.setProperty("spark.storage.memoryFraction", "0.0001")
sc = new SparkContext(clusterUrl, "test")
val data = sc.parallelize(1 to 4000000, 2).persist(StorageLevel.MEMORY_ONLY)
assert(data.count() === 4000000)
assert(data.count() === 4000000)
assert(data.count() === 4000000)
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment