Skip to content
Snippets Groups Projects
Commit 370b9581 authored by Matei Zaharia's avatar Matei Zaharia
Browse files

Added sampling for large arrays in SizeEstimator

parent 021c50a8
No related branches found
No related tags found
No related merge requests found
...@@ -5,6 +5,7 @@ import java.lang.reflect.Modifier ...@@ -5,6 +5,7 @@ import java.lang.reflect.Modifier
import java.lang.reflect.{Array => JArray} import java.lang.reflect.{Array => JArray}
import java.util.IdentityHashMap import java.util.IdentityHashMap
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
import java.util.Random
import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.ArrayBuffer
...@@ -98,8 +99,20 @@ object SizeEstimator { ...@@ -98,8 +99,20 @@ object SizeEstimator {
state.size += length * primitiveSize(elementClass) state.size += length * primitiveSize(elementClass)
} else { } else {
state.size += length * POINTER_SIZE state.size += length * POINTER_SIZE
for (i <- 0 until length) { if (length <= 100) {
state.enqueue(JArray.get(array, i)) for (i <- 0 until length) {
state.enqueue(JArray.get(array, i))
}
} else {
// Estimate the size of a large array by sampling elements.
// TODO: Add a config setting for turning this off?
var size = 0.0
val rand = new Random(42)
for (i <- 0 until 100) {
val elem = JArray.get(array, rand.nextInt(length))
size += SizeEstimator.estimate(elem)
}
state.size += ((length / 100.0) * size).toLong
} }
} }
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment