Skip to content
Snippets Groups Projects
Commit 06b6a074 authored by Imran Rashid's avatar Imran Rashid Committed by Shivaram Venkataraman
Browse files

[SPARK-9437] [CORE] avoid overflow in SizeEstimator

https://issues.apache.org/jira/browse/SPARK-9437

Author: Imran Rashid <irashid@cloudera.com>

Closes #7750 from squito/SPARK-9437_size_estimator_overflow and squashes the following commits:

29493f1 [Imran Rashid] prevent another potential overflow
bc1cb82 [Imran Rashid] avoid overflow
parent 520ec0ff
No related branches found
No related tags found
No related merge requests found
...@@ -217,10 +217,10 @@ object SizeEstimator extends Logging { ...@@ -217,10 +217,10 @@ object SizeEstimator extends Logging {
var arrSize: Long = alignSize(objectSize + INT_SIZE) var arrSize: Long = alignSize(objectSize + INT_SIZE)
if (elementClass.isPrimitive) { if (elementClass.isPrimitive) {
arrSize += alignSize(length * primitiveSize(elementClass)) arrSize += alignSize(length.toLong * primitiveSize(elementClass))
state.size += arrSize state.size += arrSize
} else { } else {
arrSize += alignSize(length * pointerSize) arrSize += alignSize(length.toLong * pointerSize)
state.size += arrSize state.size += arrSize
if (length <= ARRAY_SIZE_FOR_SAMPLING) { if (length <= ARRAY_SIZE_FOR_SAMPLING) {
...@@ -336,7 +336,7 @@ object SizeEstimator extends Logging { ...@@ -336,7 +336,7 @@ object SizeEstimator extends Logging {
// hg.openjdk.java.net/jdk8/jdk8/hotspot/file/tip/src/share/vm/classfile/classFileParser.cpp // hg.openjdk.java.net/jdk8/jdk8/hotspot/file/tip/src/share/vm/classfile/classFileParser.cpp
var alignedSize = shellSize var alignedSize = shellSize
for (size <- fieldSizes if sizeCount(size) > 0) { for (size <- fieldSizes if sizeCount(size) > 0) {
val count = sizeCount(size) val count = sizeCount(size).toLong
// If there are internal gaps, smaller field can fit in. // If there are internal gaps, smaller field can fit in.
alignedSize = math.max(alignedSize, alignSizeUp(shellSize, size) + size * count) alignedSize = math.max(alignedSize, alignSizeUp(shellSize, size) + size * count)
shellSize += size * count shellSize += size * count
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment