From 49d2ec63eccec8a3a78b15b583c36f84310fc6f0 Mon Sep 17 00:00:00 2001
From: mcheah <mcheah@palantir.com>
Date: Thu, 26 Mar 2015 22:48:42 -0700
Subject: [PATCH] [SPARK-6405] Limiting the maximum Kryo buffer size to be 2GB.
Kryo buffers are backed by byte arrays, but primitive arrays can only be
up to 2GB in size. It is misleading to allow users to set buffers past
this size.
Author: mcheah <mcheah@palantir.com>
Closes #5218 from mccheah/feature/limit-kryo-buffer and squashes the following commits:
1d6d1be [mcheah] Fixing numeric typo
e2e30ce [mcheah] Removing explicit int and double type to match style
09fd80b [mcheah] Should be >= not >. Slightly more consistent error message.
60634f9 [mcheah] [SPARK-6405] Limiting the maximum Kryo buffer size to be 2GB.
---
.../apache/spark/serializer/KryoSerializer.scala | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index f83bcaa5cc..579fb6624e 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -49,10 +49,20 @@ class KryoSerializer(conf: SparkConf)
with Logging
with Serializable {
- private val bufferSize =
- (conf.getDouble("spark.kryoserializer.buffer.mb", 0.064) * 1024 * 1024).toInt
+ private val bufferSizeMb = conf.getDouble("spark.kryoserializer.buffer.mb", 0.064)
+ if (bufferSizeMb >= 2048) {
+ throw new IllegalArgumentException("spark.kryoserializer.buffer.mb must be less than " +
+ s"2048 mb, got: + $bufferSizeMb mb.")
+ }
+ private val bufferSize = (bufferSizeMb * 1024 * 1024).toInt
+
+ val maxBufferSizeMb = conf.getInt("spark.kryoserializer.buffer.max.mb", 64)
+ if (maxBufferSizeMb >= 2048) {
+ throw new IllegalArgumentException("spark.kryoserializer.buffer.max.mb must be less than " +
+ s"2048 mb, got: + $maxBufferSizeMb mb.")
+ }
+ private val maxBufferSize = maxBufferSizeMb * 1024 * 1024
- private val maxBufferSize = conf.getInt("spark.kryoserializer.buffer.max.mb", 64) * 1024 * 1024
private val referenceTracking = conf.getBoolean("spark.kryo.referenceTracking", true)
private val registrationRequired = conf.getBoolean("spark.kryo.registrationRequired", false)
private val userRegistrator = conf.getOption("spark.kryo.registrator")
--
GitLab