From f30373f5ee60f9892c28771e34b208e4f1f675a6 Mon Sep 17 00:00:00 2001
From: rotems <roter>
Date: Fri, 4 Dec 2015 16:58:31 -0800
Subject: [PATCH] [SPARK-12080][CORE] Kryo - Support multiple user registrators
Author: rotems <roter>
Closes #10078 from Botnaim/KryoMultipleCustomRegistrators.
---
.../scala/org/apache/spark/serializer/KryoSerializer.scala | 6 ++++--
docs/configuration.md | 4 ++--
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index d5ba690ed0..7b77f78ce6 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -70,7 +70,9 @@ class KryoSerializer(conf: SparkConf)
private val referenceTracking = conf.getBoolean("spark.kryo.referenceTracking", true)
private val registrationRequired = conf.getBoolean("spark.kryo.registrationRequired", false)
- private val userRegistrator = conf.getOption("spark.kryo.registrator")
+ private val userRegistrators = conf.get("spark.kryo.registrator", "")
+ .split(',')
+ .filter(!_.isEmpty)
private val classesToRegister = conf.get("spark.kryo.classesToRegister", "")
.split(',')
.filter(!_.isEmpty)
@@ -119,7 +121,7 @@ class KryoSerializer(conf: SparkConf)
classesToRegister
.foreach { className => kryo.register(Class.forName(className, true, classLoader)) }
// Allow the user to register their own classes by setting spark.kryo.registrator.
- userRegistrator
+ userRegistrators
.map(Class.forName(_, true, classLoader).newInstance().asInstanceOf[KryoRegistrator])
.foreach { reg => reg.registerClasses(kryo) }
// scalastyle:on classforname
diff --git a/docs/configuration.md b/docs/configuration.md
index c39b489085..fd61ddc244 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -647,10 +647,10 @@ Apart from these, the following properties are also available, and may be useful
<td><code>spark.kryo.registrator</code></td>
<td>(none)</td>
<td>
- If you use Kryo serialization, set this class to register your custom classes with Kryo. This
+ If you use Kryo serialization, give a comma-separated list of classes that register your custom classes with Kryo. This
property is useful if you need to register your classes in a custom way, e.g. to specify a custom
field serializer. Otherwise <code>spark.kryo.classesToRegister</code> is simpler. It should be
- set to a class that extends
+ set to classes that extend
<a href="api/scala/index.html#org.apache.spark.serializer.KryoRegistrator">
<code>KryoRegistrator</code></a>.
See the <a href="tuning.html#data-serialization">tuning guide</a> for more details.
--
GitLab