From 1536d70331e9a4f5b5ea9dabfd72592ca1fc8e35 Mon Sep 17 00:00:00 2001
From: zsxwing <zsxwing@gmail.com>
Date: Tue, 28 Oct 2014 17:59:10 -0700
Subject: [PATCH] [SPARK-4008] Fix "kryo with fold" in KryoSerializerSuite
`zeroValue` will be serialized by `spark.closure.serializer` but `spark.closure.serializer` only supports the default Java serializer. So it must not be `ClassWithoutNoArgConstructor`, which can not be serialized by the Java serializer.
This PR changed `zeroValue` to null and updated the test to make it work correctly.
Author: zsxwing <zsxwing@gmail.com>
Closes #2856 from zsxwing/SPARK-4008 and squashes the following commits:
51da655 [zsxwing] [SPARK-4008] Fix "kryo with fold" in KryoSerializerSuite
---
.../spark/serializer/KryoSerializerSuite.scala | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index 64ac6d2d92..a70f67af2e 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -201,12 +201,17 @@ class KryoSerializerSuite extends FunSuite with SharedSparkContext {
assert(control.sum === result)
}
- // TODO: this still doesn't work
- ignore("kryo with fold") {
+ test("kryo with fold") {
val control = 1 :: 2 :: Nil
+ // zeroValue must not be a ClassWithoutNoArgConstructor instance because it will be
+ // serialized by spark.closure.serializer but spark.closure.serializer only supports
+ // the default Java serializer.
val result = sc.parallelize(control, 2).map(new ClassWithoutNoArgConstructor(_))
- .fold(new ClassWithoutNoArgConstructor(10))((t1, t2) => new ClassWithoutNoArgConstructor(t1.x + t2.x)).x
- assert(10 + control.sum === result)
+ .fold(null)((t1, t2) => {
+ val t1x = if (t1 == null) 0 else t1.x
+ new ClassWithoutNoArgConstructor(t1x + t2.x)
+ }).x
+ assert(control.sum === result)
}
test("kryo with nonexistent custom registrator should fail") {
--
GitLab