diff --git a/core/lib/kryo-1.04-mod/kryo-1.04-mod.jar b/core/lib/kryo-1.04-mod/kryo-1.04-mod.jar index 7c4a8d3af8efd62b713e4f7fe93ad6aade7690af..815c1c8d94be281274056df29b852707cb5ab812 100644 Binary files a/core/lib/kryo-1.04-mod/kryo-1.04-mod.jar and b/core/lib/kryo-1.04-mod/kryo-1.04-mod.jar differ diff --git a/core/lib/kryo-1.04-mod/objenesis-1.2.jar b/core/lib/kryo-1.04-mod/objenesis-1.2.jar new file mode 100644 index 0000000000000000000000000000000000000000..45cb6416838ba263335373cd93d507cccfcdefe6 Binary files /dev/null and b/core/lib/kryo-1.04-mod/objenesis-1.2.jar differ diff --git a/core/src/main/scala/spark/KryoSerialization.scala b/core/src/main/scala/spark/KryoSerialization.scala index cd169fa7abc9fb721d5d72832920a0411519c9d5..462dee217e4a36a6a9b740de7fbff00c0ad5e6c5 100644 --- a/core/src/main/scala/spark/KryoSerialization.scala +++ b/core/src/main/scala/spark/KryoSerialization.scala @@ -8,6 +8,7 @@ import scala.collection.immutable import scala.collection.mutable import com.esotericsoftware.kryo._ +import com.esotericsoftware.kryo.{Serializer => KSerializer} object ZigZag { def writeInt(n: Int, out: OutputStream) { @@ -115,6 +116,8 @@ class KryoSerialization extends SerializationStrategy with Logging { def createKryo(): Kryo = { val kryo = new Kryo() + + // Register some commonly used classes val toRegister: Seq[AnyRef] = Seq( // Arrays Array(1), Array(1.0), Array(1.0f), Array(1L), Array(""), Array(("", "")), @@ -123,16 +126,29 @@ class KryoSerialization extends SerializationStrategy with Logging { ("", ""), (1, 1), (1.0, 1.0), (1L, 1L), (1, 1.0), (1.0, 1), (1L, 1.0), (1.0, 1L), (1, 1L), (1L, 1), // Scala collections - Nil, List(1), immutable.Map(1 -> 1), immutable.HashMap(1 -> 1), + List(1), immutable.Map(1 -> 1), immutable.HashMap(1 -> 1), mutable.Map(1 -> 1), mutable.HashMap(1 -> 1), mutable.ArrayBuffer(1), // Options and Either - Some(1), None, Left(1), Right(1), + Some(1), Left(1), Right(1), // Higher-dimensional tuples (1, 1, 1), (1, 1, 1, 1), (1, 1, 1, 1, 1) ) for (obj <- toRegister) { kryo.register(obj.getClass) } + + // Register some commonly used Scala singleton objects. Because these + // are singletons, we must return the exact same local object when we + // deserialize rather than returning a clone as FieldSerializer would. + kryo.register(None.getClass, new KSerializer { + override def writeObjectData(buf: ByteBuffer, obj: AnyRef) {} + override def readObjectData[T](buf: ByteBuffer, cls: Class[T]): T = None.asInstanceOf[T] + }) + kryo.register(Nil.getClass, new KSerializer { + override def writeObjectData(buf: ByteBuffer, obj: AnyRef) {} + override def readObjectData[T](buf: ByteBuffer, cls: Class[T]): T = Nil.asInstanceOf[T] + }) + val regCls = System.getProperty("spark.kryo.registrator") if (regCls != null) { logInfo("Running user registrator: " + regCls) diff --git a/run b/run index 10d2845c9748d5f83a3658a299dfe0c0fa226c29..dd656f66cb137d9d9442af748741a62d582c9883 100755 --- a/run +++ b/run @@ -50,6 +50,7 @@ CLASSPATH+=:$CORE_DIR/lib/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar CLASSPATH+=:$CORE_DIR/lib/jetty-7.1.6.v20100715/servlet-api-2.5.jar CLASSPATH+=:$CORE_DIR/lib/kryo-1.04-mod/kryo-1.04-mod.jar CLASSPATH+=:$CORE_DIR/lib/kryo-1.04-mod/minlog-1.2.jar +CLASSPATH+=:$CORE_DIR/lib/kryo-1.04-mod/objenesis-1.2.jar CLASSPATH+=:$CORE_DIR/lib/kryo-1.04-mod/reflectasm-1.01.jar CLASSPATH+=:$CORE_DIR/lib/apache-log4j-1.2.16/log4j-1.2.16.jar CLASSPATH+=:$CORE_DIR/lib/slf4j-1.6.1/slf4j-api-1.6.1.jar