From 1d1a76c8c56148beec70dd14e4fdeb86de65018a Mon Sep 17 00:00:00 2001
From: Emiliano Leporati <emiliano.leporati@gmail.com>
Date: Wed, 5 Aug 2015 00:42:08 -0700
Subject: [PATCH] [SPARK-9581][SQL] Add unit test for JSON UDT
This brings #7416 up-to-date by drubbo.
Author: Emiliano Leporati <emiliano.leporati@gmail.com>
Author: Reynold Xin <rxin@databricks.com>
Closes #7917 from rxin/udt-json-test and squashes the following commits:
93e3954 [Reynold Xin] Fix test.
7035308 [Reynold Xin] Merge pull request #7416 from drubbo/master
b5bcd94 [Emiliano Leporati] removed unneded case in MyDenseVector::equals
508a399 [Emiliano Leporati] Merge remote branch 'upstream/master'
7569e42 [Emiliano Leporati] using checkAnswer
62daccd [Emiliano Leporati] added coverage for UDTs in JSON RDDs
---
.../apache/spark/sql/json/JacksonParser.scala | 2 +-
.../spark/sql/UserDefinedTypeSuite.scala | 20 +++++++++++++++++++
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala
index f1a66c84fc..b8fd3b9cc1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JacksonParser.scala
@@ -125,7 +125,7 @@ private[sql] object JacksonParser {
convertMap(factory, parser, kt)
case (_, udt: UserDefinedType[_]) =>
- udt.deserialize(convertField(factory, parser, udt.sqlType))
+ convertField(factory, parser, udt.sqlType)
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
index f29935224e..9181222f69 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
@@ -138,4 +138,24 @@ class UserDefinedTypeSuite extends QueryTest {
val actual = openHashSetUDT.deserialize(openHashSetUDT.serialize(set))
assert(actual.iterator.toSet === set.iterator.toSet)
}
+
+ test("UDTs with JSON") {
+ val data = Seq(
+ "{\"id\":1,\"vec\":[1.1,2.2,3.3,4.4]}",
+ "{\"id\":2,\"vec\":[2.25,4.5,8.75]}"
+ )
+ val schema = StructType(Seq(
+ StructField("id", IntegerType, false),
+ StructField("vec", new MyDenseVectorUDT, false)
+ ))
+
+ val stringRDD = ctx.sparkContext.parallelize(data)
+ val jsonRDD = ctx.read.schema(schema).json(stringRDD)
+ checkAnswer(
+ jsonRDD,
+ Row(1, new MyDenseVector(Array(1.1, 2.2, 3.3, 4.4))) ::
+ Row(2, new MyDenseVector(Array(2.25, 4.5, 8.75))) ::
+ Nil
+ )
+ }
}
--
GitLab