From 87f28fc24003ad60c52f899d10f38032631624dc Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 29 Oct 2015 11:17:03 +0100
Subject: [PATCH] [SPARK-11379][SQL] ExpressionEncoder can't handle top level
primitive type correctly
For inner primitive type(e.g. inside `Product`), we use `schemaFor` to get the catalyst type for it, https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala#L403.
However, for top level primitive type, we use `dataTypeFor`, which is wrong.
Author: Wenchen Fan <wenchen@databricks.com>
Closes #9337 from cloud-fan/encoder.
---
.../scala/org/apache/spark/sql/catalyst/ScalaReflection.scala | 2 +-
.../spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 9cbb7c2ffd..0b8a8abd02 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -170,7 +170,7 @@ trait ScalaReflection {
.getOrElse(BoundReference(ordinal, dataType, false))
/** Returns the current path or throws an error. */
- def getPath = path.getOrElse(BoundReference(0, dataTypeFor(tpe), true))
+ def getPath = path.getOrElse(BoundReference(0, schemaFor(tpe).dataType, true))
tpe match {
case t if !dataTypeFor(t).isInstanceOf[ObjectType] =>
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
index a374da4da1..b0dacf7f55 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
@@ -57,6 +57,7 @@ class ExpressionEncoderSuite extends SparkFunSuite {
encodeDecodeTest(false)
encodeDecodeTest(1.toShort)
encodeDecodeTest(1.toByte)
+ encodeDecodeTest("hello")
encodeDecodeTest(PrimitiveData(1, 1, 1, 1, 1, 1, true))
--
GitLab