From fd2e40614b511fb9ef3e52cc1351659fdbfd612a Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 28 Mar 2017 11:47:43 -0700
Subject: [PATCH] [SPARK-20125][SQL] Dataset of type option of map does not
work
When we build the deserializer expression for map type, we will use `StaticInvoke` to call `ArrayBasedMapData.toScalaMap`, and declare the return type as `scala.collection.immutable.Map`. If the map is inside an Option, we will wrap this `StaticInvoke` with `WrapOption`, which requires the input to be `scala.collect.Map`. Ideally this should be fine, as `scala.collection.immutable.Map` extends `scala.collect.Map`, but our `ObjectType` is too strict about this, this PR fixes it.
new regression test
Author: Wenchen Fan <wenchen@databricks.com>
Closes #17454 from cloud-fan/map.
(cherry picked from commit d4fac410e0554b7ccd44be44b7ce2fe07ed7f206)
Signed-off-by: Cheng Lian <lian@databricks.com>
---
.../main/scala/org/apache/spark/sql/types/ObjectType.scala | 5 +++++
.../src/test/scala/org/apache/spark/sql/DatasetSuite.scala | 6 ++++++
2 files changed, 11 insertions(+)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ObjectType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ObjectType.scala
index b18fba29af..2d49fe0767 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ObjectType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ObjectType.scala
@@ -44,4 +44,9 @@ case class ObjectType(cls: Class[_]) extends DataType {
def asNullable: DataType = this
override def simpleString: String = cls.getName
+
+ override def acceptsType(other: DataType): Boolean = other match {
+ case ObjectType(otherCls) => cls.isAssignableFrom(otherCls)
+ case _ => false
+ }
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 381652d337..9cc49b66b7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -1072,10 +1072,16 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
val ds2 = Seq(WithMap("hi", Map(42L -> "foo"))).toDS
checkDataset(ds2.map(t => t), WithMap("hi", Map(42L -> "foo")))
}
+
+ test("SPARK-20125: option of map") {
+ val ds = Seq(WithMapInOption(Some(Map(1 -> 1)))).toDS()
+ checkDataset(ds, WithMapInOption(Some(Map(1 -> 1))))
+ }
}
case class WithImmutableMap(id: String, map_test: scala.collection.immutable.Map[Long, String])
case class WithMap(id: String, map_test: scala.collection.Map[Long, String])
+case class WithMapInOption(m: Option[scala.collection.Map[Int, Int]])
case class Generic[T](id: T, value: Double)
--
GitLab