From 314e48a3584bad4b486b046bbf0159d64ba857bc Mon Sep 17 00:00:00 2001
From: Michael Armbrust <michael@databricks.com>
Date: Wed, 8 Mar 2017 01:32:42 -0800
Subject: [PATCH] [SPARK-18055][SQL] Use correct mirror in ExpresionEncoder
Previously, we were using the mirror of passed in `TypeTag` when reflecting to build an encoder. This fails when the outer class is built in (i.e. `Seq`'s default mirror is based on root classloader) but inner classes (i.e. `A` in `Seq[A]`) are defined in the REPL or a library.
This patch changes us to always reflect based on a mirror created using the context classloader.
Author: Michael Armbrust <michael@databricks.com>
Closes #17201 from marmbrus/replSeqEncoder.
---
.../test/scala/org/apache/spark/repl/ReplSuite.scala | 11 +++++++++++
.../sql/catalyst/encoders/ExpressionEncoder.scala | 4 ++--
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
index 55c91675ed..121a02a9be 100644
--- a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
+++ b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
@@ -473,4 +473,15 @@ class ReplSuite extends SparkFunSuite {
assertDoesNotContain("AssertionError", output)
assertDoesNotContain("Exception", output)
}
+
+ test("newProductSeqEncoder with REPL defined class") {
+ val output = runInterpreterInPasteMode("local-cluster[1,4,4096]",
+ """
+ |case class Click(id: Int)
+ |spark.implicits.newProductSeqEncoder[Click]
+ """.stripMargin)
+
+ assertDoesNotContain("error:", output)
+ assertDoesNotContain("Exception", output)
+ }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
index 0782143d46..93fc565a53 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
@@ -45,8 +45,8 @@ import org.apache.spark.util.Utils
object ExpressionEncoder {
def apply[T : TypeTag](): ExpressionEncoder[T] = {
// We convert the not-serializable TypeTag into StructType and ClassTag.
- val mirror = typeTag[T].mirror
- val tpe = typeTag[T].tpe
+ val mirror = ScalaReflection.mirror
+ val tpe = typeTag[T].in(mirror).tpe
if (ScalaReflection.optionOfProductType(tpe)) {
throw new UnsupportedOperationException(
--
GitLab