From 916fc34f98dd731f607d9b3ed657bad6cc30df2c Mon Sep 17 00:00:00 2001
From: Cheng Lian <lian@databricks.com>
Date: Tue, 1 Mar 2016 01:07:45 +0800
Subject: [PATCH] [SPARK-13540][SQL] Supports using nested classes within Scala
objects as Dataset element type
## What changes were proposed in this pull request?
Nested classes defined within Scala objects are translated into Java static nested classes. Unlike inner classes, they don't need outer scopes. But the analyzer still thinks that an outer scope is required.
This PR fixes this issue simply by checking whether a nested class is static before looking up its outer scope.
## How was this patch tested?
A test case is added to `DatasetSuite`. It checks contents of a Dataset whose element type is a nested class declared in a Scala object.
Author: Cheng Lian <lian@databricks.com>
Closes #11421 from liancheng/spark-13540-object-as-outer-scope.
---
.../apache/spark/sql/catalyst/analysis/Analyzer.scala | 10 +++++++++-
.../test/scala/org/apache/spark/sql/DatasetSuite.scala | 10 ++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 23e4709bbd..876aa0eae0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.catalyst.analysis
+import java.lang.reflect.Modifier
+
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.sql.AnalysisException
@@ -559,7 +561,13 @@ class Analyzer(
}
resolveExpression(unbound, LocalRelation(attributes), throws = true) transform {
- case n: NewInstance if n.outerPointer.isEmpty && n.cls.isMemberClass =>
+ case n: NewInstance
+ // If this is an inner class of another class, register the outer object in `OuterScopes`.
+ // Note that static inner classes (e.g., inner classes within Scala objects) don't need
+ // outer pointer registration.
+ if n.outerPointer.isEmpty &&
+ n.cls.isMemberClass &&
+ !Modifier.isStatic(n.cls.getModifiers) =>
val outer = OuterScopes.outerScopes.get(n.cls.getDeclaringClass.getName)
if (outer == null) {
throw new AnalysisException(
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 14fc37b64a..33df6375e3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -621,12 +621,22 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
ds.filter(_ => true),
Some(1), Some(2), Some(3))
}
+
+ test("SPARK-13540 Dataset of nested class defined in Scala object") {
+ checkAnswer(
+ Seq(OuterObject.InnerClass("foo")).toDS(),
+ OuterObject.InnerClass("foo"))
+ }
}
class OuterClass extends Serializable {
case class InnerClass(a: String)
}
+object OuterObject {
+ case class InnerClass(a: String)
+}
+
case class ClassData(a: String, b: Int)
case class ClassData2(c: String, d: Int)
case class ClassNullableData(a: String, b: Integer)
--
GitLab