From 10c566cc3b5f93ddd823b1c987d9de8286cdea60 Mon Sep 17 00:00:00 2001
From: Bogdan Raducanu <bogdan@databricks.com>
Date: Wed, 22 Feb 2017 15:42:40 +0100
Subject: [PATCH] [SPARK-13721][SQL] Make GeneratorOuter unresolved.
## What changes were proposed in this pull request?
This is a small change to make GeneratorOuter always unresolved. It is mostly no-op change but makes it more clear since GeneratorOuter shouldn't survive analysis phase.
This requires also handling in ResolveAliases rule.
## How was this patch tested?
Existing generator tests.
Author: Bogdan Raducanu <bogdan@databricks.com>
Author: Reynold Xin <rxin@databricks.com>
Closes #17026 from bogdanrdc/PR16958.
---
.../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 2 +-
.../spark/sql/catalyst/expressions/generators.scala | 8 +++++++-
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 39a276284c..c477cb48d0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -222,6 +222,7 @@ class Analyzer(
expr.transformUp { case u @ UnresolvedAlias(child, optGenAliasFunc) =>
child match {
case ne: NamedExpression => ne
+ case go @ GeneratorOuter(g: Generator) if g.resolved => MultiAlias(go, Nil)
case e if !e.resolved => u
case g: Generator => MultiAlias(g, Nil)
case c @ Cast(ne: NamedExpression, _, _) => Alias(c, ne.name)()
@@ -1665,7 +1666,6 @@ class Analyzer(
var resolvedGenerator: Generate = null
val newProjectList = projectList.flatMap {
-
case AliasedGenerator(generator, names, outer) if generator.childrenResolved =>
// It's a sanity check, this should not happen as the previous case will throw
// exception earlier.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
index 1b98c30d37..e84796f2ed 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
@@ -173,7 +173,6 @@ case class Stack(children: Seq[Expression]) extends Generator {
}
}
-
/**
* Only support code generation when stack produces 50 rows or less.
*/
@@ -204,6 +203,10 @@ case class Stack(children: Seq[Expression]) extends Generator {
}
}
+/**
+ * Wrapper around another generator to specify outer behavior. This is used to implement functions
+ * such as explode_outer. This expression gets replaced during analysis.
+ */
case class GeneratorOuter(child: Generator) extends UnaryExpression with Generator {
final override def eval(input: InternalRow = null): TraversableOnce[InternalRow] =
throw new UnsupportedOperationException(s"Cannot evaluate expression: $this")
@@ -212,7 +215,10 @@ case class GeneratorOuter(child: Generator) extends UnaryExpression with Generat
throw new UnsupportedOperationException(s"Cannot evaluate expression: $this")
override def elementSchema: StructType = child.elementSchema
+
+ override lazy val resolved: Boolean = false
}
+
/**
* A base class for [[Explode]] and [[PosExplode]].
*/
--
GitLab