From bc3b6cb06153d6b05f311dd78459768b6cf6a404 Mon Sep 17 00:00:00 2001
From: Nathan Howell <nhowell@godaddy.com>
Date: Thu, 9 Oct 2014 15:03:01 -0700
Subject: [PATCH] [SPARK-3858][SQL] Pass the generator alias into logical plan
node
The alias parameter is being ignored, which makes it more difficult to specify a qualifier for Generator expressions.
Author: Nathan Howell <nhowell@godaddy.com>
Closes #2721 from NathanHowell/SPARK-3858 and squashes the following commits:
8aa0f43 [Nathan Howell] [SPARK-3858][SQL] Pass the generator alias into logical plan node
---
.../src/main/scala/org/apache/spark/sql/SchemaRDD.scala | 2 +-
.../test/scala/org/apache/spark/sql/DslQuerySuite.scala | 8 ++++++++
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
index 594bf8ffc2..948122d42f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
@@ -360,7 +360,7 @@ class SchemaRDD(
join: Boolean = false,
outer: Boolean = false,
alias: Option[String] = None) =
- new SchemaRDD(sqlContext, Generate(generator, join, outer, None, logicalPlan))
+ new SchemaRDD(sqlContext, Generate(generator, join, outer, alias, logicalPlan))
/**
* Returns this RDD as a SchemaRDD. Intended primarily to force the invocation of the implicit
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala
index d001abb7e1..45e58afe9d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala
@@ -147,6 +147,14 @@ class DslQuerySuite extends QueryTest {
(1, 1, 1, 2) :: Nil)
}
+ test("SPARK-3858 generator qualifiers are discarded") {
+ checkAnswer(
+ arrayData.as('ad)
+ .generate(Explode("data" :: Nil, 'data), alias = Some("ex"))
+ .select("ex.data".attr),
+ Seq(1, 2, 3, 2, 3, 4).map(Seq(_)))
+ }
+
test("average") {
checkAnswer(
testData2.groupBy()(avg('a)),
--
GitLab