From fbaf9e08961551d3ae5c3629eca01e839b001b8e Mon Sep 17 00:00:00 2001
From: wangfei <wangfei1@huawei.com>
Date: Thu, 29 Jan 2015 15:44:53 -0800
Subject: [PATCH] [SPARK-5367][SQL] Support star expression in udf
now spark sql does not support star expression in udf, run the following sql by spark-sql will get error
```
select concat(*) from src
```
Author: wangfei <wangfei1@huawei.com>
Author: scwf <wangfei1@huawei.com>
Closes #4163 from scwf/udf-star and squashes the following commits:
9db7b39 [wangfei] addressed comments
da1da09 [scwf] minor fix
f87b5f9 [scwf] added test case
587bf7e [wangfei] compile fix
eb93c16 [wangfei] fix star resolve issue in udf
---
.../spark/sql/catalyst/analysis/Analyzer.scala | 15 ++++++++++-----
.../spark/sql/hive/execution/HiveQuerySuite.scala | 5 +++++
2 files changed, 15 insertions(+), 5 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 7f4cc234dc..cefd70acf3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -250,6 +250,12 @@ class Analyzer(catalog: Catalog,
Project(
projectList.flatMap {
case s: Star => s.expand(child.output, resolver)
+ case Alias(f @ UnresolvedFunction(_, args), name) if containsStar(args) =>
+ val expandedArgs = args.flatMap {
+ case s: Star => s.expand(child.output, resolver)
+ case o => o :: Nil
+ }
+ Alias(child = f.copy(children = expandedArgs), name)() :: Nil
case o => o :: Nil
},
child)
@@ -273,10 +279,9 @@ class Analyzer(catalog: Catalog,
case q: LogicalPlan =>
logTrace(s"Attempting to resolve ${q.simpleString}")
q transformExpressions {
- case u @ UnresolvedAttribute(name)
- if resolver(name, VirtualColumn.groupingIdName) &&
- q.isInstanceOf[GroupingAnalytics] =>
- // Resolve the virtual column GROUPING__ID for the operator GroupingAnalytics
+ case u @ UnresolvedAttribute(name) if resolver(name, VirtualColumn.groupingIdName) &&
+ q.isInstanceOf[GroupingAnalytics] =>
+ // Resolve the virtual column GROUPING__ID for the operator GroupingAnalytics
q.asInstanceOf[GroupingAnalytics].gid
case u @ UnresolvedAttribute(name) =>
// Leave unchanged if resolution fails. Hopefully will be resolved next round.
@@ -299,7 +304,7 @@ class Analyzer(catalog: Catalog,
* Returns true if `exprs` contains a [[Star]].
*/
protected def containsStar(exprs: Seq[Expression]): Boolean =
- exprs.collect { case _: Star => true}.nonEmpty
+ exprs.exists(_.collect { case _: Star => true }.nonEmpty)
}
/**
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 42819e3584..60619f5d99 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -509,6 +509,11 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
assert(sql("select key from src having key > 490").collect().size < 100)
}
+ test("SPARK-5367: resolve star expression in udf") {
+ assert(sql("select concat(*) from src limit 5").collect().size == 5)
+ assert(sql("select array(*) from src limit 5").collect().size == 5)
+ }
+
test("Query Hive native command execution result") {
val tableName = "test_native_commands"
--
GitLab