Skip to content
Snippets Groups Projects
Commit fbaf9e08 authored by wangfei's avatar wangfei Committed by Michael Armbrust
Browse files

[SPARK-5367][SQL] Support star expression in udf

now spark sql does not support star expression in udf, run the following sql by spark-sql will get error
```
select concat(*) from src
```

Author: wangfei <wangfei1@huawei.com>
Author: scwf <wangfei1@huawei.com>

Closes #4163 from scwf/udf-star and squashes the following commits:

9db7b39 [wangfei] addressed comments
da1da09 [scwf] minor fix
f87b5f9 [scwf] added test case
587bf7e [wangfei] compile fix
eb93c16 [wangfei] fix star resolve issue in udf
parent de221ea0
No related branches found
No related tags found
No related merge requests found
......@@ -250,6 +250,12 @@ class Analyzer(catalog: Catalog,
Project(
projectList.flatMap {
case s: Star => s.expand(child.output, resolver)
case Alias(f @ UnresolvedFunction(_, args), name) if containsStar(args) =>
val expandedArgs = args.flatMap {
case s: Star => s.expand(child.output, resolver)
case o => o :: Nil
}
Alias(child = f.copy(children = expandedArgs), name)() :: Nil
case o => o :: Nil
},
child)
......@@ -273,10 +279,9 @@ class Analyzer(catalog: Catalog,
case q: LogicalPlan =>
logTrace(s"Attempting to resolve ${q.simpleString}")
q transformExpressions {
case u @ UnresolvedAttribute(name)
if resolver(name, VirtualColumn.groupingIdName) &&
q.isInstanceOf[GroupingAnalytics] =>
// Resolve the virtual column GROUPING__ID for the operator GroupingAnalytics
case u @ UnresolvedAttribute(name) if resolver(name, VirtualColumn.groupingIdName) &&
q.isInstanceOf[GroupingAnalytics] =>
// Resolve the virtual column GROUPING__ID for the operator GroupingAnalytics
q.asInstanceOf[GroupingAnalytics].gid
case u @ UnresolvedAttribute(name) =>
// Leave unchanged if resolution fails. Hopefully will be resolved next round.
......@@ -299,7 +304,7 @@ class Analyzer(catalog: Catalog,
* Returns true if `exprs` contains a [[Star]].
*/
protected def containsStar(exprs: Seq[Expression]): Boolean =
exprs.collect { case _: Star => true}.nonEmpty
exprs.exists(_.collect { case _: Star => true }.nonEmpty)
}
/**
......
......@@ -509,6 +509,11 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
assert(sql("select key from src having key > 490").collect().size < 100)
}
test("SPARK-5367: resolve star expression in udf") {
assert(sql("select concat(*) from src limit 5").collect().size == 5)
assert(sql("select array(*) from src limit 5").collect().size == 5)
}
test("Query Hive native command execution result") {
val tableName = "test_native_commands"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment