Skip to content
Snippets Groups Projects
Commit beebb7ff authored by Michael Armbrust's avatar Michael Armbrust
Browse files

[SPARK-5371][SQL] Propagate types after function conversion, before futher resolution

Before it was possible for a query to flip back and forth from a resolved state, allowing resolution to propagate up before coercion had stabilized.  The issue was that `ResolvedReferences` would run after `FunctionArgumentConversion`, but before `PropagateTypes` had run.  This PR ensures we correctly `PropagateTypes` after any coercion has applied.

Author: Michael Armbrust <michael@databricks.com>

Closes #5278 from marmbrus/unionNull and squashes the following commits:

dc3581a [Michael Armbrust] [SPARK-5371][SQL] Propogate types after function conversion / before futher resolution
parent b5bd75d9
No related branches found
No related tags found
No related merge requests found
......@@ -78,6 +78,7 @@ trait HiveTypeCoercion {
FunctionArgumentConversion ::
CaseWhenCoercion ::
Division ::
PropagateTypes ::
Nil
/**
......
......@@ -80,7 +80,7 @@ case class Union(left: LogicalPlan, right: LogicalPlan) extends BinaryNode {
override lazy val resolved: Boolean =
childrenResolved &&
!left.output.zip(right.output).exists { case (l,r) => l.dataType != r.dataType }
left.output.zip(right.output).forall { case (l,r) => l.dataType == r.dataType }
override def statistics: Statistics = {
val sizeInBytes = left.statistics.sizeInBytes + right.statistics.sizeInBytes
......
......@@ -41,8 +41,32 @@ case class NestedArray1(a: NestedArray2)
*/
class SQLQuerySuite extends QueryTest {
test("SPARK-5371: union with null and sum") {
val df = Seq((1, 1)).toDF("c1", "c2")
df.registerTempTable("table1")
val query = sql(
"""
|SELECT
| MIN(c1),
| MIN(c2)
|FROM (
| SELECT
| SUM(c1) c1,
| NULL c2
| FROM table1
| UNION ALL
| SELECT
| NULL c1,
| SUM(c2) c2
| FROM table1
|) a
""".stripMargin)
checkAnswer(query, Row(1, 1) :: Nil)
}
test("explode nested Field") {
Seq(NestedArray1(NestedArray2(Seq(1,2,3)))).toDF.registerTempTable("nestedArray")
Seq(NestedArray1(NestedArray2(Seq(1, 2, 3)))).toDF.registerTempTable("nestedArray")
checkAnswer(
sql("SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) a AS ints"),
Row(1) :: Row(2) :: Row(3) :: Nil)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment