From 841972e22c653ba58e9a65433fed203ff288f13a Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@appier.com>
Date: Tue, 15 Sep 2015 13:33:32 -0700
Subject: [PATCH] [SPARK-10437] [SQL] Support aggregation expressions in Order
By
JIRA: https://issues.apache.org/jira/browse/SPARK-10437
If an expression in `SortOrder` is a resolved one, such as `count(1)`, the corresponding rule in `Analyzer` to make it work in order by will not be applied.
Author: Liang-Chi Hsieh <viirya@appier.com>
Closes #8599 from viirya/orderby-agg.
---
.../sql/catalyst/analysis/Analyzer.scala | 14 +++++++++----
.../org/apache/spark/sql/SQLQuerySuite.scala | 20 +++++++++++++++++++
2 files changed, 30 insertions(+), 4 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 591747b45c..02f34cbf58 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -561,7 +561,7 @@ class Analyzer(
}
case sort @ Sort(sortOrder, global, aggregate: Aggregate)
- if aggregate.resolved && !sort.resolved =>
+ if aggregate.resolved =>
// Try resolving the ordering as though it is in the aggregate clause.
try {
@@ -598,9 +598,15 @@ class Analyzer(
}
}
- Project(aggregate.output,
- Sort(evaluatedOrderings, global,
- aggregate.copy(aggregateExpressions = originalAggExprs ++ needsPushDown)))
+ // Since we don't rely on sort.resolved as the stop condition for this rule,
+ // we need to check this and prevent applying this rule multiple times
+ if (sortOrder == evaluatedOrderings) {
+ sort
+ } else {
+ Project(aggregate.output,
+ Sort(evaluatedOrderings, global,
+ aggregate.copy(aggregateExpressions = originalAggExprs ++ needsPushDown)))
+ }
} catch {
// Attempting to resolve in the aggregate can result in ambiguity. When this happens,
// just return the original plan.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 962b100b53..f9981356f3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -1562,6 +1562,26 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
|ORDER BY sum(b) + 1
""".stripMargin),
Row("4", 3) :: Row("1", 7) :: Row("3", 11) :: Row("2", 15) :: Nil)
+
+ checkAnswer(
+ sql(
+ """
+ |SELECT count(*)
+ |FROM orderByData
+ |GROUP BY a
+ |ORDER BY count(*)
+ """.stripMargin),
+ Row(2) :: Row(2) :: Row(2) :: Row(2) :: Nil)
+
+ checkAnswer(
+ sql(
+ """
+ |SELECT a
+ |FROM orderByData
+ |GROUP BY a
+ |ORDER BY a, count(*), sum(b)
+ """.stripMargin),
+ Row("1") :: Row("2") :: Row("3") :: Row("4") :: Nil)
}
test("SPARK-7952: fix the equality check between boolean and numeric types") {
--
GitLab