From c1b3eebf97b986439f71afd3c4eccf47b90da2cd Mon Sep 17 00:00:00 2001
From: wangfei <wangfei1@huawei.com>
Date: Thu, 29 Jan 2015 15:47:13 -0800
Subject: [PATCH] [SPARK-5373][SQL] Literal in agg grouping expressions leads
to incorrect result
`select key, count( * ) from src group by key, 1` will get the wrong answer.
e.g. for this table
```
val testData2 =
TestSQLContext.sparkContext.parallelize(
TestData2(1, 1) ::
TestData2(1, 2) ::
TestData2(2, 1) ::
TestData2(2, 2) ::
TestData2(3, 1) ::
TestData2(3, 2) :: Nil, 2).toSchemaRDD
testData2.registerTempTable("testData2")
```
result of `SELECT a, count(1) FROM testData2 GROUP BY a, 1` is
```
[1,1]
[2,2]
[3,1]
```
Author: wangfei <wangfei1@huawei.com>
Closes #4169 from scwf/agg-bug and squashes the following commits:
05751db [wangfei] fix bugs when literal in agg grouping expressioons
---
.../apache/spark/sql/catalyst/planning/patterns.scala | 9 +++++----
.../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 9 +++++++++
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
index 310d127506..b4c445b3ba 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
@@ -141,10 +141,11 @@ object PartialAggregation {
// We need to pass all grouping expressions though so the grouping can happen a second
// time. However some of them might be unnamed so we alias them allowing them to be
// referenced in the second aggregation.
- val namedGroupingExpressions: Map[Expression, NamedExpression] = groupingExpressions.map {
- case n: NamedExpression => (n, n)
- case other => (other, Alias(other, "PartialGroup")())
- }.toMap
+ val namedGroupingExpressions: Map[Expression, NamedExpression] =
+ groupingExpressions.filter(!_.isInstanceOf[Literal]).map {
+ case n: NamedExpression => (n, n)
+ case other => (other, Alias(other, "PartialGroup")())
+ }.toMap
// Replace aggregations with a new expression that computes the result from the already
// computed partial evaluations and grouping values.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index e03444d496..d684278f11 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -186,6 +186,15 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
Seq(Row(1,3), Row(2,3), Row(3,3)))
}
+ test("literal in agg grouping expressions") {
+ checkAnswer(
+ sql("SELECT a, count(1) FROM testData2 GROUP BY a, 1"),
+ Seq(Row(1,2), Row(2,2), Row(3,2)))
+ checkAnswer(
+ sql("SELECT a, count(2) FROM testData2 GROUP BY a, 2"),
+ Seq(Row(1,2), Row(2,2), Row(3,2)))
+ }
+
test("aggregates with nulls") {
checkAnswer(
sql("SELECT MIN(a), MAX(a), AVG(a), SUM(a), COUNT(a) FROM nullInts"),
--
GitLab