From 0f37d1d7ed7f6e34f98f2a3c274918de29e7a1d7 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@appier.com>
Date: Tue, 1 Dec 2015 21:51:33 -0800
Subject: [PATCH] [SPARK-11949][SQL] Check bitmasks to set nullable property
Following up #10038.
We can use bitmasks to determine which grouping expressions need to be set as nullable.
cc yhuai
Author: Liang-Chi Hsieh <viirya@appier.com>
Closes #10067 from viirya/fix-cube-following.
---
.../spark/sql/catalyst/analysis/Analyzer.scala | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 765327c474..d3163dcd4d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -224,10 +224,15 @@ class Analyzer(
case other => Alias(other, other.toString)()
}
- // TODO: We need to use bitmasks to determine which grouping expressions need to be
- // set as nullable. For example, if we have GROUPING SETS ((a,b), a), we do not need
- // to change the nullability of a.
- val attributeMap = groupByAliases.map(a => (a -> a.toAttribute.withNullability(true))).toMap
+ val nonNullBitmask = x.bitmasks.reduce(_ & _)
+
+ val attributeMap = groupByAliases.zipWithIndex.map { case (a, idx) =>
+ if ((nonNullBitmask & 1 << idx) == 0) {
+ (a -> a.toAttribute.withNullability(true))
+ } else {
+ (a -> a.toAttribute)
+ }
+ }.toMap
val aggregations: Seq[NamedExpression] = x.aggregations.map {
// If an expression is an aggregate (contains a AggregateExpression) then we dont change
--
GitLab