From 259860d23d1740954b739b639c5bdc3ede65ed25 Mon Sep 17 00:00:00 2001
From: ptkool <michael.styles@shopify.com>
Date: Mon, 1 May 2017 17:05:35 -0700
Subject: [PATCH] [SPARK-20463] Add support for IS [NOT] DISTINCT FROM.
## What changes were proposed in this pull request?
Add support for the SQL standard distinct predicate to SPARK SQL.
```
<expression> IS [NOT] DISTINCT FROM <expression>
```
## How was this patch tested?
Tested using unit tests, integration tests, manual tests.
Author: ptkool <michael.styles@shopify.com>
Closes #17764 from ptkool/is_not_distinct_from.
---
.../antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 1 +
.../org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 5 +++++
.../spark/sql/catalyst/parser/ExpressionParserSuite.scala | 5 +++++
3 files changed, 11 insertions(+)
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 1ecb3d1958..14c511f670 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -534,6 +534,7 @@ predicate
| NOT? kind=IN '(' query ')'
| NOT? kind=(RLIKE | LIKE) pattern=valueExpression
| IS NOT? kind=NULL
+ | IS NOT? kind=DISTINCT FROM right=valueExpression
;
valueExpression
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index a48a693a95..d2a9b4a9a9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -935,6 +935,7 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging {
* - (NOT) LIKE
* - (NOT) RLIKE
* - IS (NOT) NULL.
+ * - IS (NOT) DISTINCT FROM
*/
private def withPredicate(e: Expression, ctx: PredicateContext): Expression = withOrigin(ctx) {
// Invert a predicate if it has a valid NOT clause.
@@ -962,6 +963,10 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging {
IsNotNull(e)
case SqlBaseParser.NULL =>
IsNull(e)
+ case SqlBaseParser.DISTINCT if ctx.NOT != null =>
+ EqualNullSafe(e, expression(ctx.right))
+ case SqlBaseParser.DISTINCT =>
+ Not(EqualNullSafe(e, expression(ctx.right)))
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index e7f3b64a71..eb68eb9851 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -167,6 +167,11 @@ class ExpressionParserSuite extends PlanTest {
assertEqual("a = b is not null", ('a === 'b).isNotNull)
}
+ test("is distinct expressions") {
+ assertEqual("a is distinct from b", !('a <=> 'b))
+ assertEqual("a is not distinct from b", 'a <=> 'b)
+ }
+
test("binary arithmetic expressions") {
// Simple operations
assertEqual("a * b", 'a * 'b)
--
GitLab