From ea465af12ddae424af9b4e742c3d5aed2a0bc8ec Mon Sep 17 00:00:00 2001
From: ravipesala <ravindra.pesala@huawei.com>
Date: Fri, 31 Oct 2014 11:33:20 -0700
Subject: [PATCH] [SPARK-4154][SQL] Query does not work if it has "not between
" in Spark SQL and HQL
if the query contains "not between" does not work like.
SELECT * FROM src where key not between 10 and 20'
Author: ravipesala <ravindra.pesala@huawei.com>
Closes #3017 from ravipesala/SPARK-4154 and squashes the following commits:
65fc89e [ravipesala] Handled admin comments
32e6d42 [ravipesala] 'not between' is not working
---
.../org/apache/spark/sql/catalyst/SqlParser.scala | 6 ++++--
.../scala/org/apache/spark/sql/SQLQuerySuite.scala | 5 +++++
.../scala/org/apache/spark/sql/hive/HiveQl.scala | 13 +++++++++----
.../spark/sql/hive/execution/SQLQuerySuite.scala | 5 +++++
4 files changed, 23 insertions(+), 6 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
index 942b843104..b1e7570f57 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
@@ -232,8 +232,10 @@ class SqlParser extends AbstractSparkSQLParser {
| termExpression ~ (">=" ~> termExpression) ^^ { case e1 ~ e2 => GreaterThanOrEqual(e1, e2) }
| termExpression ~ ("!=" ~> termExpression) ^^ { case e1 ~ e2 => Not(EqualTo(e1, e2)) }
| termExpression ~ ("<>" ~> termExpression) ^^ { case e1 ~ e2 => Not(EqualTo(e1, e2)) }
- | termExpression ~ (BETWEEN ~> termExpression) ~ (AND ~> termExpression) ^^ {
- case e ~ el ~ eu => And(GreaterThanOrEqual(e, el), LessThanOrEqual(e, eu))
+ | termExpression ~ NOT.? ~ (BETWEEN ~> termExpression) ~ (AND ~> termExpression) ^^ {
+ case e ~ not ~ el ~ eu =>
+ val betweenExpr: Expression = And(GreaterThanOrEqual(e, el), LessThanOrEqual(e, eu))
+ not.fold(betweenExpr)(f=> Not(betweenExpr))
}
| termExpression ~ (RLIKE ~> termExpression) ^^ { case e1 ~ e2 => RLike(e1, e2) }
| termExpression ~ (REGEXP ~> termExpression) ^^ { case e1 ~ e2 => RLike(e1, e2) }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 4c36ca0f41..4acd92d33d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -909,4 +909,9 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
""".stripMargin),
(1 to 100).map(i => Seq(i, i, i)))
}
+
+ test("SPARK-4154 Query does not work if it has 'not between' in Spark SQL and HQL") {
+ checkAnswer(sql("SELECT key FROM testData WHERE key not between 0 and 10 order by key"),
+ (11 to 100).map(i => Seq(i)))
+ }
}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
index 9d9d68affa..a3573e6502 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
@@ -985,15 +985,20 @@ private[hive] object HiveQl {
In(nodeToExpr(value), list.map(nodeToExpr))
case Token("TOK_FUNCTION",
Token(BETWEEN(), Nil) ::
- Token("KW_FALSE", Nil) ::
+ kw ::
target ::
minValue ::
maxValue :: Nil) =>
val targetExpression = nodeToExpr(target)
- And(
- GreaterThanOrEqual(targetExpression, nodeToExpr(minValue)),
- LessThanOrEqual(targetExpression, nodeToExpr(maxValue)))
+ val betweenExpr =
+ And(
+ GreaterThanOrEqual(targetExpression, nodeToExpr(minValue)),
+ LessThanOrEqual(targetExpression, nodeToExpr(maxValue)))
+ kw match {
+ case Token("KW_FALSE", Nil) => betweenExpr
+ case Token("KW_TRUE", Nil) => Not(betweenExpr)
+ }
/* Boolean Logic */
case Token(AND(), left :: right:: Nil) => And(nodeToExpr(left), nodeToExpr(right))
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 4f96a327ee..76a0ec01a6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -158,4 +158,9 @@ class SQLQuerySuite extends QueryTest {
sql("SELECT case when ~1=-2 then 1 else 0 end FROM src"),
sql("SELECT 1 FROM src").collect().toSeq)
}
+
+ test("SPARK-4154 Query does not work if it has 'not between' in Spark SQL and HQL") {
+ checkAnswer(sql("SELECT key FROM src WHERE key not between 0 and 10 order by key"),
+ sql("SELECT key FROM src WHERE key between 11 and 500 order by key").collect().toSeq)
+ }
}
--
GitLab