Skip to content
Snippets Groups Projects
Commit 41a7dbdd authored by Dongjoon Hyun's avatar Dongjoon Hyun Committed by Herman van Hovell
Browse files

[SPARK-10601][SQL] Support `MINUS` set operator

## What changes were proposed in this pull request?

This PR adds `MINUS` set operator which is equivalent `EXCEPT DISTINCT`. This will slightly improve the compatibility with Oracle.

## How was this patch tested?

Pass the Jenkins with newly added testcases.

Author: Dongjoon Hyun <dongjoon@apache.org>

Closes #14570 from dongjoon-hyun/SPARK-10601.
parent bdd53716
No related branches found
No related tags found
No related merge requests found
...@@ -313,7 +313,7 @@ multiInsertQueryBody ...@@ -313,7 +313,7 @@ multiInsertQueryBody
queryTerm queryTerm
: queryPrimary #queryTermDefault : queryPrimary #queryTermDefault
| left=queryTerm operator=(INTERSECT | UNION | EXCEPT) setQuantifier? right=queryTerm #setOperation | left=queryTerm operator=(INTERSECT | UNION | EXCEPT | SETMINUS) setQuantifier? right=queryTerm #setOperation
; ;
queryPrimary queryPrimary
...@@ -611,7 +611,7 @@ qualifiedName ...@@ -611,7 +611,7 @@ qualifiedName
identifier identifier
: strictIdentifier : strictIdentifier
| ANTI | FULL | INNER | LEFT | SEMI | RIGHT | NATURAL | JOIN | CROSS | ON | ANTI | FULL | INNER | LEFT | SEMI | RIGHT | NATURAL | JOIN | CROSS | ON
| UNION | INTERSECT | EXCEPT | UNION | INTERSECT | EXCEPT | SETMINUS
; ;
strictIdentifier strictIdentifier
...@@ -751,6 +751,7 @@ FUNCTIONS: 'FUNCTIONS'; ...@@ -751,6 +751,7 @@ FUNCTIONS: 'FUNCTIONS';
DROP: 'DROP'; DROP: 'DROP';
UNION: 'UNION'; UNION: 'UNION';
EXCEPT: 'EXCEPT'; EXCEPT: 'EXCEPT';
SETMINUS: 'MINUS';
INTERSECT: 'INTERSECT'; INTERSECT: 'INTERSECT';
TO: 'TO'; TO: 'TO';
TABLESAMPLE: 'TABLESAMPLE'; TABLESAMPLE: 'TABLESAMPLE';
......
...@@ -410,6 +410,7 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging { ...@@ -410,6 +410,7 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging {
* - UNION [DISTINCT] * - UNION [DISTINCT]
* - UNION ALL * - UNION ALL
* - EXCEPT [DISTINCT] * - EXCEPT [DISTINCT]
* - MINUS [DISTINCT]
* - INTERSECT [DISTINCT] * - INTERSECT [DISTINCT]
*/ */
override def visitSetOperation(ctx: SetOperationContext): LogicalPlan = withOrigin(ctx) { override def visitSetOperation(ctx: SetOperationContext): LogicalPlan = withOrigin(ctx) {
...@@ -429,6 +430,10 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging { ...@@ -429,6 +430,10 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging {
throw new ParseException("EXCEPT ALL is not supported.", ctx) throw new ParseException("EXCEPT ALL is not supported.", ctx)
case SqlBaseParser.EXCEPT => case SqlBaseParser.EXCEPT =>
Except(left, right) Except(left, right)
case SqlBaseParser.SETMINUS if all =>
throw new ParseException("MINUS ALL is not supported.", ctx)
case SqlBaseParser.SETMINUS =>
Except(left, right)
} }
} }
......
...@@ -68,6 +68,9 @@ class PlanParserSuite extends PlanTest { ...@@ -68,6 +68,9 @@ class PlanParserSuite extends PlanTest {
assertEqual("select * from a except select * from b", a.except(b)) assertEqual("select * from a except select * from b", a.except(b))
intercept("select * from a except all select * from b", "EXCEPT ALL is not supported.") intercept("select * from a except all select * from b", "EXCEPT ALL is not supported.")
assertEqual("select * from a except distinct select * from b", a.except(b)) assertEqual("select * from a except distinct select * from b", a.except(b))
assertEqual("select * from a minus select * from b", a.except(b))
intercept("select * from a minus all select * from b", "MINUS ALL is not supported.")
assertEqual("select * from a minus distinct select * from b", a.except(b))
assertEqual("select * from a intersect select * from b", a.intersect(b)) assertEqual("select * from a intersect select * from b", a.intersect(b))
intercept("select * from a intersect all select * from b", "INTERSECT ALL is not supported.") intercept("select * from a intersect all select * from b", "INTERSECT ALL is not supported.")
assertEqual("select * from a intersect distinct select * from b", a.intersect(b)) assertEqual("select * from a intersect distinct select * from b", a.intersect(b))
......
...@@ -1103,6 +1103,16 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { ...@@ -1103,6 +1103,16 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
sql("SELECT * FROM upperCaseData EXCEPT SELECT * FROM upperCaseData"), Nil) sql("SELECT * FROM upperCaseData EXCEPT SELECT * FROM upperCaseData"), Nil)
} }
test("MINUS") {
checkAnswer(
sql("SELECT * FROM lowerCaseData MINUS SELECT * FROM upperCaseData"),
Row(1, "a") :: Row(2, "b") :: Row(3, "c") :: Row(4, "d") :: Nil)
checkAnswer(
sql("SELECT * FROM lowerCaseData MINUS SELECT * FROM lowerCaseData"), Nil)
checkAnswer(
sql("SELECT * FROM upperCaseData MINUS SELECT * FROM upperCaseData"), Nil)
}
test("INTERSECT") { test("INTERSECT") {
checkAnswer( checkAnswer(
sql("SELECT * FROM lowerCaseData INTERSECT SELECT * FROM lowerCaseData"), sql("SELECT * FROM lowerCaseData INTERSECT SELECT * FROM lowerCaseData"),
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment