From 8ae2da0b2551011e2f6cf02907a1e20c138a4b2f Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sun, 30 Oct 2016 23:24:30 +0100
Subject: [PATCH] [SPARK-18106][SQL] ANALYZE TABLE should raise a
ParseException for invalid option
## What changes were proposed in this pull request?
Currently, `ANALYZE TABLE` command accepts `identifier` for option `NOSCAN`. This PR raises a ParseException for unknown option.
**Before**
```scala
scala> sql("create table test(a int)")
res0: org.apache.spark.sql.DataFrame = []
scala> sql("analyze table test compute statistics blah")
res1: org.apache.spark.sql.DataFrame = []
```
**After**
```scala
scala> sql("create table test(a int)")
res0: org.apache.spark.sql.DataFrame = []
scala> sql("analyze table test compute statistics blah")
org.apache.spark.sql.catalyst.parser.ParseException:
Expected `NOSCAN` instead of `blah`(line 1, pos 0)
```
## How was this patch tested?
Pass the Jenkins test with a new test case.
Author: Dongjoon Hyun <dongjoon@apache.org>
Closes #15640 from dongjoon-hyun/SPARK-18106.
---
.../spark/sql/execution/SparkSqlParser.scala | 10 +++++++---
.../sql/execution/SparkSqlParserSuite.scala | 18 ++++++++++++++++--
2 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
index 1cc166d5a7..fe183d0097 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
@@ -98,9 +98,13 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
* }}}
*/
override def visitAnalyze(ctx: AnalyzeContext): LogicalPlan = withOrigin(ctx) {
- if (ctx.partitionSpec == null &&
- ctx.identifier != null &&
- ctx.identifier.getText.toLowerCase == "noscan") {
+ if (ctx.partitionSpec != null) {
+ logWarning(s"Partition specification is ignored: ${ctx.partitionSpec.getText}")
+ }
+ if (ctx.identifier != null) {
+ if (ctx.identifier.getText.toLowerCase != "noscan") {
+ throw new ParseException(s"Expected `NOSCAN` instead of `${ctx.identifier.getText}`", ctx)
+ }
AnalyzeTableCommand(visitTableIdentifier(ctx.tableIdentifier))
} else if (ctx.identifierSeq() == null) {
AnalyzeTableCommand(visitTableIdentifier(ctx.tableIdentifier), noscan = false)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
index 679150e9ae..797fe9ffa8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
@@ -23,8 +23,8 @@ import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat,
import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.execution.command.{DescribeFunctionCommand, DescribeTableCommand,
- ShowFunctionsCommand}
+import org.apache.spark.sql.execution.command.{AnalyzeTableCommand, DescribeFunctionCommand,
+ DescribeTableCommand, ShowFunctionsCommand}
import org.apache.spark.sql.execution.datasources.{CreateTable, CreateTempViewUsing}
import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType}
@@ -220,4 +220,18 @@ class SparkSqlParserSuite extends PlanTest {
intercept("explain describe tables x", "Unsupported SQL statement")
}
+
+ test("SPARK-18106 analyze table") {
+ assertEqual("analyze table t compute statistics",
+ AnalyzeTableCommand(TableIdentifier("t"), noscan = false))
+ assertEqual("analyze table t compute statistics noscan",
+ AnalyzeTableCommand(TableIdentifier("t"), noscan = true))
+ assertEqual("analyze table t partition (a) compute statistics noscan",
+ AnalyzeTableCommand(TableIdentifier("t"), noscan = true))
+
+ intercept("analyze table t compute statistics xxxx",
+ "Expected `NOSCAN` instead of `xxxx`")
+ intercept("analyze table t partition (a) compute statistics xxxx",
+ "Expected `NOSCAN` instead of `xxxx`")
+ }
}
--
GitLab