Skip to content
Snippets Groups Projects
Commit 57064726 authored by gatorsmile's avatar gatorsmile Committed by Herman van Hovell
Browse files

[SPARK-15215][SQL] Fix Explain Parsing and Output

#### What changes were proposed in this pull request?
This PR is to address a few existing issues in `EXPLAIN`:
- The `EXPLAIN` options `LOGICAL | FORMATTED | EXTENDED | CODEGEN` should not be 0 or more match. It should 0 or one match. Parser does not allow users to use more than one option in a single command.
- The option `LOGICAL` is not supported. Issue an exception when users specify this option in the command.
- The output of `EXPLAIN ` contains a weird empty line when the output of analyzed plan is empty. We should remove it. For example:
  ```
  == Parsed Logical Plan ==
  CreateTable CatalogTable(`t`,CatalogTableType(MANAGED),CatalogStorageFormat(None,Some(org.apache.hadoop.mapred.TextInputFormat),Some(org.apache.hadoop.hive.ql.io.  HiveIgnoreKeyTextOutputFormat),None,false,Map()),List(CatalogColumn(col,int,true,None)),List(),List(),List(),-1,,1462725171656,-1,Map(),None,None,None), false

  == Analyzed Logical Plan ==

  CreateTable CatalogTable(`t`,CatalogTableType(MANAGED),CatalogStorageFormat(None,Some(org.apache.hadoop.mapred.TextInputFormat),Some(org.apache.hadoop.hive.ql.io.  HiveIgnoreKeyTextOutputFormat),None,false,Map()),List(CatalogColumn(col,int,true,None)),List(),List(),List(),-1,,1462725171656,-1,Map(),None,None,None), false

  == Optimized Logical Plan ==
  CreateTable CatalogTable(`t`,CatalogTableType(MANAGED),CatalogStorageFormat(None,Some(org.apache.hadoop.mapred.TextInputFormat),Some(org.apache.hadoop.hive.ql.io.  HiveIgnoreKeyTextOutputFormat),None,false,Map()),List(CatalogColumn(col,int,true,None)),List(),List(),List(),-1,,1462725171656,-1,Map(),None,None,None), false
  ...
  ```

#### How was this patch tested?
Added and modified a few test cases

Author: gatorsmile <gatorsmile@gmail.com>

Closes #12991 from gatorsmile/explainCreateTable.
parent f4537917
No related branches found
No related tags found
No related merge requests found
......@@ -92,7 +92,7 @@ statement
| CREATE TEMPORARY? FUNCTION qualifiedName AS className=STRING
(USING resource (',' resource)*)? #createFunction
| DROP TEMPORARY? FUNCTION (IF EXISTS)? qualifiedName #dropFunction
| EXPLAIN explainOption* statement #explain
| EXPLAIN (LOGICAL | FORMATTED | EXTENDED | CODEGEN)? statement #explain
| SHOW TABLES ((FROM | IN) db=identifier)?
(LIKE? pattern=STRING)? #showTables
| SHOW DATABASES (LIKE pattern=STRING)? #showDatabases
......@@ -587,11 +587,6 @@ frameBound
| expression boundType=(PRECEDING | FOLLOWING)
;
explainOption
: LOGICAL | FORMATTED | EXTENDED | CODEGEN
;
qualifiedName
: identifier ('.' identifier)*
;
......
......@@ -48,6 +48,11 @@ class PlanParserSuite extends PlanTest {
assertEqual("SELECT * FROM a", plan)
}
test("explain") {
intercept("EXPLAIN logical SELECT 1", "Unsupported SQL statement")
intercept("EXPLAIN formatted SELECT 1", "Unsupported SQL statement")
}
test("show functions") {
assertEqual("show functions", ShowFunctions(None, None))
assertEqual("show functions foo", ShowFunctions(None, Some("foo")))
......
......@@ -208,12 +208,13 @@ class QueryExecution(val sparkSession: SparkSession, val logical: LogicalPlan) {
override def toString: String = {
def output =
analyzed.output.map(o => s"${o.name}: ${o.dataType.simpleString}").mkString(", ")
val analyzedPlan =
Seq(stringOrError(output), stringOrError(analyzed)).filter(_.nonEmpty).mkString("\n")
s"""== Parsed Logical Plan ==
|${stringOrError(logical)}
|== Analyzed Logical Plan ==
|${stringOrError(output)}
|${stringOrError(analyzed)}
|$analyzedPlan
|== Optimized Logical Plan ==
|${stringOrError(optimizedPlan)}
|== Physical Plan ==
......
......@@ -212,18 +212,22 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
/**
* Create an [[ExplainCommand]] logical plan.
* The syntax of using this command in SQL is:
* {{{
* EXPLAIN (EXTENDED | CODEGEN) SELECT * FROM ...
* }}}
*/
override def visitExplain(ctx: ExplainContext): LogicalPlan = withOrigin(ctx) {
val options = ctx.explainOption.asScala
if (options.exists(_.FORMATTED != null)) {
if (ctx.FORMATTED != null) {
throw operationNotAllowed("EXPLAIN FORMATTED", ctx)
}
if (ctx.LOGICAL != null) {
throw operationNotAllowed("EXPLAIN LOGICAL", ctx)
}
// Create the explain comment.
val statement = plan(ctx.statement)
if (isExplainableStatement(statement)) {
ExplainCommand(statement, extended = options.exists(_.EXTENDED != null),
codegen = options.exists(_.CODEGEN != null))
ExplainCommand(statement, extended = ctx.EXTENDED != null, codegen = ctx.CODEGEN != null)
} else {
ExplainCommand(OneRowRelation)
}
......
......@@ -85,7 +85,7 @@ private[sql] case class ExecutedCommandExec(cmd: RunnableCommand) extends SparkP
* (but do NOT actually execute it).
*
* {{{
* EXPLAIN (EXTENDED|CODEGEN) SELECT * FROM ...
* EXPLAIN (EXTENDED | CODEGEN) SELECT * FROM ...
* }}}
*
* @param logicalPlan plan to explain
......
......@@ -18,6 +18,7 @@
package org.apache.spark.sql.hive.execution
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.test.SQLTestUtils
......@@ -86,7 +87,7 @@ class HiveExplainSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
|CREATE TABLE t1
|AS
|SELECT * FROM jt
""".stripMargin).collect().map(_.mkString).mkString
""".stripMargin).collect().map(_.mkString).mkString
val shouldContain =
"== Parsed Logical Plan ==" :: "== Analyzed Logical Plan ==" :: "Subquery" ::
......@@ -115,19 +116,8 @@ class HiveExplainSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
"== Physical Plan =="
)
checkKeywordsExist(sql("EXPLAIN EXTENDED CODEGEN SELECT 1"),
"WholeStageCodegen",
"Generated code:",
"/* 001 */ public Object generate(Object[] references) {",
"/* 002 */ return new GeneratedIterator(references);",
"/* 003 */ }"
)
checkKeywordsNotExist(sql("EXPLAIN EXTENDED CODEGEN SELECT 1"),
"== Parsed Logical Plan ==",
"== Analyzed Logical Plan ==",
"== Optimized Logical Plan ==",
"== Physical Plan =="
)
intercept[ParseException] {
sql("EXPLAIN EXTENDED CODEGEN SELECT 1")
}
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment