Skip to content
Snippets Groups Projects
Commit 6e95897e authored by Dongjoon Hyun's avatar Dongjoon Hyun Committed by Wenchen Fan
Browse files

[SPARK-20954][SQL] `DESCRIBE [EXTENDED]` result should be compatible with previous Spark 

## What changes were proposed in this pull request?

After [SPARK-20067](https://issues.apache.org/jira/browse/SPARK-20067), `DESCRIBE` and `DESCRIBE EXTENDED` shows the following result. This is incompatible with Spark 2.1.1. This PR removes the column header line in case of those command.

**MASTER** and **BRANCH-2.2**
```scala
scala> sql("desc t").show(false)
+----------+---------+-------+
|col_name  |data_type|comment|
+----------+---------+-------+
|# col_name|data_type|comment|
|a         |int      |null   |
+----------+---------+-------+
```

**SPARK 2.1.1** and **this PR**
```scala
scala> sql("desc t").show(false)
+--------+---------+-------+
|col_name|data_type|comment|
+--------+---------+-------+
|a       |int      |null   |
+--------+---------+-------+
```

## How was this patch tested?

Pass the Jenkins with the updated test suites.

Author: Dongjoon Hyun <dongjoon@apache.org>

Closes #18203 from dongjoon-hyun/SPARK-20954.
parent 1a527bde
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 12 additions and 38 deletions
sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+ 11
− 6
View file @ 6e95897e
......@@ -522,15 +522,15 @@ case class DescribeTableCommand(
throw new AnalysisException(
s"DESC PARTITION is not allowed on a temporary view: ${table.identifier}")
}
describeSchema(catalog.lookupRelation(table).schema, result)
describeSchema(catalog.lookupRelation(table).schema, result, header = false)
} else {
val metadata = catalog.getTableMetadata(table)
if (metadata.schema.isEmpty) {
// In older version(prior to 2.1) of Spark, the table schema can be empty and should be
// inferred at runtime. We should still support it.
describeSchema(sparkSession.table(metadata.identifier).schema, result)
describeSchema(sparkSession.table(metadata.identifier).schema, result, header = false)
} else {
describeSchema(metadata.schema, result)
describeSchema(metadata.schema, result, header = false)
}
describePartitionInfo(metadata, result)
......@@ -550,7 +550,7 @@ case class DescribeTableCommand(
private def describePartitionInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = {
if (table.partitionColumnNames.nonEmpty) {
append(buffer, "# Partition Information", "", "")
describeSchema(table.partitionSchema, buffer)
describeSchema(table.partitionSchema, buffer, header = true)
}
}
......@@ -601,8 +601,13 @@ case class DescribeTableCommand(
table.storage.toLinkedHashMap.foreach(s => append(buffer, s._1, s._2, ""))
}
private def describeSchema(schema: StructType, buffer: ArrayBuffer[Row]): Unit = {
append(buffer, s"# ${output.head.name}", output(1).name, output(2).name)
private def describeSchema(
schema: StructType,
buffer: ArrayBuffer[Row],
header: Boolean): Unit = {
if (header) {
append(buffer, s"# ${output.head.name}", output(1).name, output(2).name)
}
schema.foreach { column =>
append(buffer, column.name, column.dataType.simpleString, column.getComment().orNull)
}
......
sql/core/src/test/resources/sql-tests/results/change-column.sql.out
+ 0
− 9
View file @ 6e95897e
......@@ -15,7 +15,6 @@ DESC test_change
-- !query 1 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 1 output
# col_name data_type comment
a int
b string
c int
......@@ -35,7 +34,6 @@ DESC test_change
-- !query 3 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 3 output
# col_name data_type comment
a int
b string
c int
......@@ -55,7 +53,6 @@ DESC test_change
-- !query 5 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 5 output
# col_name data_type comment
a int
b string
c int
......@@ -94,7 +91,6 @@ DESC test_change
-- !query 8 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 8 output
# col_name data_type comment
a int
b string
c int
......@@ -129,7 +125,6 @@ DESC test_change
-- !query 12 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 12 output
# col_name data_type comment
a int this is column a
b string #*02?`
c int
......@@ -148,7 +143,6 @@ DESC test_change
-- !query 14 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 14 output
# col_name data_type comment
a int this is column a
b string #*02?`
c int
......@@ -168,7 +162,6 @@ DESC test_change
-- !query 16 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 16 output
# col_name data_type comment
a int this is column a
b string #*02?`
c int
......@@ -193,7 +186,6 @@ DESC test_change
-- !query 18 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 18 output
# col_name data_type comment
a int this is column a
b string #*02?`
c int
......@@ -237,7 +229,6 @@ DESC test_change
-- !query 23 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 23 output
# col_name data_type comment
a int this is column A
b string #*02?`
c int
......
sql/core/src/test/resources/sql-tests/results/describe-table-after-alter-table.sql.out
+ 0
− 5
View file @ 6e95897e
......@@ -15,7 +15,6 @@ DESC FORMATTED table_with_comment
-- !query 1 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 1 output
# col_name data_type comment
a string
b int
c string
......@@ -45,7 +44,6 @@ DESC FORMATTED table_with_comment
-- !query 3 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 3 output
# col_name data_type comment
a string
b int
c string
......@@ -84,7 +82,6 @@ DESC FORMATTED table_comment
-- !query 6 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 6 output
# col_name data_type comment
a string
b int
......@@ -111,7 +108,6 @@ DESC formatted table_comment
-- !query 8 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 8 output
# col_name data_type comment
a string
b int
......@@ -139,7 +135,6 @@ DESC FORMATTED table_comment
-- !query 10 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 10 output
# col_name data_type comment
a string
b int
......
sql/core/src/test/resources/sql-tests/results/describe.sql.out
+ 0
− 17
View file @ 6e95897e
......@@ -54,7 +54,6 @@ DESCRIBE t
-- !query 5 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 5 output
# col_name data_type comment
a string
b int
c string
......@@ -70,7 +69,6 @@ DESC default.t
-- !query 6 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 6 output
# col_name data_type comment
a string
b int
c string
......@@ -86,7 +84,6 @@ DESC TABLE t
-- !query 7 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 7 output
# col_name data_type comment
a string
b int
c string
......@@ -102,7 +99,6 @@ DESC FORMATTED t
-- !query 8 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 8 output
# col_name data_type comment
a string
b int
c string
......@@ -132,7 +128,6 @@ DESC EXTENDED t
-- !query 9 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 9 output
# col_name data_type comment
a string
b int
c string
......@@ -162,7 +157,6 @@ DESC t PARTITION (c='Us', d=1)
-- !query 10 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 10 output
# col_name data_type comment
a string
b int
c string
......@@ -178,7 +172,6 @@ DESC EXTENDED t PARTITION (c='Us', d=1)
-- !query 11 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 11 output
# col_name data_type comment
a string
b int
c string
......@@ -206,7 +199,6 @@ DESC FORMATTED t PARTITION (c='Us', d=1)
-- !query 12 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 12 output
# col_name data_type comment
a string
b int
c string
......@@ -268,7 +260,6 @@ DESC temp_v
-- !query 16 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 16 output
# col_name data_type comment
a string
b int
c string
......@@ -280,7 +271,6 @@ DESC TABLE temp_v
-- !query 17 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 17 output
# col_name data_type comment
a string
b int
c string
......@@ -292,7 +282,6 @@ DESC FORMATTED temp_v
-- !query 18 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 18 output
# col_name data_type comment
a string
b int
c string
......@@ -304,7 +293,6 @@ DESC EXTENDED temp_v
-- !query 19 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 19 output
# col_name data_type comment
a string
b int
c string
......@@ -316,7 +304,6 @@ DESC temp_Data_Source_View
-- !query 20 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 20 output
# col_name data_type comment
intType int test comment test1
stringType string
dateType date
......@@ -349,7 +336,6 @@ DESC v
-- !query 22 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 22 output
# col_name data_type comment
a string
b int
c string
......@@ -361,7 +347,6 @@ DESC TABLE v
-- !query 23 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 23 output
# col_name data_type comment
a string
b int
c string
......@@ -373,7 +358,6 @@ DESC FORMATTED v
-- !query 24 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 24 output
# col_name data_type comment
a string
b int
c string
......@@ -396,7 +380,6 @@ DESC EXTENDED v
-- !query 25 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 25 output
# col_name data_type comment
a string
b int
c string
......
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+ 1
− 1
View file @ 6e95897e
......@@ -806,7 +806,7 @@ class HiveDDLSuite
checkAnswer(
sql(s"DESC $tabName").select("col_name", "data_type", "comment"),
Row("# col_name", "data_type", "comment") :: Row("a", "int", "test") :: Nil
Row("a", "int", "test") :: Nil
)
}
}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment