Skip to content
Snippets Groups Projects
Commit df766a47 authored by Xiao Li's avatar Xiao Li
Browse files

[SPARK-21089][SQL] Fix DESC EXTENDED/FORMATTED to Show Table Properties 

### What changes were proposed in this pull request?

Since both table properties and storage properties share the same key values, table properties are not shown in the output of DESC EXTENDED/FORMATTED when the storage properties are not empty.

This PR is to fix the above issue by renaming them to different keys.

### How was this patch tested?
Added test cases.

Author: Xiao Li <gatorsmile@gmail.com>

Closes #18294 from gatorsmile/tableProperties.
parent 0c88e8d3
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+ 2
− 2
View file @ df766a47
......@@ -75,7 +75,7 @@ case class CatalogStorageFormat(
CatalogUtils.maskCredentials(properties) match {
case props if props.isEmpty => // No-op
case props =>
map.put("Properties", props.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]"))
map.put("Storage Properties", props.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]"))
}
map
}
......@@ -316,7 +316,7 @@ case class CatalogTable(
}
}
if (properties.nonEmpty) map.put("Properties", tableProperties)
if (properties.nonEmpty) map.put("Table Properties", tableProperties)
stats.foreach(s => map.put("Statistics", s.simpleString))
map ++= storage.toLinkedHashMap
if (tracksPartitionsInCatalog) map.put("Partition Provider", "Catalog")
......
sql/core/src/test/resources/sql-tests/inputs/describe.sql
+ 3
− 0
View file @ df766a47
CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet
OPTIONS (a '1', b '2')
PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
COMMENT 'table_comment';
......@@ -13,6 +14,8 @@ CREATE TEMPORARY VIEW temp_Data_Source_View
CREATE VIEW v AS SELECT * FROM t;
ALTER TABLE t SET TBLPROPERTIES (e = '3');
ALTER TABLE t ADD PARTITION (c='Us', d=1);
DESCRIBE t;
......
sql/core/src/test/resources/sql-tests/results/describe-table-after-alter-table.sql.out
+ 1
− 1
View file @ df766a47
......@@ -57,7 +57,7 @@ Last Access [not included in comparison]
Type MANAGED
Provider parquet
Comment modified comment
Properties [type=parquet]
Table Properties [type=parquet]
Location [not included in comparison]sql/core/spark-warehouse/table_with_comment
......
sql/core/src/test/resources/sql-tests/results/describe.sql.out
+ 99
− 84
View file @ df766a47
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 31
-- Number of queries: 32
-- !query 0
CREATE TABLE t (a STRING, b INT, c STRING, d STRING) USING parquet
OPTIONS (a '1', b '2')
PARTITIONED BY (c, d) CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS
COMMENT 'table_comment'
-- !query 0 schema
......@@ -42,7 +43,7 @@ struct<>
-- !query 4
ALTER TABLE t ADD PARTITION (c='Us', d=1)
ALTER TABLE t SET TBLPROPERTIES (e = '3')
-- !query 4 schema
struct<>
-- !query 4 output
......@@ -50,10 +51,18 @@ struct<>
-- !query 5
DESCRIBE t
ALTER TABLE t ADD PARTITION (c='Us', d=1)
-- !query 5 schema
struct<col_name:string,data_type:string,comment:string>
struct<>
-- !query 5 output
-- !query 6
DESCRIBE t
-- !query 6 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 6 output
a string
b int
c string
......@@ -64,11 +73,11 @@ c string
d string
-- !query 6
-- !query 7
DESC default.t
-- !query 6 schema
-- !query 7 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 6 output
-- !query 7 output
a string
b int
c string
......@@ -79,11 +88,11 @@ c string
d string
-- !query 7
-- !query 8
DESC TABLE t
-- !query 7 schema
-- !query 8 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 7 output
-- !query 8 output
a string
b int
c string
......@@ -94,11 +103,11 @@ c string
d string
-- !query 8
-- !query 9
DESC FORMATTED t
-- !query 8 schema
-- !query 9 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 8 output
-- !query 9 output
a string
b int
c string
......@@ -118,16 +127,17 @@ Provider parquet
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
Table Properties [e=3]
Location [not included in comparison]sql/core/spark-warehouse/t
Storage Properties [a=1, b=2]
Partition Provider Catalog
-- !query 9
-- !query 10
DESC EXTENDED t
-- !query 9 schema
-- !query 10 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 9 output
-- !query 10 output
a string
b int
c string
......@@ -147,16 +157,17 @@ Provider parquet
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Comment table_comment
Table Properties [e=3]
Location [not included in comparison]sql/core/spark-warehouse/t
Storage Properties [a=1, b=2]
Partition Provider Catalog
-- !query 10
-- !query 11
DESC t PARTITION (c='Us', d=1)
-- !query 10 schema
-- !query 11 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 10 output
-- !query 11 output
a string
b int
c string
......@@ -167,11 +178,11 @@ c string
d string
-- !query 11
-- !query 12
DESC EXTENDED t PARTITION (c='Us', d=1)
-- !query 11 schema
-- !query 12 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 11 output
-- !query 12 output
a string
b int
c string
......@@ -186,19 +197,21 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]sql/core/spark-warehouse/t/c=Us/d=1
Storage Properties [a=1, b=2]
# Storage Information
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]sql/core/spark-warehouse/t
Storage Properties [a=1, b=2]
-- !query 12
-- !query 13
DESC FORMATTED t PARTITION (c='Us', d=1)
-- !query 12 schema
-- !query 13 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 12 output
-- !query 13 output
a string
b int
c string
......@@ -213,39 +226,41 @@ Database default
Table t
Partition Values [c=Us, d=1]
Location [not included in comparison]sql/core/spark-warehouse/t/c=Us/d=1
Storage Properties [a=1, b=2]
# Storage Information
Num Buckets 2
Bucket Columns [`a`]
Sort Columns [`b`]
Location [not included in comparison]sql/core/spark-warehouse/t
Location [not included in comparison]sql/core/spark-warehouse/t
Storage Properties [a=1, b=2]
-- !query 13
-- !query 14
DESC t PARTITION (c='Us', d=2)
-- !query 13 schema
-- !query 14 schema
struct<>
-- !query 13 output
-- !query 14 output
org.apache.spark.sql.catalyst.analysis.NoSuchPartitionException
Partition not found in table 't' database 'default':
c -> Us
d -> 2;
-- !query 14
-- !query 15
DESC t PARTITION (c='Us')
-- !query 14 schema
-- !query 15 schema
struct<>
-- !query 14 output
-- !query 15 output
org.apache.spark.sql.AnalysisException
Partition spec is invalid. The spec (c) must match the partition spec (c, d) defined in table '`default`.`t`';
-- !query 15
-- !query 16
DESC t PARTITION (c='Us', d)
-- !query 15 schema
-- !query 16 schema
struct<>
-- !query 15 output
-- !query 16 output
org.apache.spark.sql.catalyst.parser.ParseException
PARTITION specification is incomplete: `d`(line 1, pos 0)
......@@ -255,19 +270,8 @@ DESC t PARTITION (c='Us', d)
^^^
-- !query 16
DESC temp_v
-- !query 16 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 16 output
a string
b int
c string
d string
-- !query 17
DESC TABLE temp_v
DESC temp_v
-- !query 17 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 17 output
......@@ -278,7 +282,7 @@ d string
-- !query 18
DESC FORMATTED temp_v
DESC TABLE temp_v
-- !query 18 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 18 output
......@@ -289,7 +293,7 @@ d string
-- !query 19
DESC EXTENDED temp_v
DESC FORMATTED temp_v
-- !query 19 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 19 output
......@@ -300,10 +304,21 @@ d string
-- !query 20
DESC temp_Data_Source_View
DESC EXTENDED temp_v
-- !query 20 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 20 output
a string
b int
c string
d string
-- !query 21
DESC temp_Data_Source_View
-- !query 21 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 21 output
intType int test comment test1
stringType string
dateType date
......@@ -322,42 +337,42 @@ arrayType array<string>
structType struct<f1:string,f2:int>
-- !query 21
-- !query 22
DESC temp_v PARTITION (c='Us', d=1)
-- !query 21 schema
-- !query 22 schema
struct<>
-- !query 21 output
-- !query 22 output
org.apache.spark.sql.AnalysisException
DESC PARTITION is not allowed on a temporary view: temp_v;
-- !query 22
-- !query 23
DESC v
-- !query 22 schema
-- !query 23 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 22 output
-- !query 23 output
a string
b int
c string
d string
-- !query 23
-- !query 24
DESC TABLE v
-- !query 23 schema
-- !query 24 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 23 output
-- !query 24 output
a string
b int
c string
d string
-- !query 24
-- !query 25
DESC FORMATTED v
-- !query 24 schema
-- !query 25 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 24 output
-- !query 25 output
a string
b int
c string
......@@ -372,14 +387,14 @@ Type VIEW
View Text SELECT * FROM t
View Default Database default
View Query Output Columns [a, b, c, d]
Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c]
Table Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c]
-- !query 25
-- !query 26
DESC EXTENDED v
-- !query 25 schema
-- !query 26 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 25 output
-- !query 26 output
a string
b int
c string
......@@ -394,28 +409,20 @@ Type VIEW
View Text SELECT * FROM t
View Default Database default
View Query Output Columns [a, b, c, d]
Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c]
-- !query 26
DESC v PARTITION (c='Us', d=1)
-- !query 26 schema
struct<>
-- !query 26 output
org.apache.spark.sql.AnalysisException
DESC PARTITION is not allowed on a view: v;
Table Properties [view.query.out.col.3=d, view.query.out.col.0=a, view.query.out.numCols=4, view.default.database=default, view.query.out.col.1=b, view.query.out.col.2=c]
-- !query 27
DROP TABLE t
DESC v PARTITION (c='Us', d=1)
-- !query 27 schema
struct<>
-- !query 27 output
org.apache.spark.sql.AnalysisException
DESC PARTITION is not allowed on a view: v;
-- !query 28
DROP VIEW temp_v
DROP TABLE t
-- !query 28 schema
struct<>
-- !query 28 output
......@@ -423,7 +430,7 @@ struct<>
-- !query 29
DROP VIEW temp_Data_Source_View
DROP VIEW temp_v
-- !query 29 schema
struct<>
-- !query 29 output
......@@ -431,8 +438,16 @@ struct<>
-- !query 30
DROP VIEW v
DROP VIEW temp_Data_Source_View
-- !query 30 schema
struct<>
-- !query 30 output
-- !query 31
DROP VIEW v
-- !query 31 schema
struct<>
-- !query 31 output
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment