Skip to content
Snippets Groups Projects
Commit a1d9138a authored by gatorsmile's avatar gatorsmile Committed by Wenchen Fan
Browse files

[SPARK-17680][SQL][TEST] Added a Testcase for Verifying Unicode Character...

[SPARK-17680][SQL][TEST] Added a Testcase for Verifying Unicode Character Support for Column Names and Comments

### What changes were proposed in this pull request?

Spark SQL supports Unicode characters for column names when specified within backticks(`). When the Hive support is enabled, the version of the Hive metastore must be higher than 0.12,  See the JIRA: https://issues.apache.org/jira/browse/HIVE-6013 Hive metastore supports Unicode characters for column names since 0.13.

In Spark SQL, table comments, and view comments always allow Unicode characters without backticks.

BTW, a separate PR has been submitted for database and table name validation because we do not support Unicode characters in these two cases.
### How was this patch tested?

N/A

Author: gatorsmile <gatorsmile@gmail.com>

Closes #15255 from gatorsmile/unicodeSupport.
parent bc09a2b8
No related branches found
No related tags found
No related merge requests found
......@@ -147,6 +147,51 @@ class HiveDDLSuite
}
}
test("create Hive-serde table and view with unicode columns and comment") {
val catalog = spark.sessionState.catalog
val tabName = "tab1"
val viewName = "view1"
// scalastyle:off
// non ascii characters are not allowed in the source code, so we disable the scalastyle.
val colName1 = "和"
val colName2 = "尼"
val comment = "庙"
// scalastyle:on
withTable(tabName) {
sql(s"""
|CREATE TABLE $tabName(`$colName1` int COMMENT '$comment')
|COMMENT '$comment'
|PARTITIONED BY (`$colName2` int)
""".stripMargin)
sql(s"INSERT OVERWRITE TABLE $tabName partition (`$colName2`=2) SELECT 1")
withView(viewName) {
sql(
s"""
|CREATE VIEW $viewName(`$colName1` COMMENT '$comment', `$colName2`)
|COMMENT '$comment'
|AS SELECT `$colName1`, `$colName2` FROM $tabName
""".stripMargin)
val tableMetadata = catalog.getTableMetadata(TableIdentifier(tabName, Some("default")))
val viewMetadata = catalog.getTableMetadata(TableIdentifier(viewName, Some("default")))
assert(tableMetadata.comment == Option(comment))
assert(viewMetadata.comment == Option(comment))
assert(tableMetadata.schema.fields.length == 2 && viewMetadata.schema.fields.length == 2)
val column1InTable = tableMetadata.schema.fields.head
val column1InView = viewMetadata.schema.fields.head
assert(column1InTable.name == colName1 && column1InView.name == colName1)
assert(column1InTable.getComment() == Option(comment))
assert(column1InView.getComment() == Option(comment))
assert(tableMetadata.schema.fields(1).name == colName2 &&
viewMetadata.schema.fields(1).name == colName2)
checkAnswer(sql(s"SELECT `$colName1`, `$colName2` FROM $tabName"), Row(1, 2) :: Nil)
checkAnswer(sql(s"SELECT `$colName1`, `$colName2` FROM $viewName"), Row(1, 2) :: Nil)
}
}
}
test("create table: partition column names exist in table definition") {
val e = intercept[AnalysisException] {
sql("CREATE TABLE tbl(a int) PARTITIONED BY (a string)")
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment