Skip to content
Snippets Groups Projects
Commit 28710b42 authored by Daoyuan Wang's avatar Daoyuan Wang Committed by Reynold Xin
Browse files

[SPARK-16415][SQL] fix catalog string error

## What changes were proposed in this pull request?

In #13537 we truncate `simpleString` if it is a long `StructType`. But sometimes we need `catalogString` to reconstruct `TypeInfo`, for example in description of [SPARK-16415 ](https://issues.apache.org/jira/browse/SPARK-16415). So we need to keep the implementation of `catalogString` not affected by our truncate.

## How was this patch tested?

added a test case.

Author: Daoyuan Wang <daoyuan.wang@intel.com>

Closes #14089 from adrian-wang/catalogstring.
parent 0f7175de
No related branches found
No related tags found
No related merge requests found
......@@ -333,6 +333,12 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
Utils.truncatedString(fieldTypes, "struct<", ",", ">")
}
override def catalogString: String = {
// in catalogString, we should not truncate
val fieldTypes = fields.map(field => s"${field.name}:${field.dataType.catalogString}")
s"struct<${fieldTypes.mkString(",")}>"
}
override def sql: String = {
val fieldTypes = fields.map(f => s"${quoteIdentifier(f.name)}: ${f.dataType.sql}")
s"STRUCT<${fieldTypes.mkString(", ")}>"
......
......@@ -26,15 +26,15 @@ import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.{ExamplePointUDT, SQLTestUtils}
import org.apache.spark.sql.types.{DecimalType, StringType, StructType}
import org.apache.spark.sql.types.{DecimalType, StringType, StructField, StructType}
class HiveMetastoreCatalogSuite extends TestHiveSingleton {
import spark.implicits._
test("struct field should accept underscore in sub-column name") {
val hiveTypeStr = "struct<a: int, b_1: string, c: string>"
val dateType = CatalystSqlParser.parseDataType(hiveTypeStr)
assert(dateType.isInstanceOf[StructType])
val dataType = CatalystSqlParser.parseDataType(hiveTypeStr)
assert(dataType.isInstanceOf[StructType])
}
test("udt to metastore type conversion") {
......@@ -49,6 +49,14 @@ class HiveMetastoreCatalogSuite extends TestHiveSingleton {
logInfo(df.queryExecution.toString)
df.as('a).join(df.as('b), $"a.key" === $"b.key")
}
test("should not truncate struct type catalog string") {
def field(n: Int): StructField = {
StructField("col" + n, StringType)
}
val dataType = StructType((1 to 100).map(field))
assert(CatalystSqlParser.parseDataType(dataType.catalogString) == dataType)
}
}
class DataSourceWithHiveMetastoreCatalogSuite
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment