Skip to content
Snippets Groups Projects
Commit 6ac1c3a0 authored by Cheng Lian's avatar Cheng Lian Committed by Andrew Or
Browse files

[SPARK-14346][SQL] Lists unsupported Hive features in SHOW CREATE TABLE output

## What changes were proposed in this pull request?

This PR is a follow-up of #13079. It replaces `hasUnsupportedFeatures: Boolean` in `CatalogTable` with `unsupportedFeatures: Seq[String]`, which contains unsupported Hive features of the underlying Hive table. In this way, we can accurately report all unsupported Hive features in the exception message.

## How was this patch tested?

Updated existing test case to check exception message.

Author: Cheng Lian <lian@databricks.com>

Closes #13173 from liancheng/spark-14346-follow-up.
parent e71cd96b
No related branches found
No related tags found
No related merge requests found
......@@ -80,11 +80,8 @@ case class CatalogTablePartition(
* Note that Hive's metastore also tracks skewed columns. We should consider adding that in the
* future once we have a better understanding of how we want to handle skewed columns.
*
* @param hasUnsupportedFeatures is used to indicate whether all table metadata entries retrieved
* from the concrete underlying external catalog (e.g. Hive metastore) are supported by
* Spark SQL. For example, if the underlying Hive table has skewed columns, this information
* can't be mapped to [[CatalogTable]] since Spark SQL doesn't handle skewed columns for now.
* In this case `hasUnsupportedFeatures` is set to true. By default, it is false.
* @param unsupportedFeatures is a list of string descriptions of features that are used by the
* underlying table but not supported by Spark SQL yet.
*/
case class CatalogTable(
identifier: TableIdentifier,
......@@ -102,7 +99,7 @@ case class CatalogTable(
viewOriginalText: Option[String] = None,
viewText: Option[String] = None,
comment: Option[String] = None,
hasUnsupportedFeatures: Boolean = false) {
unsupportedFeatures: Seq[String] = Seq.empty) {
// Verify that the provided columns are part of the schema
private val colNames = schema.map(_.name).toSet
......
......@@ -633,16 +633,16 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman
}
private def showCreateHiveTable(metadata: CatalogTable): String = {
def reportUnsupportedError(): Unit = {
throw new UnsupportedOperationException(
def reportUnsupportedError(features: Seq[String]): Unit = {
throw new AnalysisException(
s"Failed to execute SHOW CREATE TABLE against table ${metadata.identifier.quotedString}, " +
"because it contains table structure(s) (e.g. skewed columns) that Spark SQL doesn't " +
"support yet."
"which is created by Hive and uses the following unsupported feature(s)\n" +
features.map(" - " + _).mkString("\n")
)
}
if (metadata.hasUnsupportedFeatures) {
reportUnsupportedError()
if (metadata.unsupportedFeatures.nonEmpty) {
reportUnsupportedError(metadata.unsupportedFeatures)
}
val builder = StringBuilder.newBuilder
......@@ -651,7 +651,7 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman
case EXTERNAL => " EXTERNAL TABLE"
case VIEW => " VIEW"
case MANAGED => " TABLE"
case INDEX => reportUnsupportedError()
case INDEX => reportUnsupportedError(Seq("index table"))
}
builder ++= s"CREATE$tableTypeString ${table.quotedString}"
......
......@@ -337,10 +337,19 @@ private[hive] class HiveClientImpl(
val schema = h.getCols.asScala.map(fromHiveColumn) ++ partCols
// Skew spec, storage handler, and bucketing info can't be mapped to CatalogTable (yet)
val hasUnsupportedFeatures =
!h.getSkewedColNames.isEmpty ||
h.getStorageHandler != null ||
!h.getBucketCols.isEmpty
val unsupportedFeatures = ArrayBuffer.empty[String]
if (!h.getSkewedColNames.isEmpty) {
unsupportedFeatures += "skewed columns"
}
if (h.getStorageHandler != null) {
unsupportedFeatures += "storage handler"
}
if (!h.getBucketCols.isEmpty) {
unsupportedFeatures += "bucketing"
}
CatalogTable(
identifier = TableIdentifier(h.getTableName, Option(h.getDbName)),
......@@ -369,7 +378,7 @@ private[hive] class HiveClientImpl(
properties = h.getParameters.asScala.toMap,
viewOriginalText = Option(h.getViewOriginalText),
viewText = Option(h.getViewExpandedText),
hasUnsupportedFeatures = hasUnsupportedFeatures)
unsupportedFeatures = unsupportedFeatures)
}
}
......
......@@ -17,7 +17,7 @@
package org.apache.spark.sql.hive
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.{AnalysisException, QueryTest}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.catalog.CatalogTable
import org.apache.spark.sql.hive.test.TestHiveSingleton
......@@ -247,7 +247,7 @@ class ShowCreateTableSuite extends QueryTest with SQLTestUtils with TestHiveSing
}
}
test("hive bucketing not supported") {
test("hive bucketing is not supported") {
withTable("t1") {
createRawHiveTable(
s"""CREATE TABLE t1 (a INT, b STRING)
......@@ -257,9 +257,11 @@ class ShowCreateTableSuite extends QueryTest with SQLTestUtils with TestHiveSing
""".stripMargin
)
intercept[UnsupportedOperationException] {
val cause = intercept[AnalysisException] {
sql("SHOW CREATE TABLE t1")
}
assert(cause.getMessage.contains(" - bucketing"))
}
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment