From aba9492d25e285d00033c408e9bfdd543ee12f72 Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Fri, 1 Sep 2017 13:21:06 -0700
Subject: [PATCH] [SPARK-21895][SQL] Support changing database in HiveClient
## What changes were proposed in this pull request?
Supporting moving tables across different database in HiveClient `alterTable`
## How was this patch tested?
N/A
Author: gatorsmile <gatorsmile@gmail.com>
Closes #19104 from gatorsmile/alterTable.
---
.../spark/sql/hive/HiveExternalCatalog.scala | 2 +-
.../spark/sql/hive/client/HiveClient.scala | 11 +++--
.../sql/hive/client/HiveClientImpl.scala | 7 +++-
.../spark/sql/hive/client/VersionsSuite.scala | 41 ++++++++++++++++++-
4 files changed, 53 insertions(+), 8 deletions(-)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index 34af37ce11..96dc983b0b 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -512,7 +512,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
identifier = TableIdentifier(newName, Some(db)),
storage = storageWithNewPath)
- client.alterTable(oldName, newTable)
+ client.alterTable(db, oldName, newTable)
}
private def getLocationFromStorageProps(table: CatalogTable): Option[String] = {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
index 8cff0ca096..ee3eb2ee8a 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
@@ -90,10 +90,15 @@ private[hive] trait HiveClient {
def dropTable(dbName: String, tableName: String, ignoreIfNotExists: Boolean, purge: Boolean): Unit
/** Alter a table whose name matches the one specified in `table`, assuming it exists. */
- final def alterTable(table: CatalogTable): Unit = alterTable(table.identifier.table, table)
+ final def alterTable(table: CatalogTable): Unit = {
+ alterTable(table.database, table.identifier.table, table)
+ }
- /** Updates the given table with new metadata, optionally renaming the table. */
- def alterTable(tableName: String, table: CatalogTable): Unit
+ /**
+ * Updates the given table with new metadata, optionally renaming the table or
+ * moving across different database.
+ */
+ def alterTable(dbName: String, tableName: String, table: CatalogTable): Unit
/** Creates a new database with the given name. */
def createDatabase(database: CatalogDatabase, ignoreIfExists: Boolean): Unit
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index 7c0b9bf19b..69dac7b062 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -495,7 +495,10 @@ private[hive] class HiveClientImpl(
shim.dropTable(client, dbName, tableName, true, ignoreIfNotExists, purge)
}
- override def alterTable(tableName: String, table: CatalogTable): Unit = withHiveState {
+ override def alterTable(
+ dbName: String,
+ tableName: String,
+ table: CatalogTable): Unit = withHiveState {
// getTableOption removes all the Hive-specific properties. Here, we fill them back to ensure
// these properties are still available to the others that share the same Hive metastore.
// If users explicitly alter these Hive-specific properties through ALTER TABLE DDL, we respect
@@ -503,7 +506,7 @@ private[hive] class HiveClientImpl(
val hiveTable = toHiveTable(
table.copy(properties = table.ignoredProperties ++ table.properties), Some(userName))
// Do not use `table.qualifiedName` here because this may be a rename
- val qualifiedTableName = s"${table.database}.$tableName"
+ val qualifiedTableName = s"$dbName.$tableName"
shim.alterTable(client, qualifiedTableName, hiveTable)
}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index cbbe869403..1d9c8da996 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -232,12 +232,49 @@ class VersionsSuite extends SparkFunSuite with Logging {
assert(client.getTable("default", "src").properties.contains("changed"))
}
- test(s"$version: alterTable(tableName: String, table: CatalogTable)") {
+ test(s"$version: alterTable(dbName: String, tableName: String, table: CatalogTable)") {
val newTable = client.getTable("default", "src").copy(properties = Map("changedAgain" -> ""))
- client.alterTable("src", newTable)
+ client.alterTable("default", "src", newTable)
assert(client.getTable("default", "src").properties.contains("changedAgain"))
}
+ test(s"$version: alterTable - rename") {
+ val newTable = client.getTable("default", "src")
+ .copy(identifier = TableIdentifier("tgt", database = Some("default")))
+ assert(!client.tableExists("default", "tgt"))
+
+ client.alterTable("default", "src", newTable)
+
+ assert(client.tableExists("default", "tgt"))
+ assert(!client.tableExists("default", "src"))
+ }
+
+ test(s"$version: alterTable - change database") {
+ val tempDB = CatalogDatabase(
+ "temporary", description = "test create", tempDatabasePath, Map())
+ client.createDatabase(tempDB, ignoreIfExists = true)
+
+ val newTable = client.getTable("default", "tgt")
+ .copy(identifier = TableIdentifier("tgt", database = Some("temporary")))
+ assert(!client.tableExists("temporary", "tgt"))
+
+ client.alterTable("default", "tgt", newTable)
+
+ assert(client.tableExists("temporary", "tgt"))
+ assert(!client.tableExists("default", "tgt"))
+ }
+
+ test(s"$version: alterTable - change database and table names") {
+ val newTable = client.getTable("temporary", "tgt")
+ .copy(identifier = TableIdentifier("src", database = Some("default")))
+ assert(!client.tableExists("default", "src"))
+
+ client.alterTable("temporary", "tgt", newTable)
+
+ assert(client.tableExists("default", "src"))
+ assert(!client.tableExists("temporary", "tgt"))
+ }
+
test(s"$version: listTables(database)") {
assert(client.listTables("default") === Seq("src", "temporary"))
}
--
GitLab