diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
index 8c68d1e3a2379e91874ec41289f6ed620a015003..be2eddbb0e423761f36e2048d9df9bdd42d20c26 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
@@ -1010,7 +1010,9 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
.orElse(Some("org.apache.hadoop.mapred.TextInputFormat")),
outputFormat = defaultHiveSerde.flatMap(_.outputFormat)
.orElse(Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat")),
- serde = defaultHiveSerde.flatMap(_.serde),
+ // Note: Keep this unspecified because we use the presence of the serde to decide
+ // whether to convert a table created by CTAS to a datasource table.
+ serde = None,
compressed = false,
properties = Map())
}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala
index 81337493c7f287618e2f400e736d6a681a29fe0f..9ce3338647398dd5c95e138c539021c5e41996e2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala
@@ -30,12 +30,10 @@ import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.logical.{Generate, ScriptTransformation}
import org.apache.spark.sql.execution.command._
import org.apache.spark.sql.execution.datasources.CreateTable
-import org.apache.spark.sql.hive.test.{TestHive, TestHiveSingleton}
-import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.test.SQLTestUtils
+import org.apache.spark.sql.hive.test.TestHive
import org.apache.spark.sql.types.StructType
-class HiveDDLCommandSuite extends PlanTest with SQLTestUtils with TestHiveSingleton {
+class HiveDDLCommandSuite extends PlanTest {
val parser = TestHive.sessionState.sqlParser
private def extractTableDesc(sql: String): (CatalogTable, Boolean) = {
@@ -558,24 +556,4 @@ class HiveDDLCommandSuite extends PlanTest with SQLTestUtils with TestHiveSingle
assert(partition2.get.apply("c") == "1" && partition2.get.apply("d") == "2")
}
- test("Test the default fileformat for Hive-serde tables") {
- withSQLConf("hive.default.fileformat" -> "orc") {
- val (desc, exists) = extractTableDesc("CREATE TABLE IF NOT EXISTS fileformat_test (id int)")
- assert(exists)
- assert(desc.storage.inputFormat == Some("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"))
- assert(desc.storage.outputFormat == Some("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"))
- assert(desc.storage.serde == Some("org.apache.hadoop.hive.ql.io.orc.OrcSerde"))
- }
-
- withSQLConf("hive.default.fileformat" -> "parquet") {
- val (desc, exists) = extractTableDesc("CREATE TABLE IF NOT EXISTS fileformat_test (id int)")
- assert(exists)
- val input = desc.storage.inputFormat
- val output = desc.storage.outputFormat
- val serde = desc.storage.serde
- assert(input == Some("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"))
- assert(output == Some("org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"))
- assert(serde == Some("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"))
- }
- }
}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 5798f4722821680737178b39af0470219091bf48..6f2a16662bf101218372bbf1d61186a8e486c64e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -492,7 +492,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
def checkRelation(
tableName: String,
- isDataSourceTable: Boolean,
+ isDataSourceParquet: Boolean,
format: String,
userSpecifiedLocation: Option[String] = None): Unit = {
val relation = EliminateSubqueryAliases(
@@ -501,7 +501,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
sessionState.catalog.getTableMetadata(TableIdentifier(tableName))
relation match {
case LogicalRelation(r: HadoopFsRelation, _, _) =>
- if (!isDataSourceTable) {
+ if (!isDataSourceParquet) {
fail(
s"${classOf[MetastoreRelation].getCanonicalName} is expected, but found " +
s"${HadoopFsRelation.getClass.getCanonicalName}.")
@@ -514,7 +514,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
assert(catalogTable.provider.get === format)
case r: MetastoreRelation =>
- if (isDataSourceTable) {
+ if (isDataSourceParquet) {
fail(
s"${HadoopFsRelation.getClass.getCanonicalName} is expected, but found " +
s"${classOf[MetastoreRelation].getCanonicalName}.")
@@ -524,15 +524,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
assert(r.catalogTable.storage.locationUri.get === location)
case None => // OK.
}
- // Also make sure that the format and serde are as desired.
+ // Also make sure that the format is the desired format.
assert(catalogTable.storage.inputFormat.get.toLowerCase.contains(format))
- assert(catalogTable.storage.outputFormat.get.toLowerCase.contains(format))
- val serde = catalogTable.storage.serde.get
- format match {
- case "sequence" | "text" => assert(serde.contains("LazySimpleSerDe"))
- case "rcfile" => assert(serde.contains("LazyBinaryColumnarSerDe"))
- case _ => assert(serde.toLowerCase.contains(format))
- }
}
// When a user-specified location is defined, the table type needs to be EXTERNAL.
@@ -594,30 +587,6 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
}
- test("CTAS with default fileformat") {
- val table = "ctas1"
- val ctas = s"CREATE TABLE IF NOT EXISTS $table SELECT key k, value FROM src"
- withSQLConf(SQLConf.CONVERT_CTAS.key -> "true") {
- withSQLConf("hive.default.fileformat" -> "textfile") {
- withTable(table) {
- sql(ctas)
- // We should use parquet here as that is the default datasource fileformat. The default
- // datasource file format is controlled by `spark.sql.sources.default` configuration.
- // This testcase verifies that setting `hive.default.fileformat` has no impact on
- // the target table's fileformat in case of CTAS.
- assert(sessionState.conf.defaultDataSourceName === "parquet")
- checkRelation(table, isDataSourceTable = true, "parquet")
- }
- }
- withSQLConf("spark.sql.sources.default" -> "orc") {
- withTable(table) {
- sql(ctas)
- checkRelation(table, isDataSourceTable = true, "orc")
- }
- }
- }
- }
-
test("CTAS without serde with location") {
withSQLConf(SQLConf.CONVERT_CTAS.key -> "true") {
withTempDir { dir =>