Skip to content
Snippets Groups Projects
Commit 0abbff28 authored by Cheng Hao's avatar Cheng Hao Committed by Michael Armbrust
Browse files

[SPARK-4825] [SQL] CTAS fails to resolve when created using saveAsTable

Fix bug when query like:
```
  test("save join to table") {
    val testData = sparkContext.parallelize(1 to 10).map(i => TestData(i, i.toString))
    sql("CREATE TABLE test1 (key INT, value STRING)")
    testData.insertInto("test1")
    sql("CREATE TABLE test2 (key INT, value STRING)")
    testData.insertInto("test2")
    testData.insertInto("test2")
    sql("SELECT COUNT(a.value) FROM test1 a JOIN test2 b ON a.key = b.key").saveAsTable("test")
    checkAnswer(
      table("test"),
      sql("SELECT COUNT(a.value) FROM test1 a JOIN test2 b ON a.key = b.key").collect().toSeq)
  }
```

Author: Cheng Hao <hao.cheng@intel.com>

Closes #3673 from chenghao-intel/spark_4825 and squashes the following commits:

e8cbd56 [Cheng Hao] alternate the pattern matching order for logical plan:CTAS
e004895 [Cheng Hao] fix bug
parent cbb634ae
No related branches found
No related tags found
No related merge requests found
...@@ -121,7 +121,7 @@ case class CreateTableAsSelect[T]( ...@@ -121,7 +121,7 @@ case class CreateTableAsSelect[T](
allowExisting: Boolean, allowExisting: Boolean,
desc: Option[T] = None) extends UnaryNode { desc: Option[T] = None) extends UnaryNode {
override def output = Seq.empty[Attribute] override def output = Seq.empty[Attribute]
override lazy val resolved = (databaseName != None && childrenResolved) override lazy val resolved = databaseName != None && childrenResolved
} }
case class WriteToFile( case class WriteToFile(
......
...@@ -261,6 +261,8 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with ...@@ -261,6 +261,8 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with
// Wait until children are resolved. // Wait until children are resolved.
case p: LogicalPlan if !p.childrenResolved => p case p: LogicalPlan if !p.childrenResolved => p
// TODO extra is in type of ASTNode which means the logical plan is not resolved
// Need to think about how to implement the CreateTableAsSelect.resolved
case CreateTableAsSelect(db, tableName, child, allowExisting, Some(extra: ASTNode)) => case CreateTableAsSelect(db, tableName, child, allowExisting, Some(extra: ASTNode)) =>
val (dbName, tblName) = processDatabaseAndTableName(db, tableName) val (dbName, tblName) = processDatabaseAndTableName(db, tableName)
val databaseName = dbName.getOrElse(hive.sessionState.getCurrentDatabase) val databaseName = dbName.getOrElse(hive.sessionState.getCurrentDatabase)
...@@ -285,6 +287,13 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with ...@@ -285,6 +287,13 @@ private[hive] class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with
} }
CreateTableAsSelect(Some(databaseName), tblName, child, allowExisting, desc) CreateTableAsSelect(Some(databaseName), tblName, child, allowExisting, desc)
case p: LogicalPlan if p.resolved => p
case p @ CreateTableAsSelect(db, tableName, child, allowExisting, None) =>
val (dbName, tblName) = processDatabaseAndTableName(db, tableName)
val databaseName = dbName.getOrElse(hive.sessionState.getCurrentDatabase)
CreateTableAsSelect(Some(databaseName), tblName, child, allowExisting, None)
} }
} }
......
...@@ -137,6 +137,19 @@ class SQLQuerySuite extends QueryTest { ...@@ -137,6 +137,19 @@ class SQLQuerySuite extends QueryTest {
sql("SELECT key, value FROM src ORDER BY key").collect().toSeq) sql("SELECT key, value FROM src ORDER BY key").collect().toSeq)
} }
test("SPARK-4825 save join to table") {
val testData = sparkContext.parallelize(1 to 10).map(i => TestData(i, i.toString))
sql("CREATE TABLE test1 (key INT, value STRING)")
testData.insertInto("test1")
sql("CREATE TABLE test2 (key INT, value STRING)")
testData.insertInto("test2")
testData.insertInto("test2")
sql("SELECT COUNT(a.value) FROM test1 a JOIN test2 b ON a.key = b.key").saveAsTable("test")
checkAnswer(
table("test"),
sql("SELECT COUNT(a.value) FROM test1 a JOIN test2 b ON a.key = b.key").collect().toSeq)
}
test("SPARK-3708 Backticks aren't handled correctly is aliases") { test("SPARK-3708 Backticks aren't handled correctly is aliases") {
checkAnswer( checkAnswer(
sql("SELECT k FROM (SELECT `key` AS `k` FROM src) a"), sql("SELECT k FROM (SELECT `key` AS `k` FROM src) a"),
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment