Skip to content
Snippets Groups Projects
Commit 58f5361c authored by scwf's avatar scwf Committed by Michael Armbrust
Browse files

[SPARK-3792][SQL] Enable JavaHiveQLSuite

Do not use TestSQLContext in JavaHiveQLSuite, that may lead to two SparkContexts in one jvm and enable JavaHiveQLSuite

Author: scwf <wangfei1@huawei.com>

Closes #2652 from scwf/fix-JavaHiveQLSuite and squashes the following commits:

be35c91 [scwf] enable JavaHiveQLSuite
parent 79b2108d
No related branches found
No related tags found
No related merge requests found
...@@ -25,34 +25,30 @@ import org.apache.spark.api.java.JavaSparkContext ...@@ -25,34 +25,30 @@ import org.apache.spark.api.java.JavaSparkContext
import org.apache.spark.sql.api.java.JavaSchemaRDD import org.apache.spark.sql.api.java.JavaSchemaRDD
import org.apache.spark.sql.execution.ExplainCommand import org.apache.spark.sql.execution.ExplainCommand
import org.apache.spark.sql.hive.test.TestHive import org.apache.spark.sql.hive.test.TestHive
import org.apache.spark.sql.test.TestSQLContext
// Implicits // Implicits
import scala.collection.JavaConversions._ import scala.collection.JavaConversions._
class JavaHiveQLSuite extends FunSuite { class JavaHiveQLSuite extends FunSuite {
lazy val javaCtx = new JavaSparkContext(TestSQLContext.sparkContext) lazy val javaCtx = new JavaSparkContext(TestHive.sparkContext)
// There is a little trickery here to avoid instantiating two HiveContexts in the same JVM // There is a little trickery here to avoid instantiating two HiveContexts in the same JVM
lazy val javaHiveCtx = new JavaHiveContext(javaCtx) { lazy val javaHiveCtx = new JavaHiveContext(javaCtx) {
override val sqlContext = TestHive override val sqlContext = TestHive
} }
ignore("SELECT * FROM src") { test("SELECT * FROM src") {
assert( assert(
javaHiveCtx.sql("SELECT * FROM src").collect().map(_.getInt(0)) === javaHiveCtx.sql("SELECT * FROM src").collect().map(_.getInt(0)) ===
TestHive.sql("SELECT * FROM src").collect().map(_.getInt(0)).toSeq) TestHive.sql("SELECT * FROM src").collect().map(_.getInt(0)).toSeq)
} }
private val explainCommandClassName =
classOf[ExplainCommand].getSimpleName.stripSuffix("$")
def isExplanation(result: JavaSchemaRDD) = { def isExplanation(result: JavaSchemaRDD) = {
val explanation = result.collect().map(_.getString(0)) val explanation = result.collect().map(_.getString(0))
explanation.size > 1 && explanation.head.startsWith(explainCommandClassName) explanation.size > 1 && explanation.head.startsWith("== Physical Plan ==")
} }
ignore("Query Hive native command execution result") { test("Query Hive native command execution result") {
val tableName = "test_native_commands" val tableName = "test_native_commands"
assertResult(0) { assertResult(0) {
...@@ -63,23 +59,18 @@ class JavaHiveQLSuite extends FunSuite { ...@@ -63,23 +59,18 @@ class JavaHiveQLSuite extends FunSuite {
javaHiveCtx.sql(s"CREATE TABLE $tableName(key INT, value STRING)").count() javaHiveCtx.sql(s"CREATE TABLE $tableName(key INT, value STRING)").count()
} }
javaHiveCtx.sql("SHOW TABLES").registerTempTable("show_tables")
assert( assert(
javaHiveCtx javaHiveCtx
.sql("SELECT result FROM show_tables") .sql("SHOW TABLES")
.collect() .collect()
.map(_.getString(0)) .map(_.getString(0))
.contains(tableName)) .contains(tableName))
assertResult(Array(Array("key", "int", "None"), Array("value", "string", "None"))) { assertResult(Array(Array("key", "int"), Array("value", "string"))) {
javaHiveCtx.sql(s"DESCRIBE $tableName").registerTempTable("describe_table")
javaHiveCtx javaHiveCtx
.sql("SELECT result FROM describe_table") .sql(s"describe $tableName")
.collect() .collect()
.map(_.getString(0).split("\t").map(_.trim)) .map(row => Array(row.get(0).asInstanceOf[String], row.get(1).asInstanceOf[String]))
.toArray .toArray
} }
...@@ -89,7 +80,7 @@ class JavaHiveQLSuite extends FunSuite { ...@@ -89,7 +80,7 @@ class JavaHiveQLSuite extends FunSuite {
TestHive.reset() TestHive.reset()
} }
ignore("Exactly once semantics for DDL and command statements") { test("Exactly once semantics for DDL and command statements") {
val tableName = "test_exactly_once" val tableName = "test_exactly_once"
val q0 = javaHiveCtx.sql(s"CREATE TABLE $tableName(key INT, value STRING)") val q0 = javaHiveCtx.sql(s"CREATE TABLE $tableName(key INT, value STRING)")
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment