diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index e671a3e95a07492d5da5faed96c33325802caf20..33ed0d5493e0e8934aab15ae0de5e82bcce91fba 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -455,7 +455,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging {
}
// Validate spark.executor.extraJavaOptions
- getOption(executorOptsKey).map { javaOpts =>
+ getOption(executorOptsKey).foreach { javaOpts =>
if (javaOpts.contains("-Dspark")) {
val msg = s"$executorOptsKey is not allowed to set Spark options (was '$javaOpts'). " +
"Set them directly on a SparkConf or in a properties file when using ./bin/spark-submit."
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index ff633cf837895ab58d1fe860f06967f4777a70ea..168ac7e04b923c0f24429a28695f6eb0c5a284d6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -44,8 +44,10 @@ import org.apache.spark.sql.types._
import org.apache.spark.sql.util.ExecutionListenerManager
/**
- * The entry point for working with structured data (rows and columns) in Spark. Allows the
- * creation of [[DataFrame]] objects as well as the execution of SQL queries.
+ * The entry point for working with structured data (rows and columns) in Spark, in Spark 1.x.
+ *
+ * As of Spark 2.0, this is replaced by [[SparkSession]]. However, we are keeping the class here
+ * for backward compatibility.
*
* @groupname basic Basic Operations
* @groupname ddl_ops Persistent Catalog DDL
@@ -165,23 +167,6 @@ class SQLContext private[sql](
sparkSession.conf.get(key)
}
- /**
- * Return the value of Spark SQL configuration property for the given key. If the key is not set
- * yet, return `defaultValue` in [[ConfigEntry]].
- */
- private[sql] def getConf[T](entry: ConfigEntry[T]): T = {
- sparkSession.conf.get(entry)
- }
-
- /**
- * Return the value of Spark SQL configuration property for the given key. If the key is not set
- * yet, return `defaultValue`. This is useful when `defaultValue` in ConfigEntry is not the
- * desired one.
- */
- private[sql] def getConf[T](entry: ConfigEntry[T], defaultValue: T): T = {
- sparkSession.conf.get(entry, defaultValue)
- }
-
/**
* Return the value of Spark SQL configuration property for the given key. If the key is not set
* yet, return `defaultValue`.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
index 5b96ab10c9915dcf081abb042e4fd2e21b0d6c18..c77c889a1b7b8a00b9f502acb4957ba68ae33364 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
@@ -68,11 +68,11 @@ package object debug {
}
/**
- * Augments [[SQLContext]] with debug methods.
+ * Augments [[SparkSession]] with debug methods.
*/
- implicit class DebugSQLContext(sqlContext: SQLContext) {
+ implicit class DebugSQLContext(sparkSession: SparkSession) {
def debug(): Unit = {
- sqlContext.setConf(SQLConf.DATAFRAME_EAGER_ANALYSIS, false)
+ sparkSession.conf.set(SQLConf.DATAFRAME_EAGER_ANALYSIS.key, false)
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index f10d8372ed3715664e5f3ad66909bcf0e4674268..80a93ee6d4f3ee8c2e1dce3c4ea2f190338b034c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -66,21 +66,6 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
Row(1, 1) :: Nil)
}
- ignore("invalid plan toString, debug mode") {
- // Turn on debug mode so we can see invalid query plans.
- import org.apache.spark.sql.execution.debug._
-
- withSQLConf(SQLConf.DATAFRAME_EAGER_ANALYSIS.key -> "true") {
- sqlContext.debug()
-
- val badPlan = testData.select('badColumn)
-
- assert(badPlan.toString contains badPlan.queryExecution.toString,
- "toString on bad query plans should include the query execution but was:\n" +
- badPlan.toString)
- }
- }
-
test("access complex data") {
assert(complexData.filter(complexData("a").getItem(0) === 2).count() == 1)
assert(complexData.filter(complexData("m").getItem("1") === 1).count() == 1)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 5065e5b80b08c9af244bcbdcf798acded1fde9ff..ec5163b658c14d5026045662ca6425f2410b1c67 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -1495,15 +1495,11 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
}
test("SPARK-4699 case sensitivity SQL query") {
- val orig = sqlContext.getConf(SQLConf.CASE_SENSITIVE)
- try {
- sqlContext.setConf(SQLConf.CASE_SENSITIVE, false)
+ withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
val data = TestData(1, "val_1") :: TestData(2, "val_2") :: Nil
val rdd = sparkContext.parallelize((0 to 1).map(i => data(i)))
rdd.toDF().registerTempTable("testTable1")
checkAnswer(sql("SELECT VALUE FROM TESTTABLE1 where KEY = 1"), Row("val_1"))
- } finally {
- sqlContext.setConf(SQLConf.CASE_SENSITIVE, orig)
}
}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
index f3076912cb1c0ce65f4e417fb09e49c955a88590..57aa4b293149bedd5c2eeb3b47eba9a79c88d83c 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionState.scala
@@ -44,8 +44,6 @@ private[hive] class HiveSessionState(sparkSession: SparkSession)
*/
lazy val metadataHive: HiveClient = sharedState.metadataHive.newSession()
- setDefaultOverrideConfs()
-
/**
* Internal catalog for managing table and database states.
*/
@@ -108,14 +106,6 @@ private[hive] class HiveSessionState(sparkSession: SparkSession)
// Helper methods, partially leftover from pre-2.0 days
// ------------------------------------------------------
- /**
- * Overrides default Hive configurations to avoid breaking changes to Spark SQL users.
- * - allow SQL11 keywords to be used as identifiers
- */
- def setDefaultOverrideConfs(): Unit = {
- conf.setConfString(ConfVars.HIVE_SUPPORT_SQL11_RESERVED_KEYWORDS.varname, "false")
- }
-
override def addJar(path: String): Unit = {
metadataHive.addJar(path)
super.addJar(path)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
index 93646a45a260deb8842eb8a3bfd0f1025d381b1d..b41d882ffa1f24f50c7d051e58d19fff1ea8b661 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -432,9 +432,6 @@ private[hive] class TestHiveSparkSession(
// Lots of tests fail if we do not change the partition whitelist from the default.
sessionState.metadataHive.runSqlHive("set hive.metastore.partition.name.whitelist.pattern=.*")
- // In case a test changed any of these values, restore all the original ones here.
- sessionState.setDefaultOverrideConfs()
-
sessionState.catalog.setCurrentDatabase("default")
} catch {
case e: Exception =>