Skip to content
Snippets Groups Projects
Commit c79c10eb authored by gatorsmile's avatar gatorsmile
Browse files

[TEST] Different behaviors of SparkContext Conf when building SparkSession

## What changes were proposed in this pull request?
If the created ACTIVE sparkContext is not EXPLICITLY passed through the Builder's API `sparkContext()`, the conf of this sparkContext will also contain the conf set through the API `config()`; otherwise, the conf of this sparkContext will NOT contain the conf set through the API `config()`

## How was this patch tested?
N/A

Author: gatorsmile <gatorsmile@gmail.com>

Closes #18517 from gatorsmile/fixTestCase2.
parent f953ca56
No related branches found
No related tags found
No related merge requests found
......@@ -98,12 +98,31 @@ class SparkSessionBuilderSuite extends SparkFunSuite {
val session = SparkSession.builder().config("key2", "value2").getOrCreate()
assert(session.conf.get("key1") == "value1")
assert(session.conf.get("key2") == "value2")
assert(session.sparkContext == sparkContext2)
assert(session.sparkContext.conf.get("key1") == "value1")
// If the created sparkContext is not passed through the Builder's API sparkContext,
// the conf of this sparkContext will also contain the conf set through the API config.
assert(session.sparkContext.conf.get("key2") == "value2")
assert(session.sparkContext.conf.get("spark.app.name") == "test")
session.stop()
}
test("create SparkContext first then pass context to SparkSession") {
sparkContext.stop()
val conf = new SparkConf().setAppName("test").setMaster("local").set("key1", "value1")
val newSC = new SparkContext(conf)
val session = SparkSession.builder().sparkContext(newSC).config("key2", "value2").getOrCreate()
assert(session.conf.get("key1") == "value1")
assert(session.conf.get("key2") == "value2")
assert(session.sparkContext == newSC)
assert(session.sparkContext.conf.get("key1") == "value1")
// If the created sparkContext is passed through the Builder's API sparkContext,
// the conf of this sparkContext will not contain the conf set through the API config.
assert(!session.sparkContext.conf.contains("key2"))
assert(session.sparkContext.conf.get("spark.app.name") == "test")
session.stop()
}
test("SPARK-15887: hive-site.xml should be loaded") {
val session = SparkSession.builder().master("local").getOrCreate()
assert(session.sessionState.newHadoopConf().get("hive.in.test") == "true")
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment