Skip to content
Snippets Groups Projects
Commit 235f4ac6 authored by bomeng's avatar bomeng Committed by Marcelo Vanzin
Browse files

[SPARK-13727][CORE] SparkConf.contains does not consider deprecated keys

The contains() method does not return consistently with get() if the key is deprecated. For example,
import org.apache.spark.SparkConf
val conf = new SparkConf()
conf.set("spark.io.compression.lz4.block.size", "12345")  # display some deprecated warning message
conf.get("spark.io.compression.lz4.block.size") # return 12345
conf.get("spark.io.compression.lz4.blockSize") # return 12345
conf.contains("spark.io.compression.lz4.block.size") # return true
conf.contains("spark.io.compression.lz4.blockSize") # return false

The fix will make the contains() and get() more consistent.

I've added a test case for this.

(Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests)
Unit tests should be sufficient.

Author: bomeng <bmeng@us.ibm.com>

Closes #11568 from bomeng/SPARK-13727.
parent d24801ad
No related branches found
No related tags found
No related merge requests found
......@@ -388,7 +388,10 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging {
def getAppId: String = get("spark.app.id")
/** Does the configuration contain a given parameter? */
def contains(key: String): Boolean = settings.containsKey(key)
def contains(key: String): Boolean = {
settings.containsKey(key) ||
configsWithAlternatives.get(key).toSeq.flatten.exists { alt => contains(alt.key) }
}
/** Copy this object */
override def clone: SparkConf = {
......
......@@ -267,6 +267,20 @@ class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSyst
conf.set("spark.akka.lookupTimeout", "4")
assert(RpcUtils.lookupRpcTimeout(conf).duration === (4 seconds))
}
test("SPARK-13727") {
val conf = new SparkConf()
// set the conf in the deprecated way
conf.set("spark.io.compression.lz4.block.size", "12345")
// get the conf in the recommended way
assert(conf.get("spark.io.compression.lz4.blockSize") === "12345")
// we can still get the conf in the deprecated way
assert(conf.get("spark.io.compression.lz4.block.size") === "12345")
// the contains() also works as expected
assert(conf.contains("spark.io.compression.lz4.block.size"))
assert(conf.contains("spark.io.compression.lz4.blockSize"))
assert(conf.contains("spark.io.unknown") === false)
}
}
class Class1 {}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment