Skip to content
Snippets Groups Projects
Commit f957796c authored by huangzhaowei's avatar huangzhaowei Committed by Tathagata Das
Browse files

[SPARK-8820] [STREAMING] Add a configuration to set checkpoint dir.

Add a configuration to set checkpoint directory  for convenience to user.
[Jira Address](https://issues.apache.org/jira/browse/SPARK-8820)

Author: huangzhaowei <carlmartinmax@gmail.com>

Closes #7218 from SaintBacchus/SPARK-8820 and squashes the following commits:

d49fe4b [huangzhaowei] Rename the configuration name
66ea47c [huangzhaowei] Add the unit test.
dd0acc1 [huangzhaowei] [SPARK-8820][Streaming] Add a configuration to set checkpoint dir.
parent cc57d705
No related branches found
No related tags found
No related merge requests found
......@@ -201,6 +201,8 @@ class StreamingContext private[streaming] (
private var shutdownHookRef: AnyRef = _
conf.getOption("spark.streaming.checkpoint.directory").foreach(checkpoint)
/**
* Return the associated Spark context
*/
......
......@@ -115,6 +115,15 @@ class StreamingContextSuite extends SparkFunSuite with BeforeAndAfter with Timeo
assert(ssc.conf.getTimeAsSeconds("spark.cleaner.ttl", "-1") === 10)
}
test("checkPoint from conf") {
val checkpointDirectory = Utils.createTempDir().getAbsolutePath()
val myConf = SparkContext.updatedConf(new SparkConf(false), master, appName)
myConf.set("spark.streaming.checkpoint.directory", checkpointDirectory)
val ssc = new StreamingContext(myConf, batchDuration)
assert(ssc.checkpointDir != null)
}
test("state matching") {
import StreamingContextState._
assert(INITIALIZED === INITIALIZED)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment