Skip to content
Snippets Groups Projects
Commit 0340c56a authored by comcmipi's avatar comcmipi Committed by Tathagata Das
Browse files

Update RecoverableNetworkWordCount.scala

Trying this example, I missed the moment when the checkpoint was iniciated

Author: comcmipi <pitonak@fns.uniba.sk>

Closes #2735 from comcmipi/patch-1 and squashes the following commits:

b6d8001 [comcmipi] Update RecoverableNetworkWordCount.scala
96fe274 [comcmipi] Update RecoverableNetworkWordCount.scala
parent 3a02d416
No related branches found
No related tags found
No related merge requests found
...@@ -56,7 +56,7 @@ import org.apache.spark.util.IntParam ...@@ -56,7 +56,7 @@ import org.apache.spark.util.IntParam
*/ */
object RecoverableNetworkWordCount { object RecoverableNetworkWordCount {
def createContext(ip: String, port: Int, outputPath: String) = { def createContext(ip: String, port: Int, outputPath: String, checkpointDirectory: String) = {
// If you do not see this printed, that means the StreamingContext has been loaded // If you do not see this printed, that means the StreamingContext has been loaded
// from the new checkpoint // from the new checkpoint
...@@ -66,6 +66,7 @@ object RecoverableNetworkWordCount { ...@@ -66,6 +66,7 @@ object RecoverableNetworkWordCount {
val sparkConf = new SparkConf().setAppName("RecoverableNetworkWordCount") val sparkConf = new SparkConf().setAppName("RecoverableNetworkWordCount")
// Create the context with a 1 second batch size // Create the context with a 1 second batch size
val ssc = new StreamingContext(sparkConf, Seconds(1)) val ssc = new StreamingContext(sparkConf, Seconds(1))
ssc.checkpoint(checkpointDirectory)
// Create a socket stream on target ip:port and count the // Create a socket stream on target ip:port and count the
// words in input stream of \n delimited text (eg. generated by 'nc') // words in input stream of \n delimited text (eg. generated by 'nc')
...@@ -101,7 +102,7 @@ object RecoverableNetworkWordCount { ...@@ -101,7 +102,7 @@ object RecoverableNetworkWordCount {
val Array(ip, IntParam(port), checkpointDirectory, outputPath) = args val Array(ip, IntParam(port), checkpointDirectory, outputPath) = args
val ssc = StreamingContext.getOrCreate(checkpointDirectory, val ssc = StreamingContext.getOrCreate(checkpointDirectory,
() => { () => {
createContext(ip, port, outputPath) createContext(ip, port, outputPath, checkpointDirectory)
}) })
ssc.start() ssc.start()
ssc.awaitTermination() ssc.awaitTermination()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment