Skip to content
Snippets Groups Projects
Commit baa89838 authored by Tathagata Das's avatar Tathagata Das Committed by Andrew Or
Browse files

[SPARK-7838] [STREAMING] Set scope for kinesis stream

Author: Tathagata Das <tathagata.das1565@gmail.com>

Closes #6369 from tdas/SPARK-7838 and squashes the following commits:

87d1c7f [Tathagata Das] Addressed comment
37775d8 [Tathagata Das] set scope for kinesis stream
parent 017b3404
No related branches found
No related tags found
No related merge requests found
......@@ -63,9 +63,12 @@ object KinesisUtils {
checkpointInterval: Duration,
storageLevel: StorageLevel
): ReceiverInputDStream[Array[Byte]] = {
ssc.receiverStream(
new KinesisReceiver(kinesisAppName, streamName, endpointUrl, validateRegion(regionName),
initialPositionInStream, checkpointInterval, storageLevel, None))
// Setting scope to override receiver stream's scope of "receiver stream"
ssc.withNamedScope("kinesis stream") {
ssc.receiverStream(
new KinesisReceiver(kinesisAppName, streamName, endpointUrl, validateRegion(regionName),
initialPositionInStream, checkpointInterval, storageLevel, None))
}
}
/**
......
......@@ -262,7 +262,7 @@ class StreamingContext private[streaming] (
*
* Note: Return statements are NOT allowed in the given body.
*/
private def withNamedScope[U](name: String)(body: => U): U = {
private[streaming] def withNamedScope[U](name: String)(body: => U): U = {
RDDOperationScope.withScope(sc, name, allowNesting = false, ignoreParent = false)(body)
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment