From 09841290055770a619a2e72fbaef1a5e694916ae Mon Sep 17 00:00:00 2001
From: Hari Shreedharan <hshreedharan@apache.org>
Date: Thu, 8 Oct 2015 18:53:38 -0700
Subject: [PATCH] [SPARK-10955] [STREAMING] Add a warning if dynamic allocation
for Streaming applications
Dynamic allocation can be painful for streaming apps and can lose data. Log a warning for streaming applications if dynamic allocation is enabled.
Author: Hari Shreedharan <hshreedharan@apache.org>
Closes #8998 from harishreedharan/ss-log-error and squashes the following commits:
462b264 [Hari Shreedharan] Improve log message.
2733d94 [Hari Shreedharan] Minor change to warning message.
eaa48cc [Hari Shreedharan] Log a warning instead of failing the application if dynamic allocation is enabled.
725f090 [Hari Shreedharan] Add config parameter to allow dynamic allocation if the user explicitly sets it.
b3f9a95 [Hari Shreedharan] Disable dynamic allocation and kill app if it is enabled.
a4a5212 [Hari Shreedharan] [streaming] SPARK-10955. Disable dynamic allocation for Streaming applications.
---
.../org/apache/spark/streaming/StreamingContext.scala | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
index 94fea63f55..9b2632c229 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
@@ -44,7 +44,7 @@ import org.apache.spark.streaming.dstream._
import org.apache.spark.streaming.receiver.{ActorReceiver, ActorSupervisorStrategy, Receiver}
import org.apache.spark.streaming.scheduler.{JobScheduler, StreamingListener}
import org.apache.spark.streaming.ui.{StreamingJobProgressListener, StreamingTab}
-import org.apache.spark.util.{CallSite, ShutdownHookManager, ThreadUtils}
+import org.apache.spark.util.{CallSite, ShutdownHookManager, ThreadUtils, Utils}
/**
* Main entry point for Spark Streaming functionality. It provides methods used to create
@@ -564,6 +564,13 @@ class StreamingContext private[streaming] (
)
}
}
+
+ if (Utils.isDynamicAllocationEnabled(sc.conf)) {
+ logWarning("Dynamic Allocation is enabled for this application. " +
+ "Enabling Dynamic allocation for Spark Streaming applications can cause data loss if " +
+ "Write Ahead Log is not enabled for non-replayable sources like Flume. " +
+ "See the programming guide for details on how to enable the Write Ahead Log")
+ }
}
/**
--
GitLab