From 3d16a545007922ee6fa36e5f5c3959406cb46484 Mon Sep 17 00:00:00 2001
From: Han JU <ju.han.felix@gmail.com>
Date: Wed, 19 Aug 2015 13:04:16 +0100
Subject: [PATCH] [SPARK-8949] Print warnings when using preferred locations
feature
Add warnings according to SPARK-8949 in `SparkContext`
- warnings in scaladoc
- log warnings when preferred locations feature is used through `SparkContext`'s constructor
However I didn't found any documentation reference of this feature. Please direct me if you know any reference to this feature.
Author: Han JU <ju.han.felix@gmail.com>
Closes #7874 from darkjh/SPARK-8949.
---
core/src/main/scala/org/apache/spark/SparkContext.scala | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 2e01a9a18c..1ddaca8a5b 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -118,9 +118,11 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* Can be generated using [[org.apache.spark.scheduler.InputFormatInfo.computePreferredLocations]]
* from a list of input files or InputFormats for the application.
*/
+ @deprecated("Passing in preferred locations has no effect at all, see SPARK-8949", "1.5.0")
@DeveloperApi
def this(config: SparkConf, preferredNodeLocationData: Map[String, Set[SplitInfo]]) = {
this(config)
+ logWarning("Passing in preferred locations has no effect at all, see SPARK-8949")
this.preferredNodeLocationData = preferredNodeLocationData
}
@@ -153,6 +155,9 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
preferredNodeLocationData: Map[String, Set[SplitInfo]] = Map()) =
{
this(SparkContext.updatedConf(new SparkConf(), master, appName, sparkHome, jars, environment))
+ if (preferredNodeLocationData.nonEmpty) {
+ logWarning("Passing in preferred locations has no effect at all, see SPARK-8949")
+ }
this.preferredNodeLocationData = preferredNodeLocationData
}
--
GitLab