Skip to content
Snippets Groups Projects
Commit 3d16a545 authored by Han JU's avatar Han JU Committed by Sean Owen
Browse files

[SPARK-8949] Print warnings when using preferred locations feature

Add warnings according to SPARK-8949 in `SparkContext`
  - warnings in scaladoc
  - log warnings when preferred locations feature is used through `SparkContext`'s constructor

However I didn't found any documentation reference of this feature. Please direct me if you know any reference to this feature.

Author: Han JU <ju.han.felix@gmail.com>

Closes #7874 from darkjh/SPARK-8949.
parent ba2a07e2
No related branches found
No related tags found
No related merge requests found
...@@ -118,9 +118,11 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli ...@@ -118,9 +118,11 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* Can be generated using [[org.apache.spark.scheduler.InputFormatInfo.computePreferredLocations]] * Can be generated using [[org.apache.spark.scheduler.InputFormatInfo.computePreferredLocations]]
* from a list of input files or InputFormats for the application. * from a list of input files or InputFormats for the application.
*/ */
@deprecated("Passing in preferred locations has no effect at all, see SPARK-8949", "1.5.0")
@DeveloperApi @DeveloperApi
def this(config: SparkConf, preferredNodeLocationData: Map[String, Set[SplitInfo]]) = { def this(config: SparkConf, preferredNodeLocationData: Map[String, Set[SplitInfo]]) = {
this(config) this(config)
logWarning("Passing in preferred locations has no effect at all, see SPARK-8949")
this.preferredNodeLocationData = preferredNodeLocationData this.preferredNodeLocationData = preferredNodeLocationData
} }
...@@ -153,6 +155,9 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli ...@@ -153,6 +155,9 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
preferredNodeLocationData: Map[String, Set[SplitInfo]] = Map()) = preferredNodeLocationData: Map[String, Set[SplitInfo]] = Map()) =
{ {
this(SparkContext.updatedConf(new SparkConf(), master, appName, sparkHome, jars, environment)) this(SparkContext.updatedConf(new SparkConf(), master, appName, sparkHome, jars, environment))
if (preferredNodeLocationData.nonEmpty) {
logWarning("Passing in preferred locations has no effect at all, see SPARK-8949")
}
this.preferredNodeLocationData = preferredNodeLocationData this.preferredNodeLocationData = preferredNodeLocationData
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment