Skip to content
Snippets Groups Projects
Commit 166d9f91 authored by Matei Zaharia's avatar Matei Zaharia
Browse files

Removed setSparkHome method on SparkContext in favor of having an

optional constructor parameter, so that the scheduler is guaranteed that
a Spark home has been set when it first builds its executor arg.
parent 1c082ad5
No related branches found
No related tags found
No related merge requests found
...@@ -87,9 +87,9 @@ extends MScheduler with spark.Scheduler with Logging ...@@ -87,9 +87,9 @@ extends MScheduler with spark.Scheduler with Logging
val sparkHome = sc.getSparkHome match { val sparkHome = sc.getSparkHome match {
case Some(path) => path case Some(path) => path
case None => case None =>
throw new SparkException("Spark home is not set; either set the " + throw new SparkException("Spark home is not set; set it through the " +
"spark.home system property or the SPARK_HOME environment variable " + "spark.home system property, the SPARK_HOME environment variable " +
"or call SparkContext.setSparkHome") "or the SparkContext constructor")
} }
val execScript = new File(sparkHome, "spark-executor").getCanonicalPath val execScript = new File(sparkHome, "spark-executor").getCanonicalPath
val params = new JHashMap[String, String] val params = new JHashMap[String, String]
......
...@@ -8,11 +8,9 @@ import scala.collection.mutable.ArrayBuffer ...@@ -8,11 +8,9 @@ import scala.collection.mutable.ArrayBuffer
class SparkContext( class SparkContext(
master: String, master: String,
frameworkName: String, frameworkName: String,
val sparkHome: String = null,
val jars: Seq[String] = Nil) val jars: Seq[String] = Nil)
extends Logging { extends Logging {
// Spark home directory, used to resolve executor when running on Mesos
private var sparkHome: Option[String] = None
private[spark] var scheduler: Scheduler = { private[spark] var scheduler: Scheduler = {
// Regular expression used for local[N] master format // Regular expression used for local[N] master format
val LOCAL_N_REGEX = """local\[([0-9]+)\]""".r val LOCAL_N_REGEX = """local\[([0-9]+)\]""".r
...@@ -63,19 +61,12 @@ extends Logging { ...@@ -63,19 +61,12 @@ extends Logging {
scheduler.waitForRegister() scheduler.waitForRegister()
} }
// Set the Spark home location // Get Spark's home location from either a value set through the constructor,
def setSparkHome(path: String) {
if (path == null)
throw new NullPointerException("Path passed to setSparkHome was null")
sparkHome = Some(path)
}
// Get Spark's home location from either a value set through setSparkHome,
// or the spark.home Java property, or the SPARK_HOME environment variable // or the spark.home Java property, or the SPARK_HOME environment variable
// (in that order of preference). If neither of these is set, return None. // (in that order of preference). If neither of these is set, return None.
def getSparkHome(): Option[String] = { def getSparkHome(): Option[String] = {
if (sparkHome != None) if (sparkHome != null)
sparkHome Some(sparkHome)
else if (System.getProperty("spark.home") != null) else if (System.getProperty("spark.home") != null)
Some(System.getProperty("spark.home")) Some(System.getProperty("spark.home"))
else if (System.getenv("SPARK_HOME") != null) else if (System.getenv("SPARK_HOME") != null)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment