Skip to content
Snippets Groups Projects
Commit 166d9f91 authored by Matei Zaharia's avatar Matei Zaharia
Browse files

Removed setSparkHome method on SparkContext in favor of having an

optional constructor parameter, so that the scheduler is guaranteed that
a Spark home has been set when it first builds its executor arg.
parent 1c082ad5
No related branches found
No related tags found
No related merge requests found
......@@ -87,9 +87,9 @@ extends MScheduler with spark.Scheduler with Logging
val sparkHome = sc.getSparkHome match {
case Some(path) => path
case None =>
throw new SparkException("Spark home is not set; either set the " +
"spark.home system property or the SPARK_HOME environment variable " +
"or call SparkContext.setSparkHome")
throw new SparkException("Spark home is not set; set it through the " +
"spark.home system property, the SPARK_HOME environment variable " +
"or the SparkContext constructor")
}
val execScript = new File(sparkHome, "spark-executor").getCanonicalPath
val params = new JHashMap[String, String]
......
......@@ -8,11 +8,9 @@ import scala.collection.mutable.ArrayBuffer
class SparkContext(
master: String,
frameworkName: String,
val sparkHome: String = null,
val jars: Seq[String] = Nil)
extends Logging {
// Spark home directory, used to resolve executor when running on Mesos
private var sparkHome: Option[String] = None
private[spark] var scheduler: Scheduler = {
// Regular expression used for local[N] master format
val LOCAL_N_REGEX = """local\[([0-9]+)\]""".r
......@@ -63,19 +61,12 @@ extends Logging {
scheduler.waitForRegister()
}
// Set the Spark home location
def setSparkHome(path: String) {
if (path == null)
throw new NullPointerException("Path passed to setSparkHome was null")
sparkHome = Some(path)
}
// Get Spark's home location from either a value set through setSparkHome,
// Get Spark's home location from either a value set through the constructor,
// or the spark.home Java property, or the SPARK_HOME environment variable
// (in that order of preference). If neither of these is set, return None.
def getSparkHome(): Option[String] = {
if (sparkHome != None)
sparkHome
if (sparkHome != null)
Some(sparkHome)
else if (System.getProperty("spark.home") != null)
Some(System.getProperty("spark.home"))
else if (System.getenv("SPARK_HOME") != null)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment