Skip to content
Snippets Groups Projects
Commit bd237d4a authored by Josh Rosen's avatar Josh Rosen
Browse files

Add synchronization to LocalScheduler.updateDependencies().

parent f1bf4f03
No related branches found
No related tags found
No related merge requests found
...@@ -108,22 +108,24 @@ private[spark] class LocalScheduler(threads: Int, maxFailures: Int, sc: SparkCon ...@@ -108,22 +108,24 @@ private[spark] class LocalScheduler(threads: Int, maxFailures: Int, sc: SparkCon
* SparkContext. Also adds any new JARs we fetched to the class loader. * SparkContext. Also adds any new JARs we fetched to the class loader.
*/ */
private def updateDependencies(newFiles: HashMap[String, Long], newJars: HashMap[String, Long]) { private def updateDependencies(newFiles: HashMap[String, Long], newJars: HashMap[String, Long]) {
// Fetch missing dependencies this.synchronized {
for ((name, timestamp) <- newFiles if currentFiles.getOrElse(name, -1L) < timestamp) { // Fetch missing dependencies
logInfo("Fetching " + name + " with timestamp " + timestamp) for ((name, timestamp) <- newFiles if currentFiles.getOrElse(name, -1L) < timestamp) {
Utils.fetchFile(name, new File(".")) logInfo("Fetching " + name + " with timestamp " + timestamp)
currentFiles(name) = timestamp Utils.fetchFile(name, new File("."))
} currentFiles(name) = timestamp
for ((name, timestamp) <- newJars if currentJars.getOrElse(name, -1L) < timestamp) { }
logInfo("Fetching " + name + " with timestamp " + timestamp) for ((name, timestamp) <- newJars if currentJars.getOrElse(name, -1L) < timestamp) {
Utils.fetchFile(name, new File(".")) logInfo("Fetching " + name + " with timestamp " + timestamp)
currentJars(name) = timestamp Utils.fetchFile(name, new File("."))
// Add it to our class loader currentJars(name) = timestamp
val localName = name.split("/").last // Add it to our class loader
val url = new File(".", localName).toURI.toURL val localName = name.split("/").last
if (!classLoader.getURLs.contains(url)) { val url = new File(".", localName).toURI.toURL
logInfo("Adding " + url + " to class loader") if (!classLoader.getURLs.contains(url)) {
classLoader.addURL(url) logInfo("Adding " + url + " to class loader")
classLoader.addURL(url)
}
} }
} }
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment