Skip to content
Snippets Groups Projects
Commit d3f095f9 authored by Denny's avatar Denny
Browse files

Fixed bug when fetching Jar dependencies.

Instead of checking currentFiles check currentJars.
parent 1ac7bf89
No related branches found
No related tags found
No related merge requests found
...@@ -141,12 +141,12 @@ private[spark] class Executor extends Logging { ...@@ -141,12 +141,12 @@ private[spark] class Executor extends Logging {
private def updateDependencies(newFiles: HashMap[String, Long], newJars: HashMap[String, Long]) { private def updateDependencies(newFiles: HashMap[String, Long], newJars: HashMap[String, Long]) {
// Fetch missing dependencies // Fetch missing dependencies
for ((name, timestamp) <- newFiles if currentFiles.getOrElse(name, -1L) < timestamp) { for ((name, timestamp) <- newFiles if currentFiles.getOrElse(name, -1L) < timestamp) {
logInfo("Fetching " + name) logInfo("Fetching " + name + " with timestamp " + timestamp)
Utils.fetchFile(name, new File(".")) Utils.fetchFile(name, new File("."))
currentFiles(name) = timestamp currentFiles(name) = timestamp
} }
for ((name, timestamp) <- newJars if currentFiles.getOrElse(name, -1L) < timestamp) { for ((name, timestamp) <- newJars if currentJars.getOrElse(name, -1L) < timestamp) {
logInfo("Fetching " + name) logInfo("Fetching " + name + " with timestamp " + timestamp)
Utils.fetchFile(name, new File(".")) Utils.fetchFile(name, new File("."))
currentJars(name) = timestamp currentJars(name) = timestamp
// Add it to our class loader // Add it to our class loader
......
...@@ -109,12 +109,12 @@ private[spark] class LocalScheduler(threads: Int, maxFailures: Int, sc: SparkCon ...@@ -109,12 +109,12 @@ private[spark] class LocalScheduler(threads: Int, maxFailures: Int, sc: SparkCon
private def updateDependencies(newFiles: HashMap[String, Long], newJars: HashMap[String, Long]) { private def updateDependencies(newFiles: HashMap[String, Long], newJars: HashMap[String, Long]) {
// Fetch missing dependencies // Fetch missing dependencies
for ((name, timestamp) <- newFiles if currentFiles.getOrElse(name, -1L) < timestamp) { for ((name, timestamp) <- newFiles if currentFiles.getOrElse(name, -1L) < timestamp) {
logInfo("Fetching " + name) logInfo("Fetching " + name + " with timestamp " + timestamp)
Utils.fetchFile(name, new File(".")) Utils.fetchFile(name, new File("."))
currentFiles(name) = timestamp currentFiles(name) = timestamp
} }
for ((name, timestamp) <- newJars if currentFiles.getOrElse(name, -1L) < timestamp) { for ((name, timestamp) <- newJars if currentJars.getOrElse(name, -1L) < timestamp) {
logInfo("Fetching " + name) logInfo("Fetching " + name + " with timestamp " + timestamp)
Utils.fetchFile(name, new File(".")) Utils.fetchFile(name, new File("."))
currentJars(name) = timestamp currentJars(name) = timestamp
// Add it to our class loader // Add it to our class loader
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment