From d3f095f9043fe8c3a55e797b49e2244ee15b58ee Mon Sep 17 00:00:00 2001
From: Denny <dennybritz@gmail.com>
Date: Wed, 10 Oct 2012 16:09:53 -0700
Subject: [PATCH] Fixed bug when fetching Jar dependencies.
Instead of checking currentFiles check currentJars.
---
core/src/main/scala/spark/executor/Executor.scala | 6 +++---
.../main/scala/spark/scheduler/local/LocalScheduler.scala | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/core/src/main/scala/spark/executor/Executor.scala b/core/src/main/scala/spark/executor/Executor.scala
index 6ecf9fa8da..dfdb22024e 100644
--- a/core/src/main/scala/spark/executor/Executor.scala
+++ b/core/src/main/scala/spark/executor/Executor.scala
@@ -141,12 +141,12 @@ private[spark] class Executor extends Logging {
private def updateDependencies(newFiles: HashMap[String, Long], newJars: HashMap[String, Long]) {
// Fetch missing dependencies
for ((name, timestamp) <- newFiles if currentFiles.getOrElse(name, -1L) < timestamp) {
- logInfo("Fetching " + name)
+ logInfo("Fetching " + name + " with timestamp " + timestamp)
Utils.fetchFile(name, new File("."))
currentFiles(name) = timestamp
}
- for ((name, timestamp) <- newJars if currentFiles.getOrElse(name, -1L) < timestamp) {
- logInfo("Fetching " + name)
+ for ((name, timestamp) <- newJars if currentJars.getOrElse(name, -1L) < timestamp) {
+ logInfo("Fetching " + name + " with timestamp " + timestamp)
Utils.fetchFile(name, new File("."))
currentJars(name) = timestamp
// Add it to our class loader
diff --git a/core/src/main/scala/spark/scheduler/local/LocalScheduler.scala b/core/src/main/scala/spark/scheduler/local/LocalScheduler.scala
index 2b38d8b52e..b84b4dc2ed 100644
--- a/core/src/main/scala/spark/scheduler/local/LocalScheduler.scala
+++ b/core/src/main/scala/spark/scheduler/local/LocalScheduler.scala
@@ -109,12 +109,12 @@ private[spark] class LocalScheduler(threads: Int, maxFailures: Int, sc: SparkCon
private def updateDependencies(newFiles: HashMap[String, Long], newJars: HashMap[String, Long]) {
// Fetch missing dependencies
for ((name, timestamp) <- newFiles if currentFiles.getOrElse(name, -1L) < timestamp) {
- logInfo("Fetching " + name)
+ logInfo("Fetching " + name + " with timestamp " + timestamp)
Utils.fetchFile(name, new File("."))
currentFiles(name) = timestamp
}
- for ((name, timestamp) <- newJars if currentFiles.getOrElse(name, -1L) < timestamp) {
- logInfo("Fetching " + name)
+ for ((name, timestamp) <- newJars if currentJars.getOrElse(name, -1L) < timestamp) {
+ logInfo("Fetching " + name + " with timestamp " + timestamp)
Utils.fetchFile(name, new File("."))
currentJars(name) = timestamp
// Add it to our class loader
--
GitLab