Skip to content
Snippets Groups Projects
Commit 328e51b6 authored by Matei Zaharia's avatar Matei Zaharia
Browse files

Various minor fixes

parent fd1d2558
No related branches found
No related tags found
No related merge requests found
No preview for this file type
...@@ -66,7 +66,7 @@ class CacheTracker(isMaster: Boolean, theCache: Cache) extends Logging { ...@@ -66,7 +66,7 @@ class CacheTracker(isMaster: Boolean, theCache: Cache) extends Logging {
if (isMaster) { if (isMaster) {
val tracker = new CacheTrackerActor val tracker = new CacheTrackerActor
tracker.start tracker.start()
trackerActor = tracker trackerActor = tracker
} else { } else {
val host = System.getProperty("spark.master.host") val host = System.getProperty("spark.master.host")
......
...@@ -34,18 +34,18 @@ extends DaemonActor with Logging { ...@@ -34,18 +34,18 @@ extends DaemonActor with Logging {
class MapOutputTracker(isMaster: Boolean) extends Logging { class MapOutputTracker(isMaster: Boolean) extends Logging {
var trackerActor: AbstractActor = null var trackerActor: AbstractActor = null
private val serverUris = new ConcurrentHashMap[Int, Array[String]]
if (isMaster) { if (isMaster) {
val tracker = new MapOutputTrackerActor(serverUris) val tracker = new MapOutputTrackerActor(serverUris)
tracker.start tracker.start()
trackerActor = tracker trackerActor = tracker
} else { } else {
val host = System.getProperty("spark.master.host") val host = System.getProperty("spark.master.host")
val port = System.getProperty("spark.master.port").toInt val port = System.getProperty("spark.master.port").toInt
trackerActor = RemoteActor.select(Node(host, port), 'MapOutputTracker) trackerActor = RemoteActor.select(Node(host, port), 'MapOutputTracker)
} }
private val serverUris = new ConcurrentHashMap[Int, Array[String]]
def registerMapOutput(shuffleId: Int, numMaps: Int, mapId: Int, serverUri: String) { def registerMapOutput(shuffleId: Int, numMaps: Int, mapId: Int, serverUri: String) {
var array = serverUris.get(shuffleId) var array = serverUris.get(shuffleId)
...@@ -82,6 +82,7 @@ class MapOutputTracker(isMaster: Boolean) extends Logging { ...@@ -82,6 +82,7 @@ class MapOutputTracker(isMaster: Boolean) extends Logging {
// We won the race to fetch the output locs; do so // We won the race to fetch the output locs; do so
logInfo("Doing the fetch; tracker actor = " + trackerActor) logInfo("Doing the fetch; tracker actor = " + trackerActor)
val fetched = (trackerActor !? GetMapOutputLocations(shuffleId)).asInstanceOf[Array[String]] val fetched = (trackerActor !? GetMapOutputLocations(shuffleId)).asInstanceOf[Array[String]]
println("Got locations: " + fetched.mkString(", "))
serverUris.put(shuffleId, fetched) serverUris.put(shuffleId, fetched)
fetching.synchronized { fetching.synchronized {
fetching -= shuffleId fetching -= shuffleId
......
#!/bin/bash #!/bin/bash
java -Xmx700M -jar `dirname $0`/sbt-launch-*.jar "$@" EXTRA_ARGS=""
if [ "$MESOS_HOME" != "" ]; then
EXTRA_ARGS="-Djava.library.path=$MESOS_HOME/lib/java"
fi
export SPARK_HOME=$(cd "$(dirname $0)/.."; pwd)
java -Xmx700M $EXTRA_ARGS -jar $SPARK_HOME/sbt/sbt-launch-*.jar "$@"
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment