Skip to content
Snippets Groups Projects
Commit 54c0f9f1 authored by Matei Zaharia's avatar Matei Zaharia
Browse files

Fix code that assumed spark.local.dir is only a single directory

parent b534fd36
No related branches found
No related tags found
No related merge requests found
...@@ -134,7 +134,7 @@ private object Utils extends Logging { ...@@ -134,7 +134,7 @@ private object Utils extends Logging {
*/ */
def fetchFile(url: String, targetDir: File) { def fetchFile(url: String, targetDir: File) {
val filename = url.split("/").last val filename = url.split("/").last
val tempDir = System.getProperty("spark.local.dir", System.getProperty("java.io.tmpdir")) val tempDir = getLocalDir
val tempFile = File.createTempFile("fetchFileTemp", null, new File(tempDir)) val tempFile = File.createTempFile("fetchFileTemp", null, new File(tempDir))
val targetFile = new File(targetDir, filename) val targetFile = new File(targetDir, filename)
val uri = new URI(url) val uri = new URI(url)
...@@ -204,6 +204,15 @@ private object Utils extends Logging { ...@@ -204,6 +204,15 @@ private object Utils extends Logging {
FileUtil.chmod(filename, "a+x") FileUtil.chmod(filename, "a+x")
} }
/**
* Get a temporary directory using Spark's spark.local.dir property, if set. This will always
* return a single directory, even though the spark.local.dir property might be a list of
* multiple paths.
*/
def getLocalDir: String = {
System.getProperty("spark.local.dir", System.getProperty("java.io.tmpdir")).split(',')(0)
}
/** /**
* Shuffle the elements of a collection into a random order, returning the * Shuffle the elements of a collection into a random order, returning the
* result in a new collection. Unlike scala.util.Random.shuffle, this method * result in a new collection. Unlike scala.util.Random.shuffle, this method
......
...@@ -89,7 +89,7 @@ private object HttpBroadcast extends Logging { ...@@ -89,7 +89,7 @@ private object HttpBroadcast extends Logging {
} }
private def createServer() { private def createServer() {
broadcastDir = Utils.createTempDir(System.getProperty("spark.local.dir", System.getProperty("java.io.tmpdir"))) broadcastDir = Utils.createTempDir(Utils.getLocalDir)
server = new HttpServer(broadcastDir) server = new HttpServer(broadcastDir)
server.start() server.start()
serverUri = server.uri serverUri = server.uri
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment