From 8c911adac56a1b1d95bc19915e0070ce7305257c Mon Sep 17 00:00:00 2001
From: hyukjinkwon <gurwls223@gmail.com>
Date: Fri, 28 Apr 2017 08:49:35 +0100
Subject: [PATCH] [SPARK-20465][CORE] Throws a proper exception when any temp
directory could not be got
## What changes were proposed in this pull request?
This PR proposes to throw an exception with better message rather than `ArrayIndexOutOfBoundsException` when temp directories could not be created.
Running the commands below:
```bash
./bin/spark-shell --conf spark.local.dir=/NONEXISTENT_DIR_ONE,/NONEXISTENT_DIR_TWO
```
produces ...
**Before**
```
Exception in thread "main" java.lang.ExceptionInInitializerError
...
Caused by: java.lang.ArrayIndexOutOfBoundsException: 0
...
```
**After**
```
Exception in thread "main" java.lang.ExceptionInInitializerError
...
Caused by: java.io.IOException: Failed to get a temp directory under [/NONEXISTENT_DIR_ONE,/NONEXISTENT_DIR_TWO].
...
```
## How was this patch tested?
Unit tests in `LocalDirsSuite.scala`.
Author: hyukjinkwon <gurwls223@gmail.com>
Closes #17768 from HyukjinKwon/throws-temp-dir-exception.
---
.../scala/org/apache/spark/util/Utils.scala | 6 ++++-
.../apache/spark/storage/LocalDirsSuite.scala | 23 ++++++++++++++++---
2 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index e042badcdd..4d37db96df 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -740,7 +740,11 @@ private[spark] object Utils extends Logging {
* always return a single directory.
*/
def getLocalDir(conf: SparkConf): String = {
- getOrCreateLocalRootDirs(conf)(0)
+ getOrCreateLocalRootDirs(conf).headOption.getOrElse {
+ val configuredLocalDirs = getConfiguredLocalDirs(conf)
+ throw new IOException(
+ s"Failed to get a temp directory under [${configuredLocalDirs.mkString(",")}].")
+ }
}
private[spark] def isRunningInYarnContainer(conf: SparkConf): Boolean = {
diff --git a/core/src/test/scala/org/apache/spark/storage/LocalDirsSuite.scala b/core/src/test/scala/org/apache/spark/storage/LocalDirsSuite.scala
index c7074078d8..f7b3a2754f 100644
--- a/core/src/test/scala/org/apache/spark/storage/LocalDirsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/LocalDirsSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.storage
-import java.io.File
+import java.io.{File, IOException}
import org.scalatest.BeforeAndAfter
@@ -33,9 +33,13 @@ class LocalDirsSuite extends SparkFunSuite with BeforeAndAfter {
Utils.clearLocalRootDirs()
}
+ after {
+ Utils.clearLocalRootDirs()
+ }
+
test("Utils.getLocalDir() returns a valid directory, even if some local dirs are missing") {
// Regression test for SPARK-2974
- assert(!new File("/NONEXISTENT_DIR").exists())
+ assert(!new File("/NONEXISTENT_PATH").exists())
val conf = new SparkConf(false)
.set("spark.local.dir", s"/NONEXISTENT_PATH,${System.getProperty("java.io.tmpdir")}")
assert(new File(Utils.getLocalDir(conf)).exists())
@@ -43,7 +47,7 @@ class LocalDirsSuite extends SparkFunSuite with BeforeAndAfter {
test("SPARK_LOCAL_DIRS override also affects driver") {
// Regression test for SPARK-2975
- assert(!new File("/NONEXISTENT_DIR").exists())
+ assert(!new File("/NONEXISTENT_PATH").exists())
// spark.local.dir only contains invalid directories, but that's not a problem since
// SPARK_LOCAL_DIRS will override it on both the driver and workers:
val conf = new SparkConfWithEnv(Map("SPARK_LOCAL_DIRS" -> System.getProperty("java.io.tmpdir")))
@@ -51,4 +55,17 @@ class LocalDirsSuite extends SparkFunSuite with BeforeAndAfter {
assert(new File(Utils.getLocalDir(conf)).exists())
}
+ test("Utils.getLocalDir() throws an exception if any temporary directory cannot be retrieved") {
+ val path1 = "/NONEXISTENT_PATH_ONE"
+ val path2 = "/NONEXISTENT_PATH_TWO"
+ assert(!new File(path1).exists())
+ assert(!new File(path2).exists())
+ val conf = new SparkConf(false).set("spark.local.dir", s"$path1,$path2")
+ val message = intercept[IOException] {
+ Utils.getLocalDir(conf)
+ }.getMessage
+ // If any temporary directory could not be retrieved under the given paths above, it should
+ // throw an exception with the message that includes the paths.
+ assert(message.contains(s"$path1,$path2"))
+ }
}
--
GitLab