From fae88af18445c5a88212b4644e121de4b30ce027 Mon Sep 17 00:00:00 2001
From: Terence Yim <terence@cask.co>
Date: Thu, 25 Feb 2016 13:29:30 +0000
Subject: [PATCH] [SPARK-13441][YARN] Fix NPE in yarn Client.createConfArchive
method
## What changes were proposed in this pull request?
Instead of using result of File.listFiles() directly, which may throw NPE, check for null first. If it is null, log a warning instead
## How was the this patch tested?
Ran the ./dev/run-tests locally
Tested manually on a cluster
Author: Terence Yim <terence@cask.co>
Closes #11337 from chtyim/fixes/SPARK-13441-null-check.
---
.../scala/org/apache/spark/deploy/yarn/Client.scala | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index d4ca255953..530f1d753b 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -537,9 +537,14 @@ private[spark] class Client(
sys.env.get(envKey).foreach { path =>
val dir = new File(path)
if (dir.isDirectory()) {
- dir.listFiles().foreach { file =>
- if (file.isFile && !hadoopConfFiles.contains(file.getName())) {
- hadoopConfFiles(file.getName()) = file
+ val files = dir.listFiles()
+ if (files == null) {
+ logWarning("Failed to list files under directory " + dir)
+ } else {
+ files.foreach { file =>
+ if (file.isFile && !hadoopConfFiles.contains(file.getName())) {
+ hadoopConfFiles(file.getName()) = file
+ }
}
}
}
--
GitLab