From 716c88aa147762f7f617adf34a17edd681d9a4ff Mon Sep 17 00:00:00 2001
From: CodingCat <zhunansjtu@gmail.com>
Date: Sun, 15 Jun 2014 23:47:58 -0700
Subject: [PATCH] SPARK-2039: apply output dir existence checking for all
output formats
https://issues.apache.org/jira/browse/SPARK-2039
apply output dir existence checking for all output formats
Author: CodingCat <zhunansjtu@gmail.com>
Closes #1088 from CodingCat/SPARK-2039 and squashes the following commits:
c52747a [CodingCat] apply output dir existence checking for all output formats
---
.../main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
index b6ad9b6c3e..fe36c80e0b 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
@@ -787,8 +787,7 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)])
val outfmt = job.getOutputFormatClass
val jobFormat = outfmt.newInstance
- if (self.conf.getBoolean("spark.hadoop.validateOutputSpecs", true) &&
- jobFormat.isInstanceOf[NewFileOutputFormat[_, _]]) {
+ if (self.conf.getBoolean("spark.hadoop.validateOutputSpecs", true)) {
// FileOutputFormat ignores the filesystem parameter
jobFormat.checkOutputSpecs(job)
}
@@ -854,8 +853,7 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)])
logDebug("Saving as hadoop file of type (" + keyClass.getSimpleName + ", " +
valueClass.getSimpleName + ")")
- if (self.conf.getBoolean("spark.hadoop.validateOutputSpecs", true) &&
- outputFormatInstance.isInstanceOf[FileOutputFormat[_, _]]) {
+ if (self.conf.getBoolean("spark.hadoop.validateOutputSpecs", true)) {
// FileOutputFormat ignores the filesystem parameter
val ignoredFs = FileSystem.get(conf)
conf.getOutputFormat.checkOutputSpecs(ignoredFs, conf)
--
GitLab