From a7f8a4f5ee757450ce8d4028021441435081cf53 Mon Sep 17 00:00:00 2001
From: Michael Armbrust <michael@databricks.com>
Date: Thu, 14 Aug 2014 13:00:21 -0700
Subject: [PATCH] Revert [SPARK-3011][SQL] _temporary directory should be
filtered out by sqlContext.parquetFile
Reverts #1924 due to build failures with hadoop 0.23.
Author: Michael Armbrust <michael@databricks.com>
Closes #1949 from marmbrus/revert1924 and squashes the following commits:
6bff940 [Michael Armbrust] Revert "[SPARK-3011][SQL] _temporary directory should be filtered out by sqlContext.parquetFile"
---
.../main/scala/org/apache/spark/sql/parquet/ParquetTypes.scala | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTypes.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTypes.scala
index b0579f76da..c79a9ac2da 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTypes.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTypes.scala
@@ -378,8 +378,7 @@ private[parquet] object ParquetTypesConverter extends Logging {
val children = fs.listStatus(path).filterNot { status =>
val name = status.getPath.getName
- name(0) == '.' || name == FileOutputCommitter.SUCCEEDED_FILE_NAME ||
- name == FileOutputCommitter.TEMP_DIR_NAME
+ name(0) == '.' || name == FileOutputCommitter.SUCCEEDED_FILE_NAME
}
// NOTE (lian): Parquet "_metadata" file can be very slow if the file consists of lots of row
--
GitLab