From b7c89a7f0ca73153dce36e0f01b81a3947ee1189 Mon Sep 17 00:00:00 2001
From: chutium <teng.qiu@gmail.com>
Date: Fri, 8 Aug 2014 13:31:08 -0700
Subject: [PATCH] [SPARK-2700] [SQL] Hidden files (such as
.impala_insert_staging) should be filtered out by sqlContext.parquetFile
Author: chutium <teng.qiu@gmail.com>
Closes #1691 from chutium/SPARK-2700 and squashes the following commits:
b76ae8c [chutium] [SPARK-2700] [SQL] fixed styling issue
d75a8bd [chutium] [SPARK-2700] [SQL] Hidden files (such as .impala_insert_staging) should be filtered out by sqlContext.parquetFile
---
.../scala/org/apache/spark/sql/parquet/ParquetTypes.scala | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTypes.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTypes.scala
index aaef1a1d47..2867dc0a8b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTypes.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTypes.scala
@@ -373,8 +373,9 @@ private[parquet] object ParquetTypesConverter extends Logging {
}
ParquetRelation.enableLogForwarding()
- val children = fs.listStatus(path).filterNot {
- _.getPath.getName == FileOutputCommitter.SUCCEEDED_FILE_NAME
+ val children = fs.listStatus(path).filterNot { status =>
+ val name = status.getPath.getName
+ name(0) == '.' || name == FileOutputCommitter.SUCCEEDED_FILE_NAME
}
// NOTE (lian): Parquet "_metadata" file can be very slow if the file consists of lots of row
--
GitLab