From 25c2776dd9ae3f9792048c78be2cbd958fd99841 Mon Sep 17 00:00:00 2001
From: Xingbo Jiang <xingbo.jiang@databricks.com>
Date: Wed, 28 Feb 2018 12:16:26 -0800
Subject: [PATCH] [SPARK-23523][SQL][FOLLOWUP] Minor refactor of
OptimizeMetadataOnlyQuery
## What changes were proposed in this pull request?
Inside `OptimizeMetadataOnlyQuery.getPartitionAttrs`, avoid using `zip` to generate attribute map.
Also include other minor update of comments and format.
## How was this patch tested?
Existing test cases.
Author: Xingbo Jiang <xingbo.jiang@databricks.com>
Closes #20693 from jiangxb1987/SPARK-23523.
---
.../spark/sql/execution/OptimizeMetadataOnlyQuery.scala | 2 +-
.../spark/sql/execution/datasources/HadoopFsRelation.scala | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
index 0613d9053f..dc4aff9f12 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
@@ -83,7 +83,7 @@ case class OptimizeMetadataOnlyQuery(catalog: SessionCatalog) extends Rule[Logic
private def getPartitionAttrs(
partitionColumnNames: Seq[String],
relation: LogicalPlan): Seq[Attribute] = {
- val attrMap = relation.output.map(_.name.toLowerCase(Locale.ROOT)).zip(relation.output).toMap
+ val attrMap = relation.output.map(a => a.name.toLowerCase(Locale.ROOT) -> a).toMap
partitionColumnNames.map { colName =>
attrMap.getOrElse(colName.toLowerCase(Locale.ROOT),
throw new AnalysisException(s"Unable to find the column `$colName` " +
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFsRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFsRelation.scala
index ac574b07ec..b2f73b7f8d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFsRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFsRelation.scala
@@ -67,9 +67,9 @@ case class HadoopFsRelation(
}
}
- // When data schema and partition schema have the overlapped columns, the output
- // schema respects the order of data schema for the overlapped columns, but respect
- // the data types of partition schema
+ // When data and partition schemas have overlapping columns, the output
+ // schema respects the order of the data schema for the overlapping columns, and it
+ // respects the data types of the partition schema.
val schema: StructType = {
StructType(dataSchema.map(f => overlappedPartCols.getOrElse(getColName(f), f)) ++
partitionSchema.filterNot(f => overlappedPartCols.contains(getColName(f))))
--
GitLab