From 39e4e7e4d89077a637c4cad3a986e0e3447d1ae7 Mon Sep 17 00:00:00 2001
From: Steve Lindemann <steve.lindemann@engineersgatelp.com>
Date: Mon, 6 Jul 2015 10:17:05 -0700
Subject: [PATCH] [SPARK-8841] [SQL] Fix partition pruning percentage log
message
When pruning partitions for a query plan, a message is logged indicating what how many partitions were selected based on predicate criteria, and what percent were pruned.
The current release erroneously uses `1 - total/selected` to compute this quantity, leading to nonsense messages like "pruned -1000% partitions". The fix is simple and obvious.
Author: Steve Lindemann <steve.lindemann@engineersgatelp.com>
Closes #7227 from srlindemann/master and squashes the following commits:
c788061 [Steve Lindemann] fix percentPruned log message
---
.../scala/org/apache/spark/sql/sources/DataSourceStrategy.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/DataSourceStrategy.scala
index ce16e050c5..66f7ba9014 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/sources/DataSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/DataSourceStrategy.scala
@@ -65,7 +65,7 @@ private[sql] object DataSourceStrategy extends Strategy with Logging {
logInfo {
val total = t.partitionSpec.partitions.length
val selected = selectedPartitions.length
- val percentPruned = (1 - total.toDouble / selected.toDouble) * 100
+ val percentPruned = (1 - selected.toDouble / total.toDouble) * 100
s"Selected $selected partitions out of $total, pruned $percentPruned% partitions."
}
--
GitLab