From 1bfd9347822df65e76201c4c471a26488d722319 Mon Sep 17 00:00:00 2001
From: ihainan <ihainan72@gmail.com>
Date: Sun, 30 Aug 2015 08:26:14 +0100
Subject: [PATCH] [SPARK-10184] [CORE] Optimization for bounds determination in
RangePartitioner
JIRA Issue: https://issues.apache.org/jira/browse/SPARK-10184
Change `cumWeight > target` to `cumWeight >= target` in `RangePartitioner.determineBounds` method to make the output partitions more balanced.
Author: ihainan <ihainan72@gmail.com>
Closes #8397 from ihainan/opt_for_rangepartitioner.
---
core/src/main/scala/org/apache/spark/Partitioner.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core/src/main/scala/org/apache/spark/Partitioner.scala b/core/src/main/scala/org/apache/spark/Partitioner.scala
index 4b9d59975b..29e581bb57 100644
--- a/core/src/main/scala/org/apache/spark/Partitioner.scala
+++ b/core/src/main/scala/org/apache/spark/Partitioner.scala
@@ -291,7 +291,7 @@ private[spark] object RangePartitioner {
while ((i < numCandidates) && (j < partitions - 1)) {
val (key, weight) = ordered(i)
cumWeight += weight
- if (cumWeight > target) {
+ if (cumWeight >= target) {
// Skip duplicate values.
if (previousBound.isEmpty || ordering.gt(key, previousBound.get)) {
bounds += key
--
GitLab