Skip to content
Snippets Groups Projects
Commit fc7edc9e authored by Sandy Ryza's avatar Sandy Ryza Committed by Reynold Xin
Browse files

SPARK-2519. Eliminate pattern-matching on Tuple2 in performance-critical...

... aggregation code

Author: Sandy Ryza <sandy@cloudera.com>

Closes #1435 from sryza/sandy-spark-2519 and squashes the following commits:

640706a [Sandy Ryza] SPARK-2519. Eliminate pattern-matching on Tuple2 in performance-critical aggregation code
parent 1c5739f6
No related branches found
No related tags found
No related merge requests found
...@@ -56,8 +56,8 @@ case class Aggregator[K, V, C] ( ...@@ -56,8 +56,8 @@ case class Aggregator[K, V, C] (
} else { } else {
val combiners = new ExternalAppendOnlyMap[K, V, C](createCombiner, mergeValue, mergeCombiners) val combiners = new ExternalAppendOnlyMap[K, V, C](createCombiner, mergeValue, mergeCombiners)
while (iter.hasNext) { while (iter.hasNext) {
val (k, v) = iter.next() val pair = iter.next()
combiners.insert(k, v) combiners.insert(pair._1, pair._2)
} }
// TODO: Make this non optional in a future release // TODO: Make this non optional in a future release
Option(context).foreach(c => c.taskMetrics.memoryBytesSpilled = combiners.memoryBytesSpilled) Option(context).foreach(c => c.taskMetrics.memoryBytesSpilled = combiners.memoryBytesSpilled)
...@@ -85,8 +85,8 @@ case class Aggregator[K, V, C] ( ...@@ -85,8 +85,8 @@ case class Aggregator[K, V, C] (
} else { } else {
val combiners = new ExternalAppendOnlyMap[K, C, C](identity, mergeCombiners, mergeCombiners) val combiners = new ExternalAppendOnlyMap[K, C, C](identity, mergeCombiners, mergeCombiners)
while (iter.hasNext) { while (iter.hasNext) {
val (k, c) = iter.next() val pair = iter.next()
combiners.insert(k, c) combiners.insert(pair._1, pair._2)
} }
// TODO: Make this non optional in a future release // TODO: Make this non optional in a future release
Option(context).foreach(c => c.taskMetrics.memoryBytesSpilled = combiners.memoryBytesSpilled) Option(context).foreach(c => c.taskMetrics.memoryBytesSpilled = combiners.memoryBytesSpilled)
......
...@@ -268,10 +268,10 @@ class ExternalAppendOnlyMap[K, V, C]( ...@@ -268,10 +268,10 @@ class ExternalAppendOnlyMap[K, V, C](
private def mergeIfKeyExists(key: K, baseCombiner: C, buffer: StreamBuffer): C = { private def mergeIfKeyExists(key: K, baseCombiner: C, buffer: StreamBuffer): C = {
var i = 0 var i = 0
while (i < buffer.pairs.length) { while (i < buffer.pairs.length) {
val (k, c) = buffer.pairs(i) val pair = buffer.pairs(i)
if (k == key) { if (pair._1 == key) {
buffer.pairs.remove(i) buffer.pairs.remove(i)
return mergeCombiners(baseCombiner, c) return mergeCombiners(baseCombiner, pair._2)
} }
i += 1 i += 1
} }
...@@ -293,9 +293,11 @@ class ExternalAppendOnlyMap[K, V, C]( ...@@ -293,9 +293,11 @@ class ExternalAppendOnlyMap[K, V, C](
} }
// Select a key from the StreamBuffer that holds the lowest key hash // Select a key from the StreamBuffer that holds the lowest key hash
val minBuffer = mergeHeap.dequeue() val minBuffer = mergeHeap.dequeue()
val (minPairs, minHash) = (minBuffer.pairs, minBuffer.minKeyHash) val minPairs = minBuffer.pairs
val minHash = minBuffer.minKeyHash
val minPair = minPairs.remove(0) val minPair = minPairs.remove(0)
var (minKey, minCombiner) = minPair val minKey = minPair._1
var minCombiner = minPair._2
assert(getKeyHashCode(minPair) == minHash) assert(getKeyHashCode(minPair) == minHash)
// For all other streams that may have this key (i.e. have the same minimum key hash), // For all other streams that may have this key (i.e. have the same minimum key hash),
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment