From 9b4da7b7906131e42fcac109d40e4d663f57c291 Mon Sep 17 00:00:00 2001
From: "iurii.ant" <sereneant@gmail.com>
Date: Tue, 25 Jul 2017 21:43:39 +0100
Subject: [PATCH] [SPARK-21491][GRAPHX] Enhance GraphX performance: breakOut
instead of .toMap
## What changes were proposed in this pull request?
`Traversable.toMap` changed to 'collections.breakOut', that eliminates intermediate tuple collection creation, see [Stack Overflow article](https://stackoverflow.com/questions/1715681/scala-2-8-breakout).
## How was this patch tested?
Unit tests run.
No performance tests performed yet.
Please review http://spark.apache.org/contributing.html before opening a pull request.
Author: iurii.ant <sereneant@gmail.com>
Closes #18693 from SereneAnt/performance_toMap-breakOut.
---
.../scala/org/apache/spark/graphx/lib/LabelPropagation.scala | 2 +-
.../scala/org/apache/spark/graphx/lib/ShortestPaths.scala | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala
index fc7547a2c7..cb3025f8be 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala
@@ -55,7 +55,7 @@ object LabelPropagation {
val count1Val = count1.getOrElse(i, 0L)
val count2Val = count2.getOrElse(i, 0L)
i -> (count1Val + count2Val)
- }.toMap
+ }(collection.breakOut) // more efficient alternative to [[collection.Traversable.toMap]]
}
def vertexProgram(vid: VertexId, attr: Long, message: Map[VertexId, Long]): VertexId = {
if (message.isEmpty) attr else message.maxBy(_._2)._1
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala
index f0c6bcb934..4cac633aed 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala
@@ -33,10 +33,11 @@ object ShortestPaths {
private def incrementMap(spmap: SPMap): SPMap = spmap.map { case (v, d) => v -> (d + 1) }
- private def addMaps(spmap1: SPMap, spmap2: SPMap): SPMap =
+ private def addMaps(spmap1: SPMap, spmap2: SPMap): SPMap = {
(spmap1.keySet ++ spmap2.keySet).map {
k => k -> math.min(spmap1.getOrElse(k, Int.MaxValue), spmap2.getOrElse(k, Int.MaxValue))
- }.toMap
+ }(collection.breakOut) // more efficient alternative to [[collection.Traversable.toMap]]
+ }
/**
* Computes shortest paths to the given set of landmark vertices.
--
GitLab