Skip to content
Snippets Groups Projects
Commit 1e9269c3 authored by jerryshao's avatar jerryshao
Browse files

reduce ZippedPartitionsRDD's getPreferredLocations complexity

parent 2ab311f4
No related branches found
No related tags found
No related merge requests found
...@@ -53,14 +53,10 @@ abstract class ZippedPartitionsBaseRDD[V: ClassManifest]( ...@@ -53,14 +53,10 @@ abstract class ZippedPartitionsBaseRDD[V: ClassManifest](
val exactMatchLocations = exactMatchPreferredLocations.reduce((x, y) => x.intersect(y)) val exactMatchLocations = exactMatchPreferredLocations.reduce((x, y) => x.intersect(y))
// Remove exact match and then do host local match. // Remove exact match and then do host local match.
val otherNodePreferredLocations = rddSplitZip.map(x => { val exactMatchHosts = exactMatchLocations.map(Utils.parseHostPort(_)._1)
x._1.preferredLocations(x._2).map(hostPort => { val matchPreferredHosts = exactMatchPreferredLocations.map(locs => locs.map(Utils.parseHostPort(_)._1))
val host = Utils.parseHostPort(hostPort)._1 .reduce((x, y) => x.intersect(y))
val otherNodeLocalLocations = matchPreferredHosts.filter { s => !exactMatchHosts.contains(s) }
if (exactMatchLocations.contains(host)) null else host
}).filter(_ != null)
})
val otherNodeLocalLocations = otherNodePreferredLocations.reduce((x, y) => x.intersect(y))
otherNodeLocalLocations ++ exactMatchLocations otherNodeLocalLocations ++ exactMatchLocations
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment