Skip to content
Snippets Groups Projects
Commit 628bb5ca authored by Matei Zaharia's avatar Matei Zaharia
Browse files

Allow null keys in Spark's reduce and group by

parent e2a67a80
No related branches found
No related tags found
No related merge requests found
......@@ -8,12 +8,16 @@ abstract class Partitioner extends Serializable {
class HashPartitioner(partitions: Int) extends Partitioner {
def numPartitions = partitions
def getPartition(key: Any) = {
val mod = key.hashCode % partitions
if (mod < 0) {
mod + partitions
def getPartition(key: Any): Int = {
if (key == null) {
return 0
} else {
mod // Guard against negative hash codes
val mod = key.hashCode % partitions
if (mod < 0) {
mod + partitions
} else {
mod // Guard against negative hash codes
}
}
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment