Skip to content
Snippets Groups Projects
Commit c3e9a120 authored by Davies Liu's avatar Davies Liu Committed by Davies Liu
Browse files

[SPARK-9831] [SQL] fix serialization with empty broadcast

Author: Davies Liu <davies@databricks.com>

Closes #8117 from davies/fix_serialization and squashes the following commits:

d21ac71 [Davies Liu] fix serialization with empty broadcast
parent 74a293f4
No related branches found
No related tags found
No related merge requests found
......@@ -299,7 +299,7 @@ private[joins] final class UnsafeHashedRelation(
binaryMap = new BytesToBytesMap(
taskMemoryManager,
shuffleMemoryManager,
nKeys * 2, // reduce hash collision
(nKeys * 1.5 + 1).toInt, // reduce hash collision
pageSizeBytes)
var i = 0
......
......@@ -103,4 +103,21 @@ class HashedRelationSuite extends SparkFunSuite {
assert(hashed2.get(unsafeData(2)) === data2)
assert(numDataRows.value.value === data.length)
}
test("test serialization empty hash map") {
val os = new ByteArrayOutputStream()
val out = new ObjectOutputStream(os)
val hashed = new UnsafeHashedRelation(
new java.util.HashMap[UnsafeRow, CompactBuffer[UnsafeRow]])
hashed.writeExternal(out)
out.flush()
val in = new ObjectInputStream(new ByteArrayInputStream(os.toByteArray))
val hashed2 = new UnsafeHashedRelation()
hashed2.readExternal(in)
val schema = StructType(StructField("a", IntegerType, true) :: Nil)
val toUnsafe = UnsafeProjection.create(schema)
val row = toUnsafe(InternalRow(0))
assert(hashed2.get(row) === null)
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment