From 182f9b7a6d3a3ee7ec7de6abc24e296aa794e4e8 Mon Sep 17 00:00:00 2001
From: Kun Xu <viper_kun@163.com>
Date: Sun, 16 Aug 2015 14:44:23 +0800
Subject: [PATCH] [SPARK-9973] [SQL] Correct in-memory columnar buffer size
The `initialSize` argument of `ColumnBuilder.initialize()` should be the
number of rows rather than bytes. However `InMemoryColumnarTableScan`
passes in a byte size, which makes Spark SQL allocate more memory than
necessary when building in-memory columnar buffers.
Author: Kun Xu <viper_kun@163.com>
Closes #8189 from viper-kun/errorSize.
---
.../apache/spark/sql/columnar/InMemoryColumnarTableScan.scala | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/columnar/InMemoryColumnarTableScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/columnar/InMemoryColumnarTableScan.scala
index d553bb6169..45f15fd04d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/columnar/InMemoryColumnarTableScan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/columnar/InMemoryColumnarTableScan.scala
@@ -121,8 +121,7 @@ private[sql] case class InMemoryRelation(
def next(): CachedBatch = {
val columnBuilders = output.map { attribute =>
val columnType = ColumnType(attribute.dataType)
- val initialBufferSize = columnType.defaultSize * batchSize
- ColumnBuilder(attribute.dataType, initialBufferSize, attribute.name, useCompression)
+ ColumnBuilder(attribute.dataType, batchSize, attribute.name, useCompression)
}.toArray
var rowCount = 0
--
GitLab