From fc29b896dae08b957ed15fa681b46162600a4050 Mon Sep 17 00:00:00 2001
From: Davies Liu <davies@databricks.com>
Date: Wed, 18 May 2016 15:45:59 -0700
Subject: [PATCH] [SPARK-15392][SQL] fix default value of size estimation of
logical plan
## What changes were proposed in this pull request?
We use autoBroadcastJoinThreshold + 1L as the default value of size estimation, that is not good in 2.0, because we will calculate the size based on size of schema, then the estimation could be less than autoBroadcastJoinThreshold if you have an SELECT on top of an DataFrame created from RDD.
This PR change the default value to Long.MaxValue.
## How was this patch tested?
Added regression tests.
Author: Davies Liu <davies@databricks.com>
Closes #13179 from davies/fix_default_size.
---
.../scala/org/apache/spark/sql/internal/SQLConf.scala | 3 +--
.../test/scala/org/apache/spark/sql/DataFrameSuite.scala | 9 +++++++++
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 7933d12e28..a7f461381b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -605,8 +605,7 @@ private[sql] class SQLConf extends Serializable with CatalystConf with Logging {
def enableRadixSort: Boolean = getConf(RADIX_SORT_ENABLED)
- def defaultSizeInBytes: Long =
- getConf(DEFAULT_SIZE_IN_BYTES, autoBroadcastJoinThreshold + 1L)
+ def defaultSizeInBytes: Long = getConf(DEFAULT_SIZE_IN_BYTES, Long.MaxValue)
def isParquetBinaryAsString: Boolean = getConf(PARQUET_BINARY_AS_STRING)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index f573abf859..df029e44c5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -1476,4 +1476,13 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
getMessage()
assert(e1.startsWith("Path does not exist"))
}
+
+ test("SPARK-15392: DataFrame created from RDD should not be broadcasted") {
+ val rdd = sparkContext.range(1, 100).map(i => Row(i, i))
+ val df = spark.createDataFrame(rdd, new StructType().add("a", LongType).add("b", LongType))
+ assert(df.queryExecution.analyzed.statistics.sizeInBytes >
+ spark.wrapped.conf.autoBroadcastJoinThreshold)
+ assert(df.selectExpr("a").queryExecution.analyzed.statistics.sizeInBytes >
+ spark.wrapped.conf.autoBroadcastJoinThreshold)
+ }
}
--
GitLab