Skip to content
Snippets Groups Projects
Commit d540dfbf authored by Wang Gengliang's avatar Wang Gengliang Committed by Wenchen Fan
Browse files

[SPARK-21273][SQL][FOLLOW-UP] Add missing test cases back and revise code style

## What changes were proposed in this pull request?

Add missing test cases back and revise code style

Follow up the previous PR: https://github.com/apache/spark/pull/18479

## How was this patch tested?

Unit test

Please review http://spark.apache.org/contributing.html before opening a pull request.

Author: Wang Gengliang <ltnwgl@gmail.com>

Closes #18548 from gengliangwang/stat_propagation_revise.
parent b8e4d567
No related branches found
No related tags found
No related merge requests found
......@@ -38,10 +38,10 @@ trait LogicalPlanVisitor[T] {
case p: Range => visitRange(p)
case p: Repartition => visitRepartition(p)
case p: RepartitionByExpression => visitRepartitionByExpr(p)
case p: ResolvedHint => visitHint(p)
case p: Sample => visitSample(p)
case p: ScriptTransformation => visitScriptTransform(p)
case p: Union => visitUnion(p)
case p: ResolvedHint => visitHint(p)
case p: LogicalPlan => default(p)
}
......
......@@ -78,6 +78,37 @@ class BasicStatsEstimationSuite extends PlanTest with StatsEstimationTestBase {
checkStats(globalLimit, stats)
}
test("sample estimation") {
val sample = Sample(0.0, 0.5, withReplacement = false, (math.random * 1000).toLong, plan)
checkStats(sample, Statistics(sizeInBytes = 60, rowCount = Some(5)))
// Child doesn't have rowCount in stats
val childStats = Statistics(sizeInBytes = 120)
val childPlan = DummyLogicalPlan(childStats, childStats)
val sample2 =
Sample(0.0, 0.11, withReplacement = false, (math.random * 1000).toLong, childPlan)
checkStats(sample2, Statistics(sizeInBytes = 14))
}
test("estimate statistics when the conf changes") {
val expectedDefaultStats =
Statistics(
sizeInBytes = 40,
rowCount = Some(10),
attributeStats = AttributeMap(Seq(
AttributeReference("c1", IntegerType)() -> ColumnStat(10, Some(1), Some(10), 0, 4, 4))))
val expectedCboStats =
Statistics(
sizeInBytes = 4,
rowCount = Some(1),
attributeStats = AttributeMap(Seq(
AttributeReference("c1", IntegerType)() -> ColumnStat(1, Some(5), Some(5), 0, 4, 4))))
val plan = DummyLogicalPlan(defaultStats = expectedDefaultStats, cboStats = expectedCboStats)
checkStats(
plan, expectedStatsCboOn = expectedCboStats, expectedStatsCboOff = expectedDefaultStats)
}
/** Check estimated stats when cbo is turned on/off. */
private def checkStats(
plan: LogicalPlan,
......@@ -99,3 +130,17 @@ class BasicStatsEstimationSuite extends PlanTest with StatsEstimationTestBase {
private def checkStats(plan: LogicalPlan, expectedStats: Statistics): Unit =
checkStats(plan, expectedStats, expectedStats)
}
/**
* This class is used for unit-testing the cbo switch, it mimics a logical plan which computes
* a simple statistics or a cbo estimated statistics based on the conf.
*/
private case class DummyLogicalPlan(
defaultStats: Statistics,
cboStats: Statistics)
extends LeafNode {
override def output: Seq[Attribute] = Nil
override def computeStats(): Statistics = if (conf.cboEnabled) cboStats else defaultStats
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment