Skip to content
Snippets Groups Projects
Commit b5c5bd98 authored by Reynold Xin's avatar Reynold Xin
Browse files

Disable generate codegen since it fails my workload.

parent 91fa80fe
No related branches found
No related tags found
No related merge requests found
......@@ -119,7 +119,7 @@ case class GenerateExec(
}
}
override def supportCodegen: Boolean = generator.supportCodegen
override def supportCodegen: Boolean = false
override def inputRDDs(): Seq[RDD[InternalRow]] = {
child.asInstanceOf[CodegenSupport].inputRDDs()
......
......@@ -116,34 +116,6 @@ class WholeStageCodegenSuite extends SparkPlanTest with SharedSQLContext {
assert(ds.collect() === Array(("a", 10.0), ("b", 3.0), ("c", 1.0)))
}
test("generate should be included in WholeStageCodegen") {
import org.apache.spark.sql.functions._
val ds = spark.range(2).select(
col("id"),
explode(array(col("id") + 1, col("id") + 2)).as("value"))
val plan = ds.queryExecution.executedPlan
assert(plan.find(p =>
p.isInstanceOf[WholeStageCodegenExec] &&
p.asInstanceOf[WholeStageCodegenExec].child.isInstanceOf[GenerateExec]).isDefined)
assert(ds.collect() === Array(Row(0, 1), Row(0, 2), Row(1, 2), Row(1, 3)))
}
test("large stack generator should not use WholeStageCodegen") {
def createStackGenerator(rows: Int): SparkPlan = {
val id = UnresolvedAttribute("id")
val stack = Stack(Literal(rows) +: Seq.tabulate(rows)(i => Add(id, Literal(i))))
spark.range(500).select(Column(stack)).queryExecution.executedPlan
}
val isCodeGenerated: SparkPlan => Boolean = {
case WholeStageCodegenExec(_: GenerateExec) => true
case _ => false
}
// Only 'stack' generators that produce 50 rows or less are code generated.
assert(createStackGenerator(50).find(isCodeGenerated).isDefined)
assert(createStackGenerator(100).find(isCodeGenerated).isEmpty)
}
test("SPARK-19512 codegen for comparing structs is incorrect") {
// this would raise CompileException before the fix
spark.range(10)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment