From ed87f6d3b48a85391628c29c43d318c26e2c6de7 Mon Sep 17 00:00:00 2001
From: yucai <yucai.yu@intel.com>
Date: Sun, 13 Dec 2015 23:08:21 -0800
Subject: [PATCH] [SPARK-12275][SQL] No plan for BroadcastHint in some
condition
When SparkStrategies.BasicOperators's "case BroadcastHint(child) => apply(child)" is hit, it only recursively invokes BasicOperators.apply with this "child". It makes many strategies have no change to process this plan, which probably leads to "No plan" issue, so we use planLater to go through all strategies.
https://issues.apache.org/jira/browse/SPARK-12275
Author: yucai <yucai.yu@intel.com>
Closes #10265 from yucai/broadcast_hint.
---
.../org/apache/spark/sql/execution/SparkStrategies.scala | 2 +-
.../scala/org/apache/spark/sql/DataFrameJoinSuite.scala | 7 +++++++
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index 25e98c0bdd..688555cf13 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -364,7 +364,7 @@ private[sql] abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
case e @ EvaluatePython(udf, child, _) =>
BatchPythonEvaluation(udf, e.output, planLater(child)) :: Nil
case LogicalRDD(output, rdd) => PhysicalRDD(output, rdd, "ExistingRDD") :: Nil
- case BroadcastHint(child) => apply(child)
+ case BroadcastHint(child) => planLater(child) :: Nil
case _ => Nil
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
index 56ad71ea4f..c70397f985 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
@@ -120,5 +120,12 @@ class DataFrameJoinSuite extends QueryTest with SharedSQLContext {
// planner should not crash without a join
broadcast(df1).queryExecution.executedPlan
+
+ // SPARK-12275: no physical plan for BroadcastHint in some condition
+ withTempPath { path =>
+ df1.write.parquet(path.getCanonicalPath)
+ val pf1 = sqlContext.read.parquet(path.getCanonicalPath)
+ assert(df1.join(broadcast(pf1)).count() === 4)
+ }
}
}
--
GitLab