From e96a70d5ab2e2b43a2df17a550fa9ed2ee0001c4 Mon Sep 17 00:00:00 2001
From: Yin Huai <yhuai@databricks.com>
Date: Tue, 1 Dec 2015 17:18:45 -0800
Subject: [PATCH] [SPARK-11596][SQL] In TreeNode's argString, if a TreeNode is
not a child of the current TreeNode, we should only return the simpleString.
In TreeNode's argString, if a TreeNode is not a child of the current TreeNode, we will only return the simpleString.
I tested the [following case provided by Cristian](https://issues.apache.org/jira/browse/SPARK-11596?focusedCommentId=15019241&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-15019241).
```
val c = (1 to 20).foldLeft[Option[DataFrame]] (None) { (curr, idx) =>
println(s"PROCESSING >>>>>>>>>>> $idx")
val df = sqlContext.sparkContext.parallelize((0 to 10).zipWithIndex).toDF("A", "B")
val union = curr.map(_.unionAll(df)).getOrElse(df)
union.cache()
Some(union)
}
c.get.explain(true)
```
Without the change, `c.get.explain(true)` took 100s. With the change, `c.get.explain(true)` took 26ms.
https://issues.apache.org/jira/browse/SPARK-11596
Author: Yin Huai <yhuai@databricks.com>
Closes #10079 from yhuai/SPARK-11596.
---
.../scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index f1cea07976..ad2bd78430 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -380,7 +380,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
/** Returns a string representing the arguments to this node, minus any children */
def argString: String = productIterator.flatMap {
case tn: TreeNode[_] if containsChild(tn) => Nil
- case tn: TreeNode[_] if tn.toString contains "\n" => s"(${tn.simpleString})" :: Nil
+ case tn: TreeNode[_] => s"(${tn.simpleString})" :: Nil
case seq: Seq[BaseType] if seq.toSet.subsetOf(children.toSet) => Nil
case seq: Seq[_] => seq.mkString("[", ",", "]") :: Nil
case set: Set[_] => set.mkString("{", ",", "}") :: Nil
--
GitLab