-
- Downloads
[SPARK-12321][SQL] JSON format for TreeNode (use reflection)
An alternative solution for https://github.com/apache/spark/pull/10295 , instead of implementing json format for all logical/physical plans and expressions, use reflection to implement it in `TreeNode`. Here I use pre-order traversal to flattern a plan tree to a plan list, and add an extra field `num-children` to each plan node, so that we can reconstruct the tree from the list. example json: logical plan tree: ``` [ { "class" : "org.apache.spark.sql.catalyst.plans.logical.Sort", "num-children" : 1, "order" : [ [ { "class" : "org.apache.spark.sql.catalyst.expressions.SortOrder", "num-children" : 1, "child" : 0, "direction" : "Ascending" }, { "class" : "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children" : 0, "name" : "i", "dataType" : "integer", "nullable" : true, "metadata" : { }, "exprId" : { "id" : 10, "jvmId" : "cd1313c7-3f66-4ed7-a320-7d91e4633ac6" }, "qualifiers" : [ ] } ] ], "global" : false, "child" : 0 }, { "class" : "org.apache.spark.sql.catalyst.plans.logical.Project", "num-children" : 1, "projectList" : [ [ { "class" : "org.apache.spark.sql.catalyst.expressions.Alias", "num-children" : 1, "child" : 0, "name" : "i", "exprId" : { "id" : 10, "jvmId" : "cd1313c7-3f66-4ed7-a320-7d91e4633ac6" }, "qualifiers" : [ ] }, { "class" : "org.apache.spark.sql.catalyst.expressions.Add", "num-children" : 2, "left" : 0, "right" : 1 }, { "class" : "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children" : 0, "name" : "a", "dataType" : "integer", "nullable" : true, "metadata" : { }, "exprId" : { "id" : 0, "jvmId" : "cd1313c7-3f66-4ed7-a320-7d91e4633ac6" }, "qualifiers" : [ ] }, { "class" : "org.apache.spark.sql.catalyst.expressions.Literal", "num-children" : 0, "value" : "1", "dataType" : "integer" } ], [ { "class" : "org.apache.spark.sql.catalyst.expressions.Alias", "num-children" : 1, "child" : 0, "name" : "j", "exprId" : { "id" : 11, "jvmId" : "cd1313c7-3f66-4ed7-a320-7d91e4633ac6" }, "qualifiers" : [ ] }, { "class" : "org.apache.spark.sql.catalyst.expressions.Multiply", "num-children" : 2, "left" : 0, "right" : 1 }, { "class" : "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children" : 0, "name" : "a", "dataType" : "integer", "nullable" : true, "metadata" : { }, "exprId" : { "id" : 0, "jvmId" : "cd1313c7-3f66-4ed7-a320-7d91e4633ac6" }, "qualifiers" : [ ] }, { "class" : "org.apache.spark.sql.catalyst.expressions.Literal", "num-children" : 0, "value" : "2", "dataType" : "integer" } ] ], "child" : 0 }, { "class" : "org.apache.spark.sql.catalyst.plans.logical.LocalRelation", "num-children" : 0, "output" : [ [ { "class" : "org.apache.spark.sql.catalyst.expressions.AttributeReference", "num-children" : 0, "name" : "a", "dataType" : "integer", "nullable" : true, "metadata" : { }, "exprId" : { "id" : 0, "jvmId" : "cd1313c7-3f66-4ed7-a320-7d91e4633ac6" }, "qualifiers" : [ ] } ] ], "data" : [ ] } ] ``` Author: Wenchen Fan <wenchen@databricks.com> Closes #10311 from cloud-fan/toJson-reflection.
Showing
- sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala 53 additions, 61 deletions...scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
- sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/interfaces.scala 0 additions, 1 deletion...spark/sql/catalyst/expressions/aggregate/interfaces.scala
- sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala 41 additions, 0 deletions.../org/apache/spark/sql/catalyst/expressions/literals.scala
- sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala 4 additions, 0 deletions...che/spark/sql/catalyst/expressions/namedExpressions.scala
- sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala 2 additions, 0 deletions...scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
- sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala 257 additions, 1 deletion.../scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
- sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala 3 additions, 3 deletions.../src/main/scala/org/apache/spark/sql/types/DataType.scala
- sql/core/src/main/scala/org/apache/spark/sql/execution/ExistingRDD.scala 1 addition, 3 deletions...in/scala/org/apache/spark/sql/execution/ExistingRDD.scala
- sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarTableScan.scala 3 additions, 3 deletions...rk/sql/execution/columnar/InMemoryColumnarTableScan.scala
- sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala 100 additions, 2 deletionssql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
- sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala 5 additions, 0 deletions...est/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
- sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala 2 additions, 0 deletions...cala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
- sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala 1 addition, 1 deletion...pache/spark/sql/hive/execution/ScriptTransformation.scala
Loading
Please register or sign in to comment