From 76f3c735aa300d7ea6b17e64cc22d7e8fc3a8322 Mon Sep 17 00:00:00 2001
From: Matei Zaharia <matei@databricks.com>
Date: Sun, 3 Apr 2016 21:08:54 -0700
Subject: [PATCH] [SPARK-14356] Update spark.sql.execution.debug to work on
Datasets
## What changes were proposed in this pull request?
Update DebugQuery to work on Datasets of any type, not just DataFrames.
## How was this patch tested?
Added unit tests, checked in spark-shell.
Author: Matei Zaharia <matei@databricks.com>
Closes #12140 from mateiz/debug-dataset.
---
.../org/apache/spark/sql/execution/debug/package.scala | 4 ++--
.../apache/spark/sql/execution/debug/DebuggingSuite.scala | 6 ++++++
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
index 7b0c8ebdfa..17eae88b49 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
@@ -77,9 +77,9 @@ package object debug {
}
/**
- * Augments [[DataFrame]]s with debug methods.
+ * Augments [[Dataset]]s with debug methods.
*/
- implicit class DebugQuery(query: DataFrame) extends Logging {
+ implicit class DebugQuery(query: Dataset[_]) extends Logging {
def debug(): Unit = {
val plan = query.queryExecution.executedPlan
val visited = new collection.mutable.HashSet[TreeNodeRef]()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
index c0fce4b96a..8aa0114d98 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.debug
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.test.SharedSQLContext
+import org.apache.spark.sql.test.SQLTestData.TestData
class DebuggingSuite extends SparkFunSuite with SharedSQLContext {
@@ -26,6 +27,11 @@ class DebuggingSuite extends SparkFunSuite with SharedSQLContext {
testData.debug()
}
+ test("Dataset.debug()") {
+ import testImplicits._
+ testData.as[TestData].debug()
+ }
+
test("debugCodegen") {
val res = codegenString(sqlContext.range(10).groupBy("id").count().queryExecution.executedPlan)
assert(res.contains("Subtree 1 / 2"))
--
GitLab