Skip to content
Snippets Groups Projects
Commit 18b3b00e authored by Juliusz Sompolski's avatar Juliusz Sompolski Committed by gatorsmile
Browse files

[SPARK-21272] SortMergeJoin LeftAnti does not update numOutputRows

## What changes were proposed in this pull request?

Updating numOutputRows metric was missing from one return path of LeftAnti SortMergeJoin.

## How was this patch tested?

Non-zero output rows manually seen in metrics.

Author: Juliusz Sompolski <julek@databricks.com>

Closes #18494 from juliuszsompolski/SPARK-21272.
parent 6a06c4b0
No related branches found
No related tags found
No related merge requests found
......@@ -290,6 +290,7 @@ case class SortMergeJoinExec(
currentLeftRow = smjScanner.getStreamedRow
val currentRightMatches = smjScanner.getBufferedMatches
if (currentRightMatches == null || currentRightMatches.length == 0) {
numOutputRows += 1
return true
}
var found = false
......
......@@ -483,6 +483,18 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
}
}
test("SortMergeJoin(left-anti) metrics") {
val anti = testData2.filter("a > 2")
withTempView("antiData") {
anti.createOrReplaceTempView("antiData")
val df = spark.sql(
"SELECT * FROM testData2 ANTI JOIN antiData ON testData2.a = antiData.a")
testSparkPlanMetrics(df, 1, Map(
0L -> ("SortMergeJoin", Map("number of output rows" -> 4L)))
)
}
}
test("save metrics") {
withTempPath { file =>
// person creates a temporary view. get the DF before listing previous execution IDs
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment