Skip to content
Snippets Groups Projects
Commit 9451fed5 authored by Andrew Or's avatar Andrew Or Committed by Josh Rosen
Browse files

[SPARK-13344][TEST] Fix harmless accumulator not found exceptions

See [JIRA](https://issues.apache.org/jira/browse/SPARK-13344) for more detail. This was caused by #10835.

Author: Andrew Or <andrew@databricks.com>

Closes #11222 from andrewor14/fix-test-accum-exceptions.
parent 97ee85da
No related branches found
No related tags found
No related merge requests found
......@@ -34,6 +34,14 @@ import org.apache.spark.serializer.JavaSerializer
class AccumulatorSuite extends SparkFunSuite with Matchers with LocalSparkContext {
import AccumulatorParam._
override def afterEach(): Unit = {
try {
Accumulators.clear()
} finally {
super.afterEach()
}
}
implicit def setAccum[A]: AccumulableParam[mutable.Set[A], A] =
new AccumulableParam[mutable.Set[A], A] {
def addInPlace(t1: mutable.Set[A], t2: mutable.Set[A]) : mutable.Set[A] = {
......
......@@ -28,6 +28,14 @@ class InternalAccumulatorSuite extends SparkFunSuite with LocalSparkContext {
import InternalAccumulator._
import AccumulatorParam._
override def afterEach(): Unit = {
try {
Accumulators.clear()
} finally {
super.afterEach()
}
}
test("get param") {
assert(getParam(EXECUTOR_DESERIALIZE_TIME) === LongAccumulatorParam)
assert(getParam(EXECUTOR_RUN_TIME) === LongAccumulatorParam)
......
......@@ -18,14 +18,26 @@
package org.apache.spark
// scalastyle:off
import org.scalatest.{FunSuite, Outcome}
import org.scalatest.{BeforeAndAfterAll, FunSuite, Outcome}
/**
* Base abstract class for all unit tests in Spark for handling common functionality.
*/
private[spark] abstract class SparkFunSuite extends FunSuite with Logging {
private[spark] abstract class SparkFunSuite
extends FunSuite
with BeforeAndAfterAll
with Logging {
// scalastyle:on
protected override def afterAll(): Unit = {
try {
// Avoid leaking map entries in tests that use accumulators without SparkContext
Accumulators.clear()
} finally {
super.afterAll()
}
}
/**
* Log the suite name and the test name before and after each test.
*
......@@ -42,8 +54,6 @@ private[spark] abstract class SparkFunSuite extends FunSuite with Logging {
test()
} finally {
logInfo(s"\n\n===== FINISHED $shortSuiteName: '$testName' =====\n")
// Avoid leaking map entries in tests that use accumulators without SparkContext
Accumulators.clear()
}
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment