Skip to content
Snippets Groups Projects
Commit 9451fed5 authored by Andrew Or's avatar Andrew Or Committed by Josh Rosen
Browse files

[SPARK-13344][TEST] Fix harmless accumulator not found exceptions

See [JIRA](https://issues.apache.org/jira/browse/SPARK-13344) for more detail. This was caused by #10835.

Author: Andrew Or <andrew@databricks.com>

Closes #11222 from andrewor14/fix-test-accum-exceptions.
parent 97ee85da
No related branches found
No related tags found
No related merge requests found
...@@ -34,6 +34,14 @@ import org.apache.spark.serializer.JavaSerializer ...@@ -34,6 +34,14 @@ import org.apache.spark.serializer.JavaSerializer
class AccumulatorSuite extends SparkFunSuite with Matchers with LocalSparkContext { class AccumulatorSuite extends SparkFunSuite with Matchers with LocalSparkContext {
import AccumulatorParam._ import AccumulatorParam._
override def afterEach(): Unit = {
try {
Accumulators.clear()
} finally {
super.afterEach()
}
}
implicit def setAccum[A]: AccumulableParam[mutable.Set[A], A] = implicit def setAccum[A]: AccumulableParam[mutable.Set[A], A] =
new AccumulableParam[mutable.Set[A], A] { new AccumulableParam[mutable.Set[A], A] {
def addInPlace(t1: mutable.Set[A], t2: mutable.Set[A]) : mutable.Set[A] = { def addInPlace(t1: mutable.Set[A], t2: mutable.Set[A]) : mutable.Set[A] = {
......
...@@ -28,6 +28,14 @@ class InternalAccumulatorSuite extends SparkFunSuite with LocalSparkContext { ...@@ -28,6 +28,14 @@ class InternalAccumulatorSuite extends SparkFunSuite with LocalSparkContext {
import InternalAccumulator._ import InternalAccumulator._
import AccumulatorParam._ import AccumulatorParam._
override def afterEach(): Unit = {
try {
Accumulators.clear()
} finally {
super.afterEach()
}
}
test("get param") { test("get param") {
assert(getParam(EXECUTOR_DESERIALIZE_TIME) === LongAccumulatorParam) assert(getParam(EXECUTOR_DESERIALIZE_TIME) === LongAccumulatorParam)
assert(getParam(EXECUTOR_RUN_TIME) === LongAccumulatorParam) assert(getParam(EXECUTOR_RUN_TIME) === LongAccumulatorParam)
......
...@@ -18,14 +18,26 @@ ...@@ -18,14 +18,26 @@
package org.apache.spark package org.apache.spark
// scalastyle:off // scalastyle:off
import org.scalatest.{FunSuite, Outcome} import org.scalatest.{BeforeAndAfterAll, FunSuite, Outcome}
/** /**
* Base abstract class for all unit tests in Spark for handling common functionality. * Base abstract class for all unit tests in Spark for handling common functionality.
*/ */
private[spark] abstract class SparkFunSuite extends FunSuite with Logging { private[spark] abstract class SparkFunSuite
extends FunSuite
with BeforeAndAfterAll
with Logging {
// scalastyle:on // scalastyle:on
protected override def afterAll(): Unit = {
try {
// Avoid leaking map entries in tests that use accumulators without SparkContext
Accumulators.clear()
} finally {
super.afterAll()
}
}
/** /**
* Log the suite name and the test name before and after each test. * Log the suite name and the test name before and after each test.
* *
...@@ -42,8 +54,6 @@ private[spark] abstract class SparkFunSuite extends FunSuite with Logging { ...@@ -42,8 +54,6 @@ private[spark] abstract class SparkFunSuite extends FunSuite with Logging {
test() test()
} finally { } finally {
logInfo(s"\n\n===== FINISHED $shortSuiteName: '$testName' =====\n") logInfo(s"\n\n===== FINISHED $shortSuiteName: '$testName' =====\n")
// Avoid leaking map entries in tests that use accumulators without SparkContext
Accumulators.clear()
} }
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment