Skip to content
Snippets Groups Projects
Commit 11fa8741 authored by Tathagata Das's avatar Tathagata Das Committed by Shixiong Zhu
Browse files

[SQL][HOTFIX] Fix flakiness in StateStoreRDDSuite

## What changes were proposed in this pull request?
StateStoreCoordinator.reportActiveInstance is async, so subsequence state checks must be in eventually.
## How was this patch tested?
Jenkins tests

Author: Tathagata Das <tathagata.das1565@gmail.com>

Closes #11924 from tdas/state-store-flaky-fix.
parent b5f8c36e
No related branches found
No related tags found
No related merge requests found
......@@ -71,7 +71,6 @@ class StateStoreCoordinatorSuite extends SparkFunSuite with SharedSparkContext {
assert(coordinatorRef.verifyIfInstanceActive(id1, exec) === true)
assert(coordinatorRef.verifyIfInstanceActive(id2, exec) === true)
assert(coordinatorRef.verifyIfInstanceActive(id3, exec) === true)
}
coordinatorRef.deactivateInstances("x")
......
......@@ -23,6 +23,8 @@ import java.nio.file.Files
import scala.util.Random
import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
import org.scalatest.concurrent.Eventually._
import org.scalatest.time.SpanSugar._
import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
import org.apache.spark.LocalSparkContext._
......@@ -121,9 +123,12 @@ class StateStoreRDDSuite extends SparkFunSuite with BeforeAndAfter with BeforeAn
val coordinatorRef = sqlContext.streams.stateStoreCoordinator
coordinatorRef.reportActiveInstance(StateStoreId(path, opId, 0), "host1", "exec1")
coordinatorRef.reportActiveInstance(StateStoreId(path, opId, 1), "host2", "exec2")
assert(
coordinatorRef.getLocation(StateStoreId(path, opId, 0)) ===
Some(ExecutorCacheTaskLocation("host1", "exec1").toString))
eventually(timeout(10 seconds)) {
assert(
coordinatorRef.getLocation(StateStoreId(path, opId, 0)) ===
Some(ExecutorCacheTaskLocation("host1", "exec1").toString))
}
val rdd = makeRDD(sc, Seq("a", "b", "a")).mapPartitionWithStateStore(
increment, path, opId, storeVersion = 0, keySchema, valueSchema)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment