Repository: spark Updated Branches: refs/heads/branch-2.1 3857d5ba8 -> 063a98e52
[SPARK-18900][FLAKY-TEST] StateStoreSuite.maintenance ## What changes were proposed in this pull request? It was pretty flaky before 10 days ago. https://spark-tests.appspot.com/test-details?suite_name=org.apache.spark.sql.execution.streaming.state.StateStoreSuite&test_name=maintenance Since no code changes went into this code path to not be so flaky, I'm just increasing the timeouts such that load related flakiness shouldn't be a problem. As you may see from the testing, I haven't been able to reproduce it. ## How was this patch tested? 2000 retries 5 times Author: Burak Yavuz <[email protected]> Closes #16314 from brkyvz/maint-flaky. (cherry picked from commit b2dd8ec6b2c05c996e2d7c0bf8db0073c1ee0b94) Signed-off-by: Tathagata Das <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/063a98e5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/063a98e5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/063a98e5 Branch: refs/heads/branch-2.1 Commit: 063a98e52189567245ca28696f0b61a7ae103f3f Parents: 3857d5b Author: Burak Yavuz <[email protected]> Authored: Tue Dec 20 19:28:18 2016 -0800 Committer: Tathagata Das <[email protected]> Committed: Tue Dec 20 19:28:29 2016 -0800 ---------------------------------------------------------------------- .../sql/execution/streaming/state/StateStoreSuite.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/063a98e5/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala index 3404b11..4863a4c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/StateStoreSuite.scala @@ -395,6 +395,8 @@ class StateStoreSuite extends SparkFunSuite with BeforeAndAfter with PrivateMeth } } + val timeoutDuration = 60 seconds + quietly { withSpark(new SparkContext(conf)) { sc => withCoordinatorRef(sc) { coordinatorRef => @@ -403,7 +405,7 @@ class StateStoreSuite extends SparkFunSuite with BeforeAndAfter with PrivateMeth // Generate sufficient versions of store for snapshots generateStoreVersions() - eventually(timeout(10 seconds)) { + eventually(timeout(timeoutDuration)) { // Store should have been reported to the coordinator assert(coordinatorRef.getLocation(storeId).nonEmpty, "active instance was not reported") @@ -422,14 +424,14 @@ class StateStoreSuite extends SparkFunSuite with BeforeAndAfter with PrivateMeth generateStoreVersions() // Earliest delta file should get cleaned up - eventually(timeout(10 seconds)) { + eventually(timeout(timeoutDuration)) { assert(!fileExists(provider, 1, isSnapshot = false), "earliest file not deleted") } // If driver decides to deactivate all instances of the store, then this instance // should be unloaded coordinatorRef.deactivateInstances(dir) - eventually(timeout(10 seconds)) { + eventually(timeout(timeoutDuration)) { assert(!StateStore.isLoaded(storeId)) } @@ -439,7 +441,7 @@ class StateStoreSuite extends SparkFunSuite with BeforeAndAfter with PrivateMeth // If some other executor loads the store, then this instance should be unloaded coordinatorRef.reportActiveInstance(storeId, "other-host", "other-exec") - eventually(timeout(10 seconds)) { + eventually(timeout(timeoutDuration)) { assert(!StateStore.isLoaded(storeId)) } @@ -450,7 +452,7 @@ class StateStoreSuite extends SparkFunSuite with BeforeAndAfter with PrivateMeth } // Verify if instance is unloaded if SparkContext is stopped - eventually(timeout(10 seconds)) { + eventually(timeout(timeoutDuration)) { require(SparkEnv.get === null) assert(!StateStore.isLoaded(storeId)) assert(!StateStore.isMaintenanceRunning) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
