honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix SnapshotDisruptionIT Race Condition (#37358) 

* Due to a race between retrying the snapshot creation and the failed snapshot create trying to delete the snapshot there is no guarantee that the snapshot is eventually created by retries
   * Adjusted the assertion accordingly
* Closes #36779
This commit is contained in:
Armin Braun 2019-01-11 16:09:26 +01:00 committed by GitHub
parent f4abf9628a
commit 1eba1d1df9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 18 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
server/src/test/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java
View File

@ -133,21 +133,7 @@ public class SnapshotDisruptionIT extends ESIntegTestCase {
logger.info("--> waiting for disruption to start");
assertTrue(disruptionStarted.await(1, TimeUnit.MINUTES));
logger.info("--> wait until the snapshot is done");
assertBusy(() -> {
ClusterState state = dataNodeClient().admin().cluster().prepareState().get().getState();
SnapshotsInProgress snapshots = state.custom(SnapshotsInProgress.TYPE);
SnapshotDeletionsInProgress snapshotDeletionsInProgress = state.custom(SnapshotDeletionsInProgress.TYPE);
if (snapshots != null && snapshots.entries().size() > 0) {
logger.info("Current snapshot state [{}]", snapshots.entries().get(0).state());
fail("Snapshot is still running");
} else if (snapshotDeletionsInProgress != null && snapshotDeletionsInProgress.hasDeletionsInProgress()) {
logger.info("Current snapshot deletion state [{}]", snapshotDeletionsInProgress);
fail("Snapshot deletion is still running");
} else {
logger.info("Snapshot is no longer in the cluster state");
}
}, 1, TimeUnit.MINUTES);
assertAllSnapshotsCompleted();
logger.info("--> verify that snapshot was successful or no longer exist");
assertBusy(() -> {
@ -177,14 +163,25 @@ public class SnapshotDisruptionIT extends ESIntegTestCase {
}
}
logger.info("--> verify that snapshot eventually will be created due to retries");
assertAllSnapshotsCompleted();
}
private void assertAllSnapshotsCompleted() throws Exception {
logger.info("--> wait until the snapshot is done");
assertBusy(() -> {
try {
assertSnapshotExists("test-repo", "test-snap-2");
} catch (SnapshotMissingException ex) {
throw new AssertionError(ex);
ClusterState state = dataNodeClient().admin().cluster().prepareState().get().getState();
SnapshotsInProgress snapshots = state.custom(SnapshotsInProgress.TYPE);
SnapshotDeletionsInProgress snapshotDeletionsInProgress = state.custom(SnapshotDeletionsInProgress.TYPE);
if (snapshots != null && snapshots.entries().isEmpty() == false) {
logger.info("Current snapshot state [{}]", snapshots.entries().get(0).state());
fail("Snapshot is still running");
} else if (snapshotDeletionsInProgress != null && snapshotDeletionsInProgress.hasDeletionsInProgress()) {
logger.info("Current snapshot deletion state [{}]", snapshotDeletionsInProgress);
fail("Snapshot deletion is still running");
} else {
logger.info("Snapshot is no longer in the cluster state");
}
}, 1, TimeUnit.MINUTES);
}, 1L, TimeUnit.MINUTES);
}
private void assertSnapshotExists(String repository, String snapshot) {