honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Tests: set the upper bound on the number of simulated failures in a restore test 

The number and distribution of errors in some restore test may cause restore process to continue to fail for a prolong time. This test caps the total number of simulated failures to make sure that restore is guaranteed to eventually succeed after a limited number of retries.
This commit is contained in:
Igor Motov 2015-09-02 17:47:30 -04:00
parent f202ecdc84
commit e288c9b0ce
2 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
View File

@ -669,6 +669,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
CountResponse countResponse = client.prepareCount("test-idx").get();
assertThat(countResponse.getCount(), equalTo(100L));
logger.info("--> total number of simulated failures during restore: [{}]", getFailureCount("test-repo"));
}

13
core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
View File

@ -108,6 +108,8 @@ public class MockRepository extends FsRepository {
private final double randomDataFileIOExceptionRate;
private final long maximumNumberOfFailures;
private final long waitAfterUnblock;
private final MockBlobStore mockBlobStore;
@ -127,6 +129,7 @@ public class MockRepository extends FsRepository {
super(name, overrideSettings(repositorySettings, clusterService), indexShardRepository, environment);
randomControlIOExceptionRate = repositorySettings.settings().getAsDouble("random_control_io_exception_rate", 0.0);
randomDataFileIOExceptionRate = repositorySettings.settings().getAsDouble("random_data_file_io_exception_rate", 0.0);
maximumNumberOfFailures = repositorySettings.settings().getAsLong("max_failure_number", 100L);
blockOnControlFiles = repositorySettings.settings().getAsBoolean("block_on_control", false);
blockOnDataFiles = repositorySettings.settings().getAsBoolean("block_on_data", false);
blockOnInitialization = repositorySettings.settings().getAsBoolean("block_on_init", false);
@ -160,8 +163,8 @@ public class MockRepository extends FsRepository {
return settingsBuilder().put(settings).put("location", location.toAbsolutePath()).build();
}
private void addFailure() {
failureCounter.incrementAndGet();
private long incrementAndGetFailureCount() {
return failureCounter.incrementAndGet();
}
@Override
@ -269,9 +272,8 @@ public class MockRepository extends FsRepository {
private void maybeIOExceptionOrBlock(String blobName) throws IOException {
if (blobName.startsWith("__")) {
if (shouldFail(blobName, randomDataFileIOExceptionRate)) {
if (shouldFail(blobName, randomDataFileIOExceptionRate) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) {
logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path());
addFailure();
throw new IOException("Random IOException");
} else if (blockOnDataFiles) {
logger.info("blocking I/O operation for file [{}] at path [{}]", blobName, path());
@ -286,9 +288,8 @@ public class MockRepository extends FsRepository {
}
}
} else {
if (shouldFail(blobName, randomControlIOExceptionRate)) {
if (shouldFail(blobName, randomControlIOExceptionRate) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) {
logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path());
addFailure();
throw new IOException("Random IOException");
} else if (blockOnControlFiles) {
logger.info("blocking I/O operation for file [{}] at path [{}]", blobName, path());