From e288c9b0ce65ae006068333792a97479bf82bc03 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 2 Sep 2015 17:47:30 -0400 Subject: [PATCH] Tests: set the upper bound on the number of simulated failures in a restore test The number and distribution of errors in some restore test may cause restore process to continue to fail for a prolong time. This test caps the total number of simulated failures to make sure that restore is guaranteed to eventually succeed after a limited number of retries. --- .../snapshots/SharedClusterSnapshotRestoreIT.java | 1 + .../snapshots/mockstore/MockRepository.java | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index b327c064e2e..aea102d5e26 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -669,6 +669,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); CountResponse countResponse = client.prepareCount("test-idx").get(); assertThat(countResponse.getCount(), equalTo(100L)); + logger.info("--> total number of simulated failures during restore: [{}]", getFailureCount("test-repo")); } diff --git a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java index e430827d881..be10f6b399e 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java +++ b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java @@ -108,6 +108,8 @@ public class MockRepository extends FsRepository { private final double randomDataFileIOExceptionRate; + private final long maximumNumberOfFailures; + private final long waitAfterUnblock; private final MockBlobStore mockBlobStore; @@ -127,6 +129,7 @@ public class MockRepository extends FsRepository { super(name, overrideSettings(repositorySettings, clusterService), indexShardRepository, environment); randomControlIOExceptionRate = repositorySettings.settings().getAsDouble("random_control_io_exception_rate", 0.0); randomDataFileIOExceptionRate = repositorySettings.settings().getAsDouble("random_data_file_io_exception_rate", 0.0); + maximumNumberOfFailures = repositorySettings.settings().getAsLong("max_failure_number", 100L); blockOnControlFiles = repositorySettings.settings().getAsBoolean("block_on_control", false); blockOnDataFiles = repositorySettings.settings().getAsBoolean("block_on_data", false); blockOnInitialization = repositorySettings.settings().getAsBoolean("block_on_init", false); @@ -160,8 +163,8 @@ public class MockRepository extends FsRepository { return settingsBuilder().put(settings).put("location", location.toAbsolutePath()).build(); } - private void addFailure() { - failureCounter.incrementAndGet(); + private long incrementAndGetFailureCount() { + return failureCounter.incrementAndGet(); } @Override @@ -269,9 +272,8 @@ public class MockRepository extends FsRepository { private void maybeIOExceptionOrBlock(String blobName) throws IOException { if (blobName.startsWith("__")) { - if (shouldFail(blobName, randomDataFileIOExceptionRate)) { + if (shouldFail(blobName, randomDataFileIOExceptionRate) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) { logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path()); - addFailure(); throw new IOException("Random IOException"); } else if (blockOnDataFiles) { logger.info("blocking I/O operation for file [{}] at path [{}]", blobName, path()); @@ -286,9 +288,8 @@ public class MockRepository extends FsRepository { } } } else { - if (shouldFail(blobName, randomControlIOExceptionRate)) { + if (shouldFail(blobName, randomControlIOExceptionRate) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) { logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path()); - addFailure(); throw new IOException("Random IOException"); } else if (blockOnControlFiles) { logger.info("blocking I/O operation for file [{}] at path [{}]", blobName, path());