Retry each operation up to maxErrors times

Previously we only allowed maxErrors per directory, which tended to
fail with large directories.  Also rename counter to something more
intuitive.
This commit is contained in:
Andrew Gaul 2012-10-10 17:51:03 -07:00 committed by Adrian Cole
parent 4b146dfff6
commit 5dc495a2bf
1 changed files with 12 additions and 6 deletions

View File

@ -93,7 +93,7 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
Map<StorageMetadata, Exception> exceptions = Maps.newHashMap(); Map<StorageMetadata, Exception> exceptions = Maps.newHashMap();
PageSet<? extends StorageMetadata> listing; PageSet<? extends StorageMetadata> listing;
int maxErrors = 3; // TODO parameterize int maxErrors = 3; // TODO parameterize
for (int i = 0; i < maxErrors; ) { for (int numErrors = 0; numErrors < maxErrors; ) {
// fetch partial directory listing // fetch partial directory listing
try { try {
listing = connection.list(containerName, options).get(); listing = connection.list(containerName, options).get();
@ -101,11 +101,11 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
break; break;
} catch (ExecutionException ee) { } catch (ExecutionException ee) {
++i; ++numErrors;
if (i == maxErrors) { if (numErrors == maxErrors) {
throw Throwables.propagate(ee.getCause()); throw Throwables.propagate(ee.getCause());
} }
retryHandler.imposeBackoffExponentialDelay(i, message); retryHandler.imposeBackoffExponentialDelay(numErrors, message);
continue; continue;
} }
@ -155,8 +155,8 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, message); exceptions = awaitCompletion(responses, userExecutor, maxTime, logger, message);
if (!exceptions.isEmpty()) { if (!exceptions.isEmpty()) {
++i; ++numErrors;
retryHandler.imposeBackoffExponentialDelay(i, message); retryHandler.imposeBackoffExponentialDelay(numErrors, message);
continue; continue;
} }
@ -165,6 +165,12 @@ public class DeleteAllKeysInList implements ClearListStrategy, ClearContainerStr
break; break;
} }
options = options.afterMarker(marker); options = options.afterMarker(marker);
// Reset numErrors if we execute a successful iteration. This ensures
// that we only try an unsuccessful operation maxErrors times but
// allow progress with directories containing many blobs in the face
// of some failures.
numErrors = 0;
} }
if (!exceptions.isEmpty()) if (!exceptions.isEmpty())
throw new BlobRuntimeException(String.format("error %s: %s", message, exceptions)); throw new BlobRuntimeException(String.format("error %s: %s", message, exceptions));