From 00847f9ffc5fade4e63530b79c7d38962c1de08f Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Thu, 18 Apr 2013 12:54:00 -0700 Subject: [PATCH] JCLOUDS-460: Add jitter to avoid thundering herd When issuing many simultaneous requests to Synaptic Atmos I observed: HTTP/1.1 failed with code 500, error: AtmosError [code=1040, message=The server is busy. Please try again.] Previously all clients slept for fixed intervals and thus retried around the same time. This commit adds a random delay which should better distribute load on the provider. --- .../org/jclouds/http/handlers/BackoffLimitedRetryHandler.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java b/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java index 51620421e2..5d8b5b10e0 100644 --- a/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java +++ b/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java @@ -19,6 +19,7 @@ package org.jclouds.http.handlers; import static org.jclouds.http.HttpUtils.releasePayload; import java.io.IOException; +import java.util.Random; import javax.annotation.Resource; import javax.inject.Named; @@ -126,6 +127,9 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException public void imposeBackoffExponentialDelay(long period, long maxPeriod, int pow, int failureCount, int max, String commandDescription) { long delayMs = (long) (period * Math.pow(failureCount, pow)); + // Add random delay to avoid thundering herd problem when multiple + // simultaneous failed requests retry after sleeping for the same delay. + delayMs += new Random().nextInt((int) (delayMs / 10)); delayMs = delayMs > maxPeriod ? maxPeriod : delayMs; logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, max, delayMs, commandDescription); try {