Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

JCLOUDS-460: Add jitter to avoid thundering herd 

When issuing many simultaneous requests to Synaptic Atmos I observed:

HTTP/1.1 failed with code 500, error: AtmosError
[code=1040, message=The server is busy. Please try again.]

Previously all clients slept for fixed intervals and thus retried
around the same time.  This commit adds a random delay which should
better distribute load on the provider.
This commit is contained in:
Andrew Gaul 2013-04-18 12:54:00 -07:00 committed by Andrew Gaul
parent 9f11888f4b
commit 00847f9ffc
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java
View File

@ -19,6 +19,7 @@ package org.jclouds.http.handlers;
import static org.jclouds.http.HttpUtils.releasePayload; import static org.jclouds.http.HttpUtils.releasePayload;
import java.io.IOException; import java.io.IOException;
import java.util.Random;
import javax.annotation.Resource; import javax.annotation.Resource;
import javax.inject.Named; import javax.inject.Named;
@ -126,6 +127,9 @@ public class BackoffLimitedRetryHandler implements HttpRetryHandler, IOException
public void imposeBackoffExponentialDelay(long period, long maxPeriod, int pow, int failureCount, int max, public void imposeBackoffExponentialDelay(long period, long maxPeriod, int pow, int failureCount, int max,
String commandDescription) { String commandDescription) {
long delayMs = (long) (period * Math.pow(failureCount, pow)); long delayMs = (long) (period * Math.pow(failureCount, pow));
// Add random delay to avoid thundering herd problem when multiple
// simultaneous failed requests retry after sleeping for the same delay.
delayMs += new Random().nextInt((int) (delayMs / 10));
delayMs = delayMs > maxPeriod ? maxPeriod : delayMs; delayMs = delayMs > maxPeriod ? maxPeriod : delayMs;
logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, max, delayMs, commandDescription); logger.debug("Retry %d/%d: delaying for %d ms: %s", failureCount, max, delayMs, commandDescription);
try { try {