HTTPCLIENT-1105: Built-in way to do auto-retry for certain status codes

Contributed by Dan Checkoway <dcheckoway at gmail.com>


git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk@1141078 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Oleg Kalnichevski 2011-06-29 12:56:38 +00:00
parent 97557998d5
commit a03573ca24
4 changed files with 607 additions and 0 deletions

View File

@ -0,0 +1,66 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.client;
import org.apache.http.HttpResponse;
import org.apache.http.protocol.HttpContext;
/**
* Strategy interface that allows API users to plug in their own logic to
* control whether or not a retry should automatically be done, how many times
* it should be retried and so on.
*
*/
public interface ServiceUnavailableRetryStrategy {
/**
* Determines if a method should be retried given the response from the target server.
*
* @param response the response from the target server
* @param executionCount the number of times this method has been
* unsuccessfully executed
* @param context the context for the request execution
* @return <code>true</code> if the method should be retried, <code>false</code>
* otherwise
*/
boolean retryRequest(HttpResponse response, int executionCount, HttpContext context);
/**
* @return The interval between the subsequent auto-retries.
*/
long getRetryInterval();
/**
* @return the multiplying factor for continuous errors situations returned
* by the server-side. Each retry attempt will multiply this factor
* with the retry interval.
*/
int getRetryFactor();
}

View File

@ -0,0 +1,171 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.impl.client;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.annotation.ThreadSafe;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.HttpContext;
import org.apache.http.client.*;
@ThreadSafe
public class AutoRetryHttpClient implements HttpClient {
private final HttpClient backend;
private final ServiceUnavailableRetryStrategy retryStrategy;
private final Log log = LogFactory.getLog(getClass());
public AutoRetryHttpClient(
final HttpClient client, final ServiceUnavailableRetryStrategy retryStrategy) {
super();
if (client == null) {
throw new IllegalArgumentException("HttpClient may not be null");
}
if (retryStrategy == null) {
throw new IllegalArgumentException(
"ServiceUnavailableRetryStrategy may not be null");
}
this.backend = client;
this.retryStrategy = retryStrategy;
}
/**
* Constructs a {@code AutoRetryHttpClient} with default caching settings that
* stores cache entries in memory and uses a vanilla
* {@link DefaultHttpClient} for backend requests.
*/
public AutoRetryHttpClient() {
this(new DefaultHttpClient(), new DefaultServiceUnavailableRetryStrategy());
}
/**
* Constructs a {@code AutoRetryHttpClient} with the given caching options that
* stores cache entries in memory and uses a vanilla
* {@link DefaultHttpClient} for backend requests.
*
* @param config
* retry configuration module options
*/
public AutoRetryHttpClient(ServiceUnavailableRetryStrategy config) {
this(new DefaultHttpClient(), config);
}
/**
* Constructs a {@code AutoRetryHttpClient} with default caching settings that
* stores cache entries in memory and uses the given {@link HttpClient} for
* backend requests.
*
* @param client
* used to make origin requests
*/
public AutoRetryHttpClient(HttpClient client) {
this(client, new DefaultServiceUnavailableRetryStrategy());
}
public HttpResponse execute(HttpHost target, HttpRequest request)
throws IOException {
HttpContext defaultContext = null;
return execute(target, request, defaultContext);
}
public <T> T execute(HttpHost target, HttpRequest request,
ResponseHandler<? extends T> responseHandler) throws IOException {
return execute(target, request, responseHandler, null);
}
public <T> T execute(HttpHost target, HttpRequest request,
ResponseHandler<? extends T> responseHandler, HttpContext context)
throws IOException {
HttpResponse resp = execute(target, request, context);
return responseHandler.handleResponse(resp);
}
public HttpResponse execute(HttpUriRequest request) throws IOException {
HttpContext context = null;
return execute(request, context);
}
public HttpResponse execute(HttpUriRequest request, HttpContext context)
throws IOException {
URI uri = request.getURI();
HttpHost httpHost = new HttpHost(uri.getHost(), uri.getPort(),
uri.getScheme());
return execute(httpHost, request, context);
}
public <T> T execute(HttpUriRequest request,
ResponseHandler<? extends T> responseHandler) throws IOException {
return execute(request, responseHandler, null);
}
public <T> T execute(HttpUriRequest request,
ResponseHandler<? extends T> responseHandler, HttpContext context)
throws IOException {
HttpResponse resp = execute(request, context);
return responseHandler.handleResponse(resp);
}
public HttpResponse execute(HttpHost target, HttpRequest request,
HttpContext context) throws IOException {
for (int c = 1;; c++) {
HttpResponse response = backend.execute(target, request, context);
if (retryStrategy.retryRequest(response, c, context)) {
long nextInterval = retryStrategy.getRetryInterval() * retryStrategy.getRetryFactor();
try {
log.trace("Wait for " + nextInterval);
Thread.sleep(nextInterval);
} catch (InterruptedException e) {
throw new InterruptedIOException(e.getMessage());
}
} else {
return response;
}
}
}
public ClientConnectionManager getConnectionManager() {
return backend.getConnectionManager();
}
public HttpParams getParams() {
return backend.getParams();
}
}

View File

@ -0,0 +1,129 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.impl.client;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.HttpResponse;
import org.apache.http.client.ServiceUnavailableRetryStrategy;
import org.apache.http.protocol.HttpContext;
/**
* Default implementation for the <code>ServiceUnavailableRetryStrategy</code>
* interface.
*
*/
public class DefaultServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy {
private List<Integer> retryResponseCodes = new ArrayList<Integer>();
/**
* Maximum number of allowed retries if the server responds with a HTTP code
* in our retry code list. Default value is 1.
*/
private int maxRetries = 1;
/**
* Retry interval between subsequent requests, in milliseconds. Default
* value is 1 second.
*/
private long retryInterval = 1000;
/**
* Multiplying factor for continuous errors situations returned by the
* server-side. Each retry attempt will multiply this factor with the retry
* interval. Default value is 1, which means each retry interval will be
* constant.
*/
private int retryFactor = 1;
public void addResponseCodeForRetry(int responseCode) {
retryResponseCodes.add(responseCode);
}
public boolean retryRequest(final HttpResponse response, int executionCount, final HttpContext context) {
return executionCount <= maxRetries && retryResponseCodes.contains(
response.getStatusLine().getStatusCode());
}
/**
* @return The maximum number of allowed auto-retries in case the server
* response code is contained in this retry strategy. Default value
* is 1, meaning no-retry.
*/
public int getMaxRetries() {
return maxRetries;
}
public void setMaxRetries(int maxRetries) {
if (maxRetries < 1) {
throw new IllegalArgumentException(
"MaxRetries should be greater than 1");
}
this.maxRetries = maxRetries;
}
/**
* @return The interval between the subsequent auto-retries. Default value
* is 1000 ms, meaning there is 1 second X
* <code>getRetryFactor()</code> between the subsequent auto
* retries.
*
*/
public long getRetryInterval() {
return retryInterval;
}
public void setRetryInterval(long retryInterval) {
if (retryInterval < 1) {
throw new IllegalArgumentException(
"Retry interval should be greater than 1");
}
this.retryInterval = retryInterval;
}
/**
* @return the multiplying factor for continuous errors situations returned
* by the server-side. Each retry attempt will multiply this factor
* with the retry interval. default value is 1, meaning the retry
* intervals are constant.
*/
public int getRetryFactor() {
return retryFactor;
}
public void setRetryFactor(int factor) {
if (factor < 1) {
throw new IllegalArgumentException(
"Retry factor should be greater than 1");
}
this.retryFactor = factor;
}
}

View File

@ -0,0 +1,241 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.impl.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.Date;
import java.util.Random;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.HttpVersion;
import org.apache.http.client.HttpClient;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.impl.client.AutoRetryHttpClient;
import org.apache.http.impl.client.DefaultServiceUnavailableRetryStrategy;
import org.apache.http.impl.cookie.DateUtils;
import org.apache.http.message.BasicHttpRequest;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.junit.Before;
import org.junit.Test;
public class TestAutoRetryHttpClient{
private AutoRetryHttpClient impl;
private HttpClient mockBackend;
private HttpHost host;
@Before
public void setUp() {
mockBackend = mock(HttpClient.class);
host = new HttpHost("foo.example.com");
}
static HttpResponse make200Response() {
HttpResponse out = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK");
out.setHeader("Date", DateUtils.formatDate(new Date()));
out.setHeader("Server", "MockOrigin/1.0");
out.setHeader("Content-Length", "128");
out.setEntity(makeBody(128));
return out;
}
static HttpRequest makeDefaultRequest() {
return new BasicHttpRequest("GET","/",HttpVersion.HTTP_1_1);
}
static HttpResponse make500Response() {
return new BasicHttpResponse(HttpVersion.HTTP_1_1,
HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal Server Error");
}
static HttpResponse make503Response() {
return new BasicHttpResponse(HttpVersion.HTTP_1_1,
HttpStatus.SC_SERVICE_UNAVAILABLE, "Service Unavailable");
}
static HttpResponse make502Response() {
return new BasicHttpResponse(HttpVersion.HTTP_1_1,
HttpStatus.SC_BAD_GATEWAY, "Bad Gateway");
}
/** Generates a response body with random content.
* @param nbytes length of the desired response body
* @return an {@link HttpEntity}
*/
static HttpEntity makeBody(int nbytes) {
return new ByteArrayEntity(getRandomBytes(nbytes));
}
static byte[] getRandomBytes(int nbytes) {
byte[] bytes = new byte[nbytes];
(new Random()).nextBytes(bytes);
return bytes;
}
@Test
public void testAddOneStatusInRetryConfig(){
DefaultServiceUnavailableRetryStrategy retryStrategy = new DefaultServiceUnavailableRetryStrategy();
retryStrategy.addResponseCodeForRetry(503);
HttpContext context = new BasicHttpContext();
HttpResponse response1 = new BasicHttpResponse(HttpVersion.HTTP_1_1, 503, "Oppsie");
assertTrue(retryStrategy.retryRequest(response1, 1, context));
HttpResponse response2 = new BasicHttpResponse(HttpVersion.HTTP_1_1, 502, "Oppsie");
assertFalse(retryStrategy.retryRequest(response2, 1, context));
}
@Test
public void testAddMultipleStatusesInRetryConfig(){
DefaultServiceUnavailableRetryStrategy retryStrategy = new DefaultServiceUnavailableRetryStrategy();
retryStrategy.addResponseCodeForRetry(503);
retryStrategy.addResponseCodeForRetry(502);
HttpContext context = new BasicHttpContext();
HttpResponse response1 = new BasicHttpResponse(HttpVersion.HTTP_1_1, 503, "Oppsie");
assertTrue(retryStrategy.retryRequest(response1, 1, context));
HttpResponse response2 = new BasicHttpResponse(HttpVersion.HTTP_1_1, 502, "Oppsie");
assertTrue(retryStrategy.retryRequest(response2, 1, context));
HttpResponse response3 = new BasicHttpResponse(HttpVersion.HTTP_1_1, 500, "Oppsie");
assertFalse(retryStrategy.retryRequest(response3, 1, context));
}
@Test
public void testNoAutoRetry() throws java.io.IOException{
DefaultServiceUnavailableRetryStrategy retryStrategy = new DefaultServiceUnavailableRetryStrategy();
retryStrategy.setMaxRetries(2);
retryStrategy.setRetryInterval(100);
impl = new AutoRetryHttpClient(mockBackend,retryStrategy);
HttpRequest req1 = makeDefaultRequest();
HttpResponse resp1 = make500Response();
HttpResponse resp2 = make200Response();
when(mockBackend.execute(host, req1,(HttpContext)null)).thenReturn(resp1).thenReturn(resp2);
HttpResponse result = impl.execute(host, req1);
verify(mockBackend,times(1)).execute(host, req1,(HttpContext)null);
assertEquals(resp1,result);
assertEquals(500,result.getStatusLine().getStatusCode());
}
@Test
public void testMultipleAutoRetry() throws java.io.IOException{
DefaultServiceUnavailableRetryStrategy retryStrategy = new DefaultServiceUnavailableRetryStrategy();
retryStrategy.addResponseCodeForRetry(503);
retryStrategy.addResponseCodeForRetry(502);
retryStrategy.setMaxRetries(5);
retryStrategy.setRetryInterval(100);
impl = new AutoRetryHttpClient(mockBackend,retryStrategy);
HttpRequest req1 = makeDefaultRequest();
HttpResponse resp1 = make503Response();
HttpResponse resp2 = make502Response();
HttpResponse resp3 = make200Response();
when(mockBackend.execute(host, req1,(HttpContext)null)).thenReturn(resp1).thenReturn(resp2).thenReturn(resp3);
HttpResponse result = impl.execute(host, req1);
verify(mockBackend,times(3)).execute(host, req1,(HttpContext)null);
assertEquals(resp3,result);
assertEquals(200,result.getStatusLine().getStatusCode());
}
@Test
public void test503SingleAutoRetry() throws java.io.IOException{
DefaultServiceUnavailableRetryStrategy retryStrategy = new DefaultServiceUnavailableRetryStrategy();
retryStrategy.addResponseCodeForRetry(503);
retryStrategy.setMaxRetries(5);
retryStrategy.setRetryInterval(100);
impl = new AutoRetryHttpClient(mockBackend,retryStrategy);
HttpRequest req1 = makeDefaultRequest();
HttpResponse resp1 = make503Response();
HttpResponse resp2 = make200Response();
when(mockBackend.execute(host, req1,(HttpContext)null)).thenReturn(resp1).thenReturn(resp2);
HttpResponse result = impl.execute(host, req1);
verify(mockBackend,times(2)).execute(host, req1,(HttpContext)null);
assertEquals(resp2,result);
assertEquals(200,result.getStatusLine().getStatusCode());
}
@Test
public void testRetryInterval() throws java.io.IOException{
DefaultServiceUnavailableRetryStrategy retryStrategy = new DefaultServiceUnavailableRetryStrategy();
retryStrategy.addResponseCodeForRetry(503);
retryStrategy.setMaxRetries(5);
retryStrategy.setRetryFactor(2);
retryStrategy.setRetryInterval(100); // 0.1 seconds
impl = new AutoRetryHttpClient(mockBackend,retryStrategy);
HttpRequest req1 = makeDefaultRequest();
HttpResponse resp1 = make503Response();
HttpResponse resp2 = make200Response();
when(mockBackend.execute(host, req1,(HttpContext)null)).thenReturn(resp1).thenReturn(resp2);
long currentTime = System.currentTimeMillis();
HttpResponse result = impl.execute(host, req1);
long elapsedTime = System.currentTimeMillis();
verify(mockBackend,times(2)).execute(host, req1,(HttpContext)null);
assertEquals(resp2,result);
assertEquals(200,result.getStatusLine().getStatusCode());
assertTrue((elapsedTime - currentTime) > 100);
}
}