mirror of https://github.com/apache/jclouds.git
Reauthenticate on Keystone HTTP 401 (JCLOUDS-178)
The number of retries here is not the same as for 500 errors; expected behavior is a quick fail while retaining some robustness. This fix should not reintroduce JCLOUDS-231.
This commit is contained in:
parent
952d8444d4
commit
578a77d631
|
@ -19,6 +19,8 @@ package org.jclouds.openstack.keystone.v2_0.handlers;
|
|||
import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
|
||||
import static org.jclouds.http.HttpUtils.releasePayload;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.jclouds.domain.Credentials;
|
||||
|
@ -27,9 +29,14 @@ import org.jclouds.http.HttpResponse;
|
|||
import org.jclouds.http.HttpRetryHandler;
|
||||
import org.jclouds.logging.Logger;
|
||||
import org.jclouds.openstack.keystone.v2_0.domain.Access;
|
||||
import org.jclouds.openstack.v2_0.reference.AuthHeaders;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.cache.Cache;
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
import com.google.common.collect.Multimap;
|
||||
import com.google.common.util.concurrent.Uninterruptibles;
|
||||
import com.google.inject.Inject;
|
||||
import com.google.inject.Singleton;
|
||||
|
||||
|
@ -37,13 +44,16 @@ import com.google.inject.Singleton;
|
|||
* This will parse and set an appropriate exception on the command object.
|
||||
*
|
||||
* @author Adrian Cole
|
||||
*
|
||||
* @author Zack Shoylev
|
||||
*/
|
||||
@Singleton
|
||||
public class RetryOnRenew implements HttpRetryHandler {
|
||||
@Resource
|
||||
protected Logger logger = Logger.NULL;
|
||||
|
||||
@VisibleForTesting
|
||||
final static int NUM_RETRIES = 5;
|
||||
|
||||
private final LoadingCache<Credentials, Access> authenticationResponseCache;
|
||||
|
||||
@Inject
|
||||
|
@ -51,32 +61,56 @@ public class RetryOnRenew implements HttpRetryHandler {
|
|||
this.authenticationResponseCache = authenticationResponseCache;
|
||||
}
|
||||
|
||||
/*
|
||||
* The reason retries need to be tracked is that it is possible that a token
|
||||
* can be expired at any time. The reason we track by request is that only
|
||||
* some requests might return a 401 (such as temporary URLs). However
|
||||
* consistent failures of the magnitude this code tracks should indicate a
|
||||
* problem.
|
||||
*/
|
||||
private static final Cache<HttpCommand, Integer> retryCountMap = CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(5, TimeUnit.MINUTES).build();
|
||||
|
||||
@Override
|
||||
public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) {
|
||||
boolean retry = false; // default
|
||||
try {
|
||||
switch (response.getStatusCode()) {
|
||||
case 401:
|
||||
// Do not retry on 401 from authentication request
|
||||
Multimap<String, String> headers = command.getCurrentRequest().getHeaders();
|
||||
if (headers != null && headers.containsKey("X-Auth-User")
|
||||
&& headers.containsKey("X-Auth-Key") && !headers.containsKey("X-Auth-Token")) {
|
||||
retry = false;
|
||||
if (headers != null && headers.containsKey(AuthHeaders.AUTH_USER)
|
||||
&& headers.containsKey(AuthHeaders.AUTH_KEY) && !headers.containsKey(AuthHeaders.AUTH_TOKEN)) {
|
||||
return false;
|
||||
} else {
|
||||
byte[] content = closeClientButKeepContentStream(response);
|
||||
//TODO: what is the error when the session token expires??
|
||||
if (content != null && new String(content).contains("lease renew")) {
|
||||
logger.debug("invalidating authentication token");
|
||||
closeClientButKeepContentStream(response);
|
||||
// This is not an authentication request returning 401
|
||||
// Check if we already had seen this request
|
||||
Integer count = retryCountMap.getIfPresent(command);
|
||||
|
||||
if (count == null) {
|
||||
// First time this non-authentication request failed
|
||||
logger.debug("invalidating authentication token - first time for %s", command);
|
||||
retryCountMap.put(command, 1);
|
||||
authenticationResponseCache.invalidateAll();
|
||||
retry = true;
|
||||
return true;
|
||||
} else {
|
||||
retry = false;
|
||||
// This request has failed before
|
||||
if (count + 1 >= NUM_RETRIES) {
|
||||
logger.debug("too many 401s - giving up after: %s for %s", count, command);
|
||||
return false;
|
||||
} else {
|
||||
// Retry just in case
|
||||
logger.debug("invalidating authentication token - retry %s for %s", count, command);
|
||||
retryCountMap.put(command, count + 1);
|
||||
// Wait between retries
|
||||
authenticationResponseCache.invalidateAll();
|
||||
Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
return retry;
|
||||
|
||||
return false;
|
||||
} finally {
|
||||
releasePayload(response);
|
||||
}
|
||||
|
|
|
@ -21,6 +21,7 @@ import static org.easymock.EasyMock.expect;
|
|||
import static org.easymock.EasyMock.expectLastCall;
|
||||
import static org.easymock.EasyMock.replay;
|
||||
import static org.easymock.EasyMock.verify;
|
||||
import static org.testng.Assert.assertFalse;
|
||||
import static org.testng.Assert.assertTrue;
|
||||
|
||||
import org.jclouds.domain.Credentials;
|
||||
|
@ -53,7 +54,7 @@ public class RetryOnRenewTest {
|
|||
cache.invalidateAll();
|
||||
expectLastCall();
|
||||
|
||||
expect(response.getPayload()).andReturn(Payloads.newStringPayload("token expired, please renew")).anyTimes();
|
||||
expect(response.getPayload()).andReturn(Payloads.newStringPayload("")).anyTimes();
|
||||
expect(response.getStatusCode()).andReturn(401).atLeastOnce();
|
||||
|
||||
replay(command);
|
||||
|
@ -68,4 +69,34 @@ public class RetryOnRenewTest {
|
|||
verify(response);
|
||||
verify(cache);
|
||||
}
|
||||
@Test
|
||||
public void test401ShouldRetry4Times() {
|
||||
HttpCommand command = createMock(HttpCommand.class);
|
||||
HttpRequest request = createMock(HttpRequest.class);
|
||||
HttpResponse response = createMock(HttpResponse.class);
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
LoadingCache<Credentials, Access> cache = createMock(LoadingCache.class);
|
||||
|
||||
expect(command.getCurrentRequest()).andReturn(request).anyTimes();
|
||||
expect(request.getHeaders()).andStubReturn(null);
|
||||
|
||||
cache.invalidateAll();
|
||||
expectLastCall().anyTimes();
|
||||
|
||||
expect(response.getPayload()).andReturn(Payloads.newStringPayload("")).anyTimes();
|
||||
expect(response.getStatusCode()).andReturn(401).anyTimes();
|
||||
|
||||
replay(command, request, response, cache);
|
||||
|
||||
RetryOnRenew retry = new RetryOnRenew(cache);
|
||||
|
||||
for (int i = 0; i < RetryOnRenew.NUM_RETRIES - 1; ++i) {
|
||||
assertTrue(retry.shouldRetryRequest(command, response), "Expected retry to succeed");
|
||||
}
|
||||
|
||||
assertFalse(retry.shouldRetryRequest(command, response), "Expected retry to fail on attempt " + RetryOnRenew.NUM_RETRIES);
|
||||
|
||||
verify(command, response, cache);
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue