From 5f88f056275aa49b2126f13b38a681af11e6b666 Mon Sep 17 00:00:00 2001 From: Jonathan Moore Date: Thu, 16 Dec 2010 11:54:30 +0000 Subject: [PATCH] HTTPCLIENT-975: committed patch for stale-while-revalidate from Michajlo Matijkiw (michajlo_matijkiw at comcast dot com). Stale-while-revalidate functionality is currently off by default until we can add bounding to the revalidation queue (or make it configurable). git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk@1049941 13f79535-47bb-0310-9956-ffa450edef68 --- .../cache/AsynchronousValidationRequest.java | 97 +++++++++ .../client/cache/AsynchronousValidator.java | 129 +++++++++++ .../http/impl/client/cache/CacheConfig.java | 28 ++- .../impl/client/cache/CacheKeyGenerator.java | 2 +- .../client/cache/CacheValidityPolicy.java | 19 ++ .../impl/client/cache/CachingHttpClient.java | 21 ++ .../TestAsynchronousValidationRequest.java | 126 +++++++++++ .../cache/TestAsynchronousValidator.java | 204 ++++++++++++++++++ .../client/cache/TestCacheValidityPolicy.java | 59 ++++- .../client/cache/TestCachingHttpClient.java | 10 + .../client/cache/TestRFC5861Compliance.java | 50 +++++ 11 files changed, 741 insertions(+), 4 deletions(-) create mode 100644 httpclient-cache/src/main/java/org/apache/http/impl/client/cache/AsynchronousValidationRequest.java create mode 100644 httpclient-cache/src/main/java/org/apache/http/impl/client/cache/AsynchronousValidator.java create mode 100644 httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestAsynchronousValidationRequest.java create mode 100644 httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestAsynchronousValidator.java diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/AsynchronousValidationRequest.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/AsynchronousValidationRequest.java new file mode 100644 index 000000000..fede49e51 --- /dev/null +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/AsynchronousValidationRequest.java @@ -0,0 +1,97 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.http.impl.client.cache; + +import java.io.IOException; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.ProtocolException; +import org.apache.http.client.cache.HttpCacheEntry; +import org.apache.http.protocol.HttpContext; + +/** + * Class used to represent an asynchronous revalidation event, such as with "stale-while-revalidate" + */ +class AsynchronousValidationRequest implements Runnable { + private final AsynchronousValidator parent; + private final CachingHttpClient cachingClient; + private final HttpHost target; + private final HttpRequest request; + private final HttpContext context; + private final HttpCacheEntry cacheEntry; + private final String identifier; + + private final Log log = LogFactory.getLog(getClass()); + + /** + * Used internally by {@link AsynchronousValidator} to schedule a revalidation. Once revalidation + * is complete, the {@link Set} bookKeeping will be locked and the {@link String} identifier will be + * removed from it. + * + * @param cachingClient + * @param target + * @param request + * @param context + * @param cacheEntry + * @param bookKeeping + * @param identifier + */ + AsynchronousValidationRequest(AsynchronousValidator parent, + CachingHttpClient cachingClient, HttpHost target, + HttpRequest request, HttpContext context, + HttpCacheEntry cacheEntry, + String identifier) { + this.parent = parent; + this.cachingClient = cachingClient; + this.target = target; + this.request = request; + this.context = context; + this.cacheEntry = cacheEntry; + this.identifier = identifier; + } + + public void run() { + try { + cachingClient.revalidateCacheEntry(target, request, context, cacheEntry); + } catch (IOException ioe) { + log.debug("Asynchronous revalidation failed due to exception: " + ioe); + } catch (ProtocolException pe) { + log.error("ProtocolException thrown during asynchronous revalidation: " + pe); + } finally { + parent.markComplete(identifier); + } + } + + String getIdentifier() { + return identifier; + } + +} diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/AsynchronousValidator.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/AsynchronousValidator.java new file mode 100644 index 000000000..dfdecbae7 --- /dev/null +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/AsynchronousValidator.java @@ -0,0 +1,129 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.http.impl.client.cache; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.client.cache.HttpCacheEntry; +import org.apache.http.protocol.HttpContext; + +/** + * Class used for asynchronous revalidations to be used when the "stale- + * while-revalidate" directive is present + */ +public class AsynchronousValidator { + private final CachingHttpClient cachingClient; + private final ExecutorService executor; + private final Set queued; + private final CacheKeyGenerator cacheKeyGenerator; + + private final Log log = LogFactory.getLog(getClass()); + + /** + * Create AsynchronousValidator which will make revalidation requests using the supplied {@link CachingHttpClient}, and + * a {@link ThreadPoolExecutor} returned by {@link Executors#newFixedThreadPool(int)}. + * @param cachingClient + * @param numThreads + */ + public AsynchronousValidator(CachingHttpClient cachingClient, int numThreads) { + this(cachingClient, Executors.newFixedThreadPool(numThreads)); + } + + /** + * Create AsynchronousValidator which will make revalidation requests using the supplied {@link CachingHttpClient} and + * {@link ExecutorService}. + * @param cachingClient + * @param executor + */ + AsynchronousValidator(CachingHttpClient cachingClient, ExecutorService executor) { + this.cachingClient = cachingClient; + this.executor = executor; + this.queued = new HashSet(); + this.cacheKeyGenerator = new CacheKeyGenerator(); + } + + /** + * Schedules an asynchronous revalidation + * + * @param target + * @param request + * @param context + * @param entry + */ + public synchronized void revalidateCacheEntry(HttpHost target, HttpRequest request, HttpContext context, HttpCacheEntry entry) { + // getVariantURI will fall back on getURI if no variants exist + String uri = cacheKeyGenerator.getVariantURI(target, request, entry); + + if (!queued.contains(uri)) { + AsynchronousValidationRequest revalidationRequest = new AsynchronousValidationRequest( + this, cachingClient, target, request, context, entry, uri); + + try { + executor.execute(revalidationRequest); + queued.add(uri); + } catch (RejectedExecutionException ree) { + log.debug("Revalidation for [" + uri + "] not scheduled: " + ree); + } + } + } + + /** + * Will remove identifier from internal list of jobs in progress. This is meant to be called + * by {@link AsynchrnousValidationRequest#run()} once the revalidation is complete, using the identifier + * passed in durinc constructions. + * @param identifier + */ + synchronized void markComplete(String identifier) { + queued.remove(identifier); + } + + /** + * Get the set of identifiers (URIs) for revalidations + * @return + */ + Set getScheduledIdentifiers() { + return Collections.unmodifiableSet(queued); + } + + /** + * Return underlying {@link ExecutorService} + * @return + */ + ExecutorService getExecutor() { + return executor; + } +} diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java index aee6ea76e..16707d58b 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java @@ -52,14 +52,21 @@ public class CacheConfig { public final static boolean DEFAULT_HEURISTIC_CACHING_ENABLED = false; /** Default coefficient used to heuristically determine freshness lifetime from - * cache entry. + * cache entry. */ public final static float DEFAULT_HEURISTIC_COEFFICIENT = 0.1f; - /** Default lifetime to be assumed when we cannot calculate freshness heuristically + /** Default lifetime in seconds to be assumed when we cannot calculate freshness + * heuristically */ public final static long DEFAULT_HEURISTIC_LIFETIME = 0; + /** Default number of worker threads to allow for background revalidations + * resulting from the stale-while-revalidate directive; 0 disables handling + * asynchronous revalidations. + */ + private static final int DEFAULT_STALE_WHILE_REVALIDATE_WORKERS = 0; + private int maxObjectSizeBytes = DEFAULT_MAX_OBJECT_SIZE_BYTES; private int maxCacheEntries = DEFAULT_MAX_CACHE_ENTRIES; private int maxUpdateRetries = DEFAULT_MAX_UPDATE_RETRIES; @@ -67,6 +74,7 @@ public class CacheConfig { private float heuristicCoefficient = DEFAULT_HEURISTIC_COEFFICIENT; private long heuristicDefaultLifetime = DEFAULT_HEURISTIC_LIFETIME; private boolean isSharedCache = true; + private int staleWhileRevalidateWorkers = DEFAULT_STALE_WHILE_REVALIDATE_WORKERS; /** * Returns the current maximum object size that will be cached. @@ -173,5 +181,21 @@ public class CacheConfig { this.heuristicDefaultLifetime = heuristicDefaultLifetime; } + /** + * Set number of worker threads to allow for background revalidations resulting from, + * the stale-while-revalidate directive, 0 disables handling of directive + * @return + */ + public int getStaleWhileRevalidateWorkers() { + return staleWhileRevalidateWorkers; + } + + /** + * Get number of worker threads to allow for background revalidations resulting from, + * the stale-while-revalidate directive, 0 disables handling of directive + */ + public void setStaleWhileRevalidateWorkers(int staleWhileRevalidateWorkers) { + this.staleWhileRevalidateWorkers = staleWhileRevalidateWorkers; + } } diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheKeyGenerator.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheKeyGenerator.java index a6c92658e..647debd85 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheKeyGenerator.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheKeyGenerator.java @@ -132,7 +132,7 @@ class CacheKeyGenerator { * * @param host The host for this request * @param req the {@link HttpRequest} - * @param entry the parent entry used to track the varients + * @param entry the parent entry used to track the variants * @return String the extracted variant URI */ public String getVariantURI(HttpHost host, HttpRequest req, HttpCacheEntry entry) { diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java index ea9f4d763..5c8c60089 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java @@ -120,6 +120,25 @@ class CacheValidityPolicy { return hasCacheControlDirective(entry, "proxy-revalidate"); } + public boolean mayReturnStaleWhileRevalidating(final HttpCacheEntry entry, Date now) { + for (Header h : entry.getHeaders("Cache-Control")) { + for(HeaderElement elt : h.getElements()) { + if ("stale-while-revalidate".equalsIgnoreCase(elt.getName())) { + try { + int allowedStalenessLifetime = Integer.parseInt(elt.getValue()); + if (getStalenessSecs(entry, now) <= allowedStalenessLifetime) { + return true; + } + } catch (NumberFormatException nfe) { + // skip malformed directive + } + } + } + } + + return false; + } + public boolean mayReturnStaleIfError(HttpRequest request, HttpCacheEntry entry, Date now) { long stalenessSecs = getStalenessSecs(entry, now); diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java index d4cc31b02..03b95bb63 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java @@ -95,6 +95,8 @@ public class CachingHttpClient implements HttpClient { private final ResponseProtocolCompliance responseCompliance; private final RequestProtocolCompliance requestCompliance; + private final AsynchronousValidator asynchRevalidator; + private final Log log = LogFactory.getLog(getClass()); CachingHttpClient( @@ -124,6 +126,8 @@ public class CachingHttpClient implements HttpClient { this.responseCompliance = new ResponseProtocolCompliance(); this.requestCompliance = new RequestProtocolCompliance(); + + this.asynchRevalidator = makeAsynchronousValidator(config.getStaleWhileRevalidateWorkers()); } public CachingHttpClient() { @@ -193,6 +197,15 @@ public class CachingHttpClient implements HttpClient { this.conditionalRequestBuilder = conditionalRequestBuilder; this.responseCompliance = responseCompliance; this.requestCompliance = requestCompliance; + this.asynchRevalidator = makeAsynchronousValidator(config.getStaleWhileRevalidateWorkers()); + } + + private AsynchronousValidator makeAsynchronousValidator( + int numWorkers) { + if (numWorkers > 0) { + return new AsynchronousValidator(this, numWorkers); + } + return null; } /** @@ -461,6 +474,14 @@ public class CachingHttpClient implements HttpClient { log.debug("Revalidating the cache entry"); try { + if (asynchRevalidator != null && validityPolicy.mayReturnStaleWhileRevalidating(entry, now)) { + final HttpResponse resp = responseGenerator.generateResponse(entry); + resp.addHeader(HeaderConstants.WARNING, "110 localhost \"Response is stale\""); + + asynchRevalidator.revalidateCacheEntry(target, request, context, entry); + + return resp; + } return revalidateCacheEntry(target, request, context, entry); } catch (IOException ioex) { if (validityPolicy.mustRevalidate(entry) diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestAsynchronousValidationRequest.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestAsynchronousValidationRequest.java new file mode 100644 index 000000000..12ffa6d02 --- /dev/null +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestAsynchronousValidationRequest.java @@ -0,0 +1,126 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.http.impl.client.cache; + +import java.io.IOException; + +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.ProtocolException; +import org.apache.http.client.cache.HttpCacheEntry; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.protocol.HttpContext; +import org.easymock.classextension.EasyMock; +import org.junit.Before; +import org.junit.Test; + +public class TestAsynchronousValidationRequest { + + private AsynchronousValidator mockParent; + private CachingHttpClient mockClient; + private HttpHost target; + private HttpRequest request; + private HttpContext mockContext; + private HttpCacheEntry mockCacheEntry; + + @Before + public void setUp() { + mockParent = EasyMock.createMock(AsynchronousValidator.class); + mockClient = EasyMock.createMock(CachingHttpClient.class); + target = new HttpHost("foo.example.com"); + request = new HttpGet("/"); + mockContext = EasyMock.createMock(HttpContext.class); + mockCacheEntry = EasyMock.createMock(HttpCacheEntry.class); + } + + @Test + public void testRunCallsCachingClientAndRemovesIdentifier() throws ProtocolException, IOException { + String identifier = "foo"; + + AsynchronousValidationRequest asynchRequest = new AsynchronousValidationRequest( + mockParent, mockClient, target, request, mockContext, mockCacheEntry, + identifier); + + // response not used + EasyMock.expect(mockClient.revalidateCacheEntry(target, request, mockContext, mockCacheEntry)).andReturn(null); + mockParent.markComplete(identifier); + + replayMocks(); + asynchRequest.run(); + verifyMocks(); + } + + @Test + public void testRunGracefullyHandlesProtocolException() throws IOException, ProtocolException { + String identifier = "foo"; + + AsynchronousValidationRequest impl = new AsynchronousValidationRequest( + mockParent, mockClient, target, request, mockContext, mockCacheEntry, + identifier); + + // response not used + EasyMock.expect( + mockClient.revalidateCacheEntry(target, request, mockContext, + mockCacheEntry)).andThrow(new ProtocolException()); + mockParent.markComplete(identifier); + + replayMocks(); + impl.run(); + verifyMocks(); + } + + @Test + public void testRunGracefullyHandlesIOException() throws IOException, ProtocolException { + String identifier = "foo"; + + AsynchronousValidationRequest impl = new AsynchronousValidationRequest( + mockParent, mockClient, target, request, mockContext, mockCacheEntry, + identifier); + + // response not used + EasyMock.expect( + mockClient.revalidateCacheEntry(target, request, mockContext, + mockCacheEntry)).andThrow(new IOException()); + mockParent.markComplete(identifier); + + replayMocks(); + impl.run(); + verifyMocks(); + } + + public void replayMocks() { + EasyMock.replay(mockClient); + EasyMock.replay(mockContext); + EasyMock.replay(mockCacheEntry); + } + + public void verifyMocks() { + EasyMock.verify(mockClient); + EasyMock.verify(mockContext); + EasyMock.verify(mockCacheEntry); + } +} diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestAsynchronousValidator.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestAsynchronousValidator.java new file mode 100644 index 000000000..03ba5ff2a --- /dev/null +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestAsynchronousValidator.java @@ -0,0 +1,204 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.http.impl.client.cache; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; + +import junit.framework.Assert; + +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.ProtocolException; +import org.apache.http.client.cache.HeaderConstants; +import org.apache.http.client.cache.HttpCacheEntry; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.message.BasicHeader; +import org.apache.http.protocol.HttpContext; +import org.easymock.Capture; +import org.easymock.classextension.EasyMock; +import org.junit.Before; +import org.junit.Test; + +public class TestAsynchronousValidator { + + private AsynchronousValidator impl; + + private CachingHttpClient mockClient; + private HttpHost target; + private HttpRequest request; + private HttpContext mockContext; + private HttpCacheEntry mockCacheEntry; + + private ExecutorService mockExecutor; + + @Before + public void setUp() { + mockClient = EasyMock.createMock(CachingHttpClient.class); + target = new HttpHost("foo.example.com"); + request = new HttpGet("/"); + mockContext = EasyMock.createMock(HttpContext.class); + mockCacheEntry = EasyMock.createMock(HttpCacheEntry.class); + + mockExecutor = EasyMock.createMock(ExecutorService.class); + + } + + @Test + public void testRevalidateCacheEntrySchedulesExecutionAndPopulatesIdentifier() { + impl = new AsynchronousValidator(mockClient, mockExecutor); + + EasyMock.expect(mockCacheEntry.hasVariants()).andReturn(false); + mockExecutor.execute(EasyMock.isA(AsynchronousValidationRequest.class)); + + replayMocks(); + impl.revalidateCacheEntry(target, request, mockContext, mockCacheEntry); + verifyMocks(); + + Assert.assertEquals(1, impl.getScheduledIdentifiers().size()); + } + + @Test + public void testMarkCompleteRemovesIdentifier() { + impl = new AsynchronousValidator(mockClient, mockExecutor); + + EasyMock.expect(mockCacheEntry.hasVariants()).andReturn(false); + Capture cap = new Capture(); + mockExecutor.execute(EasyMock.capture(cap)); + + replayMocks(); + impl.revalidateCacheEntry(target, request, mockContext, mockCacheEntry); + verifyMocks(); + + Assert.assertEquals(1, impl.getScheduledIdentifiers().size()); + + impl.markComplete(cap.getValue().getIdentifier()); + + Assert.assertEquals(0, impl.getScheduledIdentifiers().size()); + } + + @Test + public void testRevalidateCacheEntryDoesNotPopulateIdentifierOnRejectedExecutionException() { + impl = new AsynchronousValidator(mockClient, mockExecutor); + + EasyMock.expect(mockCacheEntry.hasVariants()).andReturn(false); + mockExecutor.execute(EasyMock.isA(AsynchronousValidationRequest.class)); + EasyMock.expectLastCall().andThrow(new RejectedExecutionException()); + + replayMocks(); + impl.revalidateCacheEntry(target, request, mockContext, mockCacheEntry); + verifyMocks(); + + Assert.assertEquals(0, impl.getScheduledIdentifiers().size()); + } + + @Test + public void testRevalidateCacheEntryProperlyCollapsesRequest() { + impl = new AsynchronousValidator(mockClient, mockExecutor); + + EasyMock.expect(mockCacheEntry.hasVariants()).andReturn(false); + mockExecutor.execute(EasyMock.isA(AsynchronousValidationRequest.class)); + + EasyMock.expect(mockCacheEntry.hasVariants()).andReturn(false); + + replayMocks(); + impl.revalidateCacheEntry(target, request, mockContext, mockCacheEntry); + impl.revalidateCacheEntry(target, request, mockContext, mockCacheEntry); + verifyMocks(); + + Assert.assertEquals(1, impl.getScheduledIdentifiers().size()); + } + + @Test + public void testVariantsBothRevalidated() { + impl = new AsynchronousValidator(mockClient, mockExecutor); + + HttpRequest req1 = new HttpGet("/"); + req1.addHeader(new BasicHeader("Accept-Encoding", "identity")); + + HttpRequest req2 = new HttpGet("/"); + req2.addHeader(new BasicHeader("Accept-Encoding", "gzip")); + + Header[] variantHeaders = new Header[] { + new BasicHeader(HeaderConstants.VARY, "Accept-Encoding") + }; + + EasyMock.expect(mockCacheEntry.hasVariants()).andReturn(true).times(2); + EasyMock.expect(mockCacheEntry.getHeaders(HeaderConstants.VARY)).andReturn(variantHeaders).times(2); + mockExecutor.execute(EasyMock.isA(AsynchronousValidationRequest.class)); + EasyMock.expectLastCall().times(2); + + replayMocks(); + impl.revalidateCacheEntry(target, req1, mockContext, mockCacheEntry); + impl.revalidateCacheEntry(target, req2, mockContext, mockCacheEntry); + verifyMocks(); + + Assert.assertEquals(2, impl.getScheduledIdentifiers().size()); + + } + + @Test + public void testRevalidateCacheEntryEndToEnd() throws ProtocolException, IOException, InterruptedException { + impl = new AsynchronousValidator(mockClient, 1); + + EasyMock.expect(mockCacheEntry.hasVariants()).andReturn(false); + EasyMock.expect(mockClient.revalidateCacheEntry(target, request, mockContext, mockCacheEntry)).andReturn(null); + + replayMocks(); + impl.revalidateCacheEntry(target, request, mockContext, mockCacheEntry); + + try { + // shut down backend executor and make sure all finishes properly, 1 second should be sufficient + ExecutorService implExecutor = impl.getExecutor(); + implExecutor.shutdown(); + implExecutor.awaitTermination(1, TimeUnit.SECONDS); + } catch (InterruptedException ie) { + + } finally { + verifyMocks(); + + Assert.assertEquals(0, impl.getScheduledIdentifiers().size()); + } + } + + public void replayMocks() { + EasyMock.replay(mockExecutor); + EasyMock.replay(mockClient); + EasyMock.replay(mockContext); + EasyMock.replay(mockCacheEntry); + } + + public void verifyMocks() { + EasyMock.verify(mockExecutor); + EasyMock.verify(mockClient); + EasyMock.verify(mockContext); + EasyMock.verify(mockCacheEntry); + } +} diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java index aa58dc9b6..412e3282d 100644 --- a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java @@ -411,7 +411,7 @@ public class TestCacheValidityPolicy { HttpRequest req = new BasicHttpRequest("GET","/",HttpVersion.HTTP_1_1); assertTrue(impl.mayReturnStaleIfError(req, entry, now)); } - + @Test public void testMayReturnStaleIfErrorInRequestIsTrueWithinStaleness(){ Header[] headers = new Header[] { @@ -446,4 +446,61 @@ public class TestCacheValidityPolicy { req.setHeader("Cache-Control","stale-if-error=1"); assertFalse(impl.mayReturnStaleIfError(req, entry, now)); } + + @Test + public void testMayReturnStaleWhileRevalidatingIsFalseWhenDirectiveIsAbsent() { + Date now = new Date(); + + Header[] headers = new Header[] { new BasicHeader("Cache-control", "public") }; + HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(headers); + + CacheValidityPolicy impl = new CacheValidityPolicy(); + + assertFalse(impl.mayReturnStaleWhileRevalidating(entry, now)); + } + + @Test + public void testMayReturnStaleWhileRevalidatingIsTrueWhenWithinStaleness() { + Date now = new Date(); + Date tenSecondsAgo = new Date(now.getTime() - 10 * 1000L); + Header[] headers = new Header[] { + new BasicHeader("Date", DateUtils.formatDate(tenSecondsAgo)), + new BasicHeader("Cache-Control", "max-age=5, stale-while-revalidate=15") + }; + HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(now, now, headers); + + CacheValidityPolicy impl = new CacheValidityPolicy(); + + assertTrue(impl.mayReturnStaleWhileRevalidating(entry, now)); + } + + @Test + public void testMayReturnStaleWhileRevalidatingIsFalseWhenPastStaleness() { + Date now = new Date(); + Date twentyFiveSecondsAgo = new Date(now.getTime() - 25 * 1000L); + Header[] headers = new Header[] { + new BasicHeader("Date", DateUtils.formatDate(twentyFiveSecondsAgo)), + new BasicHeader("Cache-Control", "max-age=5, stale-while-revalidate=15") + }; + HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(now, now, headers); + + CacheValidityPolicy impl = new CacheValidityPolicy(); + + assertFalse(impl.mayReturnStaleWhileRevalidating(entry, now)); + } + + @Test + public void testMayReturnStaleWhileRevalidatingIsFalseWhenDirectiveEmpty() { + Date now = new Date(); + Date tenSecondsAgo = new Date(now.getTime() - 10 * 1000L); + Header[] headers = new Header[] { + new BasicHeader("Date", DateUtils.formatDate(tenSecondsAgo)), + new BasicHeader("Cache-Control", "max-age=5, stale-while-revalidate=") + }; + HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(now, now, headers); + + CacheValidityPolicy impl = new CacheValidityPolicy(); + + assertFalse(impl.mayReturnStaleWhileRevalidating(entry, now)); + } } diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java index 970491e31..cc61fb11b 100644 --- a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java @@ -62,6 +62,7 @@ import org.easymock.Capture; import org.easymock.classextension.EasyMock; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; public class TestCachingHttpClient { @@ -340,6 +341,9 @@ public class TestCachingHttpClient { Assert.assertEquals(0, impl.getCacheUpdates()); } + // TODO: re-enable when background validation enabled by default, or adjust + // test to specify background validation in CacheConfig + @Ignore @Test public void testUnsuitableValidatableCacheEntryCausesRevalidation() throws Exception { mockImplMethods(REVALIDATE_CACHE_ENTRY); @@ -351,6 +355,7 @@ public class TestCachingHttpClient { getCacheEntryReturns(mockCacheEntry); cacheEntrySuitable(false); cacheEntryValidatable(true); + mayReturnStaleWhileRevalidating(false); revalidateCacheEntryReturns(mockBackendResponse); replayMocks(); @@ -1942,6 +1947,11 @@ public class TestCachingHttpClient { EasyMock.expect(mockValidityPolicy.isRevalidatable( EasyMock.anyObject())).andReturn(b); } + + private void mayReturnStaleWhileRevalidating(boolean b) { + EasyMock.expect(mockValidityPolicy.mayReturnStaleWhileRevalidating( + EasyMock.anyObject(), EasyMock.anyObject())).andReturn(b); + } private void conditionalVariantRequestBuilderReturns(Map variantEntries, HttpRequest validate) throws Exception { diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestRFC5861Compliance.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestRFC5861Compliance.java index 5e94bd81e..edf823717 100644 --- a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestRFC5861Compliance.java +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestRFC5861Compliance.java @@ -27,12 +27,17 @@ package org.apache.http.impl.client.cache; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.util.Date; +import org.apache.http.Header; import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; +import org.apache.http.HttpVersion; +import org.apache.http.impl.cookie.DateUtils; +import org.apache.http.message.BasicHttpRequest; import org.junit.Test; /** @@ -154,4 +159,49 @@ public class TestRFC5861Compliance extends AbstractProtocolTest { assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, result.getStatusLine().getStatusCode()); } + + /* + * When present in an HTTP response, the stale-while-revalidate Cache- + * Control extension indicates that caches MAY serve the response in + * which it appears after it becomes stale, up to the indicated number + * of seconds. + * + * http://tools.ietf.org/html/rfc5861 + */ + @Test + public void testStaleWhileRevalidateReturnsStaleEntryWithWarning() + throws Exception { + + params.setStaleWhileRevalidateWorkers(1); + impl = new CachingHttpClient(mockBackend, cache, params); + + HttpRequest req1 = new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_1); + HttpResponse resp1 = HttpTestUtils.make200Response(); + Date now = new Date(); + Date tenSecondsAgo = new Date(now.getTime() - 10 * 1000L); + resp1.setHeader("Cache-Control", "public, max-age=5, stale-while-revalidate=15"); + resp1.setHeader("ETag","\"etag\""); + resp1.setHeader("Date", DateUtils.formatDate(tenSecondsAgo)); + + backendExpectsAnyRequest().andReturn(resp1).times(1,2); + + HttpRequest req2 = new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_1); + + replayMocks(); + impl.execute(host, req1); + HttpResponse result = impl.execute(host, req2); + verifyMocks(); + + assertEquals(HttpStatus.SC_OK, result.getStatusLine().getStatusCode()); + boolean warning110Found = false; + for(Header h : result.getHeaders("Warning")) { + for(WarningValue wv : WarningValue.getWarningValues(h)) { + if (wv.getWarnCode() == 110) { + warning110Found = true; + break; + } + } + } + assertTrue(warning110Found); + } }