Apache
/
httpcomponents-client
mirror of https://github.com/apache/httpcomponents-client.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

HTTPCLIENT-975: added CacheConfig settings to provide more detailed 

control over the background revalidation thread pool, and enabled
the stale-while-revalidate feature by default with a single worker
thread and a max queue size of 100 pending revalidations, which
should be a pretty safe setting.


git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk@1050360 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Moore 2010-12-17 12:15:50 +00:00
parent d012c1041c
commit 83efe306fd
6 changed files with 109 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
httpclient-cache/src/main/java/org/apache/http/impl/client/cache/AsynchronousValidator.java vendored
View File

@ -29,10 +29,12 @@ package org.apache.http.impl.client.cache;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -59,8 +61,14 @@ public class AsynchronousValidator {
* @param cachingClient
* @param numThreads
*/
public AsynchronousValidator(CachingHttpClient cachingClient, int numThreads) {
this(cachingClient, Executors.newFixedThreadPool(numThreads));
public AsynchronousValidator(CachingHttpClient cachingClient, CacheConfig config) {
this(cachingClient,
new ThreadPoolExecutor(config.getAsynchronousWorkersCore(),
config.getAsynchronousWorkersMax(),
(long)config.getAsynchronousWorkerIdleLifetimeSecs(),
TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(config.getRevalidationQueueSize()))
);
}
/**

106
httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java vendored
View File

@ -51,21 +51,34 @@ public class CacheConfig {
*/
public final static boolean DEFAULT_HEURISTIC_CACHING_ENABLED = false;
/** Default coefficient used to heuristically determine freshness lifetime from
* cache entry.
/** Default coefficient used to heuristically determine freshness
* lifetime from the Last-Modified time of a cache entry.
*/
public final static float DEFAULT_HEURISTIC_COEFFICIENT = 0.1f;
/** Default lifetime in seconds to be assumed when we cannot calculate freshness
* heuristically
/** Default lifetime in seconds to be assumed when we cannot calculate
* freshness heuristically.
*/
public final static long DEFAULT_HEURISTIC_LIFETIME = 0;
/** Default number of worker threads to allow for background revalidations
* resulting from the stale-while-revalidate directive; 0 disables handling
* asynchronous revalidations.
* resulting from the stale-while-revalidate directive.
*/
private static final int DEFAULT_STALE_WHILE_REVALIDATE_WORKERS = 0;
private static final int DEFAULT_ASYNCHRONOUS_WORKERS_MAX = 1;
/** Default minimum number of worker threads to allow for background
* revalidations resulting from the stale-while-revalidate directive.
*/
private static final int DEFAULT_ASYNCHRONOUS_WORKERS_CORE = 1;
/** Default maximum idle lifetime for a background revalidation thread
* before it gets reclaimed.
*/
private static final int DEFAULT_ASYNCHRONOUS_WORKER_IDLE_LIFETIME_SECS = 60;
/** Default maximum queue length for background revalidation requests.
*/
private static final int DEFAULT_REVALIDATION_QUEUE_SIZE = 100;
private int maxObjectSizeBytes = DEFAULT_MAX_OBJECT_SIZE_BYTES;
private int maxCacheEntries = DEFAULT_MAX_CACHE_ENTRIES;
@ -74,7 +87,10 @@ public class CacheConfig {
private float heuristicCoefficient = DEFAULT_HEURISTIC_COEFFICIENT;
private long heuristicDefaultLifetime = DEFAULT_HEURISTIC_LIFETIME;
private boolean isSharedCache = true;
private int staleWhileRevalidateWorkers = DEFAULT_STALE_WHILE_REVALIDATE_WORKERS;
private int asynchronousWorkersMax = DEFAULT_ASYNCHRONOUS_WORKERS_MAX;
private int asynchronousWorkersCore = DEFAULT_ASYNCHRONOUS_WORKERS_CORE;
private int asynchronousWorkerIdleLifetimeSecs = DEFAULT_ASYNCHRONOUS_WORKER_IDLE_LIFETIME_SECS;
private int revalidationQueueSize = DEFAULT_REVALIDATION_QUEUE_SIZE;
/**
* Returns the current maximum object size that will be cached.
@ -182,20 +198,76 @@ public class CacheConfig {
}
/**
* Set number of worker threads to allow for background revalidations resulting from,
* the stale-while-revalidate directive, 0 disables handling of directive
* @return
* Returns the maximum number of threads to allow for background
* revalidations due to the stale-while-revalidate directive. A
* value of 0 means background revalidations are disabled.
*/
public int getStaleWhileRevalidateWorkers() {
return staleWhileRevalidateWorkers;
public int getAsynchronousWorkersMax() {
return asynchronousWorkersMax;
}
/**
* Get number of worker threads to allow for background revalidations resulting from,
* the stale-while-revalidate directive, 0 disables handling of directive
* Sets the maximum number of threads to allow for background
* revalidations due to the stale-while-revalidate directive.
* @param max number of threads; a value of 0 disables background
* revalidations.
*/
public void setStaleWhileRevalidateWorkers(int staleWhileRevalidateWorkers) {
this.staleWhileRevalidateWorkers = staleWhileRevalidateWorkers;
public void setAsynchronousWorkersMax(int max) {
this.asynchronousWorkersMax = max;
}
/**
* Returns the minimum number of threads to keep alive for background
* revalidations due to the stale-while-revalidate directive.
*/
public int getAsynchronousWorkersCore() {
return asynchronousWorkersCore;
}
/**
* Sets the minimum number of threads to keep alive for background
* revalidations due to the stale-while-revalidate directive.
* @param min should be greater than zero and less than or equal
* to <code>getAsynchronousWorkersMax()</code>
*/
public void setAsynchronousWorkersCore(int min) {
this.asynchronousWorkersCore = min;
}
/**
* Returns the current maximum idle lifetime in seconds for a
* background revalidation worker thread. If a worker thread is idle
* for this long, and there are more than the core number of worker
* threads alive, the worker will be reclaimed.
*/
public int getAsynchronousWorkerIdleLifetimeSecs() {
return asynchronousWorkerIdleLifetimeSecs;
}
/**
* Sets the current maximum idle lifetime in seconds for a
* background revalidation worker thread. If a worker thread is idle
* for this long, and there are more than the core number of worker
* threads alive, the worker will be reclaimed.
* @param secs idle lifetime in seconds
*/
public void setAsynchronousWorkerIdleLifetimeSecs(int secs) {
this.asynchronousWorkerIdleLifetimeSecs = secs;
}
/**
* Returns the current maximum queue size for background revalidations.
*/
public int getRevalidationQueueSize() {
return revalidationQueueSize;
}
/**
* Sets the current maximum queue size for background revalidations.
*/
public void setRevalidationQueueSize(int size) {
this.revalidationQueueSize = size;
}
}

10
httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java vendored
View File

@ -127,7 +127,7 @@ public class CachingHttpClient implements HttpClient {
this.responseCompliance = new ResponseProtocolCompliance();
this.requestCompliance = new RequestProtocolCompliance();
this.asynchRevalidator = makeAsynchronousValidator(config.getStaleWhileRevalidateWorkers());
this.asynchRevalidator = makeAsynchronousValidator(config);
}
public CachingHttpClient() {
@ -197,13 +197,13 @@ public class CachingHttpClient implements HttpClient {
this.conditionalRequestBuilder = conditionalRequestBuilder;
this.responseCompliance = responseCompliance;
this.requestCompliance = requestCompliance;
this.asynchRevalidator = makeAsynchronousValidator(config.getStaleWhileRevalidateWorkers());
this.asynchRevalidator = makeAsynchronousValidator(config);
}
private AsynchronousValidator makeAsynchronousValidator(
int numWorkers) {
if (numWorkers > 0) {
return new AsynchronousValidator(this, numWorkers);
CacheConfig config) {
if (config.getAsynchronousWorkersMax() > 0) {
return new AsynchronousValidator(this, config);
}
return null;
}

5
httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestAsynchronousValidator.java vendored
View File

@ -166,7 +166,10 @@ public class TestAsynchronousValidator {
@Test
public void testRevalidateCacheEntryEndToEnd() throws ProtocolException, IOException, InterruptedException {
impl = new AsynchronousValidator(mockClient, 1);
CacheConfig config = new CacheConfig();
config.setAsynchronousWorkersMax(1);
config.setAsynchronousWorkersCore(1);
impl = new AsynchronousValidator(mockClient, config);
EasyMock.expect(mockCacheEntry.hasVariants()).andReturn(false);
EasyMock.expect(mockClient.revalidateCacheEntry(target, request, mockContext, mockCacheEntry)).andReturn(null);

4
httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java vendored
View File

@ -62,7 +62,6 @@ import org.easymock.Capture;
import org.easymock.classextension.EasyMock;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
public class TestCachingHttpClient {
@ -341,9 +340,6 @@ public class TestCachingHttpClient {
Assert.assertEquals(0, impl.getCacheUpdates());
}
// TODO: re-enable when background validation enabled by default, or adjust
// test to specify background validation in CacheConfig
@Ignore
@Test
public void testUnsuitableValidatableCacheEntryCausesRevalidation() throws Exception {
mockImplMethods(REVALIDATE_CACHE_ENTRY);

2
httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestRFC5861Compliance.java vendored
View File

@ -172,7 +172,7 @@ public class TestRFC5861Compliance extends AbstractProtocolTest {
public void testStaleWhileRevalidateReturnsStaleEntryWithWarning()
throws Exception {
params.setStaleWhileRevalidateWorkers(1);
params.setAsynchronousWorkersMax(1);
impl = new CachingHttpClient(mockBackend, cache, params);
HttpRequest req1 = new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_1);