From c76574f9476acab626f711ff966aee65a35d3845 Mon Sep 17 00:00:00 2001 From: Oleg Kalnichevski Date: Fri, 22 Oct 2010 21:47:45 +0000 Subject: [PATCH] Dedicated branch for testing maven-notice-plugin git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/branches/notice-plugin-test@1026503 13f79535-47bb-0310-9956-ffa450edef68 --- httpclient-benchmark/pom.xml | 12 +--- httpclient-cache/pom.xml | 29 -------- .../http/impl/client/cache/CacheConfig.java | 68 ++++++++++++++++++- .../client/cache/CacheValidityPolicy.java | 45 ++++++++++++ .../CachedResponseSuitabilityChecker.java | 9 +++ .../impl/client/cache/CachingHttpClient.java | 21 ++++++ .../client/cache/TestCacheValidityPolicy.java | 42 ++++++++++++ .../TestCachedResponseSuitabilityChecker.java | 37 ++++++++++ .../client/cache/TestCachingHttpClient.java | 53 +++++++++++++++ httpclient-contrib/pom.xml | 10 +-- httpclient-osgi/pom.xml | 26 +------ httpclient/pom.xml | 56 +-------------- .../examples/client/ClientAbortMethod.java | 8 +-- .../examples/client/ClientAuthentication.java | 17 +++-- .../client/ClientChunkEncodedPost.java | 23 +++---- .../examples/client/ClientCustomContext.java | 25 ++++--- .../http/examples/client/ClientCustomSSL.java | 17 +++-- .../client/ClientEvictExpiredConnections.java | 43 ++++++------ .../examples/client/ClientExecuteDirect.java | 6 +- .../examples/client/ClientExecuteProxy.java | 4 +- .../examples/client/ClientExecuteSOCKS.java | 18 ++--- .../http/examples/client/ClientFormLogin.java | 14 ++-- .../client/ClientGZipContentCompression.java | 38 +++++------ .../ClientInteractiveAuthentication.java | 31 ++++----- .../client/ClientKerberosAuthentication.java | 28 ++++---- .../client/ClientMultiThreadedExecution.java | 44 ++++++------ .../ClientPreemptiveBasicAuthentication.java | 29 ++++---- .../ClientPreemptiveDigestAuthentication.java | 33 ++++----- .../client/ClientProxyAuthentication.java | 27 ++++---- .../client/ClientWithResponseHandler.java | 12 ++-- .../examples/conn/ManagerConnectDirect.java | 2 +- .../examples/conn/ManagerConnectProxy.java | 2 +- .../examples/conn/OperatorConnectDirect.java | 2 +- .../examples/conn/OperatorConnectProxy.java | 8 +-- httpmime/pom.xml | 32 +-------- pom.xml | 13 ++++ src/main/resources/NOTICE.txt | 8 --- 37 files changed, 505 insertions(+), 387 deletions(-) delete mode 100644 src/main/resources/NOTICE.txt diff --git a/httpclient-benchmark/pom.xml b/httpclient-benchmark/pom.xml index 23e53fc62..24e9575c5 100644 --- a/httpclient-benchmark/pom.xml +++ b/httpclient-benchmark/pom.xml @@ -38,15 +38,7 @@ HttpComponents HttpClient - Benchmarks http://hc.apache.org/httpcomponents-client - jar - - - - Apache License - ../LICENSE.txt - repo - - + jar @@ -126,7 +118,7 @@ java - org.apache.http.client.benchmark.Benchmark + org.apache.http.client.benchmark.Benchmark diff --git a/httpclient-cache/pom.xml b/httpclient-cache/pom.xml index df8eaaaa8..b0f0a970a 100644 --- a/httpclient-cache/pom.xml +++ b/httpclient-cache/pom.xml @@ -40,13 +40,6 @@ http://hc.apache.org/httpcomponents-client jar - - - Apache License - ../LICENSE.txt - repo - - spy @@ -122,13 +115,6 @@ - - src/main/resources - false - - META-INF/* - - src/main/resources true @@ -136,21 +122,6 @@ **/*.properties - - .. - META-INF - - LICENSE.txt - - - - ../src/main/resources - META-INF - true - - NOTICE.txt - - diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java index 587f4a2f3..ef86af13a 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java @@ -47,10 +47,25 @@ public class CacheConfig { */ public final static int DEFAULT_MAX_UPDATE_RETRIES = 1; + /** Default setting for heuristic caching + */ + public final static boolean DEFAULT_HEURISTIC_CACHING_ENABLED = false; + + /** Default coefficient used to heuristically determine freshness lifetime from + * cache entry. + */ + public final static float DEFAULT_HEURISTIC_COEFFICIENT = 0.1f; + + /** Default lifetime to be assumed when we cannot calculate freshness heuristically + */ + public final static long DEFAULT_HEURISTIC_LIFETIME = 0; + private int maxObjectSizeBytes = DEFAULT_MAX_OBJECT_SIZE_BYTES; private int maxCacheEntries = DEFAULT_MAX_CACHE_ENTRIES; private int maxUpdateRetries = DEFAULT_MAX_UPDATE_RETRIES; - + private boolean heuristicCachingEnabled = false; + private float heuristicCoefficient = DEFAULT_HEURISTIC_COEFFICIENT; + private long heuristicDefaultLifetime = DEFAULT_HEURISTIC_LIFETIME; private boolean isSharedCache = true; /** @@ -118,4 +133,55 @@ public class CacheConfig { public void setMaxUpdateRetries(int maxUpdateRetries){ this.maxUpdateRetries = maxUpdateRetries; } + + /** + * Returns if heuristic freshness caching is in enabled + * @return + */ + public boolean isHeuristicCachingEnabled() { + return heuristicCachingEnabled; + } + + /** + * Set if heuristic freshness caching is enabled + * @param heursiticCachingEnabled + */ + public void setHeuristicCachingEnabled(boolean heuristicCachingEnabled) { + this.heuristicCachingEnabled = heuristicCachingEnabled; + } + + /** + * Returns coefficient used in heuristic freshness caching + * @return + */ + public float getHeuristicCoefficient() { + return heuristicCoefficient; + } + + /** + * Set coefficient to be used in heuristic freshness caching + * @param heuristicCoefficient + */ + public void setHeuristicCoefficient(float heuristicCoefficient) { + this.heuristicCoefficient = heuristicCoefficient; + } + + /** + * Get the default lifetime to be used if heuristic freshness calculation is + * not possible + * @return + */ + public long getHeuristicDefaultLifetime() { + return heuristicDefaultLifetime; + } + + /** + * Set default lifetime to be used if heuristic freshness calculation is not possible + * @param heuristicDefaultLifetime + */ + public void setHeuristicDefaultLifetime(long heuristicDefaultLifetime) { + this.heuristicDefaultLifetime = heuristicDefaultLifetime; + } + + } diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java index 64b85b5ca..3349e072e 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java @@ -73,6 +73,39 @@ class CacheValidityPolicy { return (getCurrentAgeSecs(entry, now) < getFreshnessLifetimeSecs(entry)); } + /** + * Decides if this response is fresh enough based Last-Modified and Date, if available. + * This entry is meant to be used when isResponseFresh returns false. The algorithm is as follows: + * + * if last-modified and date are defined, freshness lifetime is coefficient*(date-lastModified), + * else freshness lifetime is defaultLifetime + * + * @param entry + * @param now + * @param coefficient + * @param defaultLifetime + * @return + */ + public boolean isResponseHeuristicallyFresh(final HttpCacheEntry entry, + Date now, float coefficient, long defaultLifetime) { + return (getCurrentAgeSecs(entry, now) < getHeuristicFreshnessLifetimeSecs(entry, coefficient, defaultLifetime)); + } + + public long getHeuristicFreshnessLifetimeSecs(HttpCacheEntry entry, + float coefficient, long defaultLifetime) { + Date dateValue = getDateValue(entry); + Date lastModifiedValue = getLastModifiedValue(entry); + + if (dateValue != null && lastModifiedValue != null) { + long diff = dateValue.getTime() - lastModifiedValue.getTime(); + if (diff < 0) + return 0; + return (long)(coefficient * (diff / 1000)); + } + + return defaultLifetime; + } + public boolean isRevalidatable(final HttpCacheEntry entry) { return entry.getFirstHeader(HeaderConstants.ETAG) != null || entry.getFirstHeader(HeaderConstants.LAST_MODIFIED) != null; @@ -98,6 +131,18 @@ class CacheValidityPolicy { return null; } + protected Date getLastModifiedValue(final HttpCacheEntry entry) { + Header dateHdr = entry.getFirstHeader(HeaderConstants.LAST_MODIFIED); + if (dateHdr == null) + return null; + try { + return DateUtils.parseDate(dateHdr.getValue()); + } catch (DateParseException dpe) { + // ignore malformed date + } + return null; + } + protected long getContentLengthValue(final HttpCacheEntry entry) { Header cl = entry.getFirstHeader(HTTP.CONTENT_LEN); if (cl == null) diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachedResponseSuitabilityChecker.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachedResponseSuitabilityChecker.java index c23281499..2a3980e31 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachedResponseSuitabilityChecker.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachedResponseSuitabilityChecker.java @@ -52,6 +52,9 @@ class CachedResponseSuitabilityChecker { private final Log log = LogFactory.getLog(getClass()); private final boolean sharedCache; + private final boolean useHeuristicCaching; + private final float heuristicCoefficient; + private final long heuristicDefaultLifetime; private final CacheValidityPolicy validityStrategy; CachedResponseSuitabilityChecker(final CacheValidityPolicy validityStrategy, @@ -59,6 +62,9 @@ class CachedResponseSuitabilityChecker { super(); this.validityStrategy = validityStrategy; this.sharedCache = config.isSharedCache(); + this.useHeuristicCaching = config.isHeuristicCachingEnabled(); + this.heuristicCoefficient = config.getHeuristicCoefficient(); + this.heuristicDefaultLifetime = config.getHeuristicDefaultLifetime(); } CachedResponseSuitabilityChecker(CacheConfig config) { @@ -67,6 +73,9 @@ class CachedResponseSuitabilityChecker { private boolean isFreshEnough(HttpCacheEntry entry, HttpRequest request, Date now) { if (validityStrategy.isResponseFresh(entry, now)) return true; + if (useHeuristicCaching && + validityStrategy.isResponseHeuristicallyFresh(entry, now, heuristicCoefficient, heuristicDefaultLifetime)) + return true; if (originInsistsOnFreshness(entry)) return false; long maxstale = getMaxStale(request); if (maxstale == -1) return false; diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java index 9f6d7eb09..9a80caf29 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java @@ -407,6 +407,11 @@ public class CachingHttpClient implements HttpClient { log.debug("Cache miss [host: " + target + "; uri: " + rl.getUri() + "]"); } + if (!mayCallBackend(request)) { + return new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT, + "Gateway Timeout"); + } + Set variantEntries = null; try { responseCache.getVariantCacheEntries(target, request); @@ -447,6 +452,11 @@ public class CachingHttpClient implements HttpClient { return cachedResponse; } + if (!mayCallBackend(request)) { + return new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT, + "Gateway Timeout"); + } + if (validityPolicy.isRevalidatable(entry)) { log.debug("Revalidating the cache entry"); @@ -472,6 +482,17 @@ public class CachingHttpClient implements HttpClient { return callBackend(target, request, context); } + private boolean mayCallBackend(HttpRequest request) { + for (Header h: request.getHeaders("Cache-Control")) { + for (HeaderElement elt : h.getElements()) { + if ("only-if-cached".equals(elt.getName())) { + return false; + } + } + } + return true; + } + private boolean explicitFreshnessRequest(HttpRequest request, HttpCacheEntry entry, Date now) { for(Header h : request.getHeaders("Cache-Control")) { for(HeaderElement elt : h.getElements()) { diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java index f2aad49f1..49e209043 100644 --- a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java @@ -250,6 +250,48 @@ public class TestCacheValidityPolicy { Assert.assertEquals(4, impl.getFreshnessLifetimeSecs(entry)); } + @Test + public void testHeuristicFreshnessLifetime() { + Date now = new Date(); + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date elevenSecondsAgo = new Date(now.getTime() - 11 * 1000L); + + Header[] headers = new Header[] { + new BasicHeader("Date", DateUtils.formatDate(oneSecondAgo)), + new BasicHeader("Last-Modified", DateUtils.formatDate(elevenSecondsAgo)) + }; + + HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(headers); + CacheValidityPolicy impl = new CacheValidityPolicy(); + Assert.assertEquals(1, impl.getHeuristicFreshnessLifetimeSecs(entry, 0.1f, 0)); + } + + @Test + public void testHeuristicFreshnessLifetimeDefaultsProperly() { + long defaultFreshness = 10; + + HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(); + + CacheValidityPolicy impl = new CacheValidityPolicy(); + Assert.assertEquals(defaultFreshness, impl.getHeuristicFreshnessLifetimeSecs(entry, 0.1f, defaultFreshness)); + } + + @Test + public void testHeuristicFreshnessLifetimeIsNonNegative() { + Date now = new Date(); + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date elevenSecondsAgo = new Date(now.getTime() - 1 * 1000L); + + Header[] headers = new Header[] { + new BasicHeader("Date", DateUtils.formatDate(elevenSecondsAgo)), + new BasicHeader("Last-Modified", DateUtils.formatDate(oneSecondAgo)) + }; + + HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(headers); + CacheValidityPolicy impl = new CacheValidityPolicy(); + Assert.assertTrue(impl.getHeuristicFreshnessLifetimeSecs(entry, 0.1f, 10) >= 0); + } + @Test public void testResponseIsFreshIfFreshnessLifetimeExceedsCurrentAge() { final HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(); diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachedResponseSuitabilityChecker.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachedResponseSuitabilityChecker.java index a317f8cb2..782990bcb 100644 --- a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachedResponseSuitabilityChecker.java +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachedResponseSuitabilityChecker.java @@ -223,4 +223,41 @@ public class TestCachedResponseSuitabilityChecker { Assert.assertFalse(impl.canCachedResponseBeUsed(host, request, entry, now)); } + @Test + public void testSuitableIfCacheEntryIsHeuristicallyFreshEnough() { + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date twentyOneSecondsAgo = new Date(now.getTime() - 21 * 1000L); + + Header[] headers = { + new BasicHeader("Date", DateUtils.formatDate(oneSecondAgo)), + new BasicHeader("Last-Modified", DateUtils.formatDate(twentyOneSecondsAgo)), + new BasicHeader("Content-Length", "128") + }; + + entry = HttpTestUtils.makeCacheEntry(oneSecondAgo, oneSecondAgo, headers); + + CacheConfig config = new CacheConfig(); + config.setHeuristicCachingEnabled(true); + config.setHeuristicCoefficient(0.1f); + impl = new CachedResponseSuitabilityChecker(config); + + Assert.assertTrue(impl.canCachedResponseBeUsed(host, request, entry, now)); + } + + @Test + public void testSuitableIfCacheEntryIsHeuristicallyFreshEnoughByDefault() { + Header[] headers = { + new BasicHeader("Date", DateUtils.formatDate(tenSecondsAgo)), + new BasicHeader("Content-Length", "128") + }; + + entry = getEntry(headers); + + CacheConfig config = new CacheConfig(); + config.setHeuristicCachingEnabled(true); + config.setHeuristicDefaultLifetime(20); + impl = new CachedResponseSuitabilityChecker(config); + + Assert.assertTrue(impl.canCachedResponseBeUsed(host, request, entry, now)); + } } \ No newline at end of file diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java index fd629badb..3d84d09ee 100644 --- a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java @@ -1931,6 +1931,59 @@ public class TestCachingHttpClient { Assert.assertSame(resp, result); } + @Test + public void testIfOnlyIfCachedAndNoCacheEntryBackendNotCalled() throws IOException { + impl = new CachingHttpClient(mockBackend); + + request.addHeader("Cache-Control", "only-if-cached"); + + HttpResponse resp = impl.execute(host, request); + + Assert.assertEquals(HttpStatus.SC_GATEWAY_TIMEOUT, resp.getStatusLine().getStatusCode()); + } + + @Test + public void testIfOnlyIfCachedAndEntryNotSuitableBackendNotCalled() throws Exception { + + request.setHeader("Cache-Control", "only-if-cached"); + + entry = HttpTestUtils.makeCacheEntry(new Header[]{new BasicHeader("Cache-Control", "must-revalidate")}); + + requestIsFatallyNonCompliant(null); + requestProtocolValidationIsCalled(); + cacheInvalidatorWasCalled(); + requestPolicyAllowsCaching(true); + getCacheEntryReturns(entry); + cacheEntrySuitable(false); + + replayMocks(); + HttpResponse resp = impl.execute(host, request); + verifyMocks(); + + Assert.assertEquals(HttpStatus.SC_GATEWAY_TIMEOUT, resp.getStatusLine().getStatusCode()); + } + + @Test + public void testIfOnlyIfCachedAndEntryExistsAndIsSuitableReturnsEntry() throws Exception { + + request.setHeader("Cache-Control", "only-if-cached"); + + requestIsFatallyNonCompliant(null); + requestProtocolValidationIsCalled(); + cacheInvalidatorWasCalled(); + requestPolicyAllowsCaching(true); + getCacheEntryReturns(entry); + cacheEntrySuitable(true); + responseIsGeneratedFromCache(); + entryHasStaleness(0); + + replayMocks(); + HttpResponse resp = impl.execute(host, request); + verifyMocks(); + + Assert.assertSame(mockCachedResponse, resp); + } + private void getCacheEntryReturns(HttpCacheEntry result) throws IOException { EasyMock.expect(mockCache.getCacheEntry(host, request)).andReturn(result); } diff --git a/httpclient-contrib/pom.xml b/httpclient-contrib/pom.xml index 18a42fa50..73c1223d6 100644 --- a/httpclient-contrib/pom.xml +++ b/httpclient-contrib/pom.xml @@ -38,15 +38,7 @@ HttpComponents HttpClient - Contributed Components http://hc.apache.org/httpcomponents-client - jar - - - - Apache License - ../LICENSE.txt - repo - - + jar diff --git a/httpclient-osgi/pom.xml b/httpclient-osgi/pom.xml index 7288b1820..596b25a87 100644 --- a/httpclient-osgi/pom.xml +++ b/httpclient-osgi/pom.xml @@ -1,5 +1,5 @@ -