From 6b539c8c86eff4d73446c7d100c71624842f3746 Mon Sep 17 00:00:00 2001 From: Oleg Kalnichevski Date: Tue, 19 Oct 2010 20:17:16 +0000 Subject: [PATCH] HTTPCLIENT-990: Allow heuristic freshness caching Contributed by Michajlo Matijkiw git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk@1024393 13f79535-47bb-0310-9956-ffa450edef68 --- .../http/impl/client/cache/CacheConfig.java | 68 ++++++++++++++++++- .../client/cache/CacheValidityPolicy.java | 45 ++++++++++++ .../CachedResponseSuitabilityChecker.java | 9 +++ .../client/cache/TestCacheValidityPolicy.java | 42 ++++++++++++ .../TestCachedResponseSuitabilityChecker.java | 37 ++++++++++ 5 files changed, 200 insertions(+), 1 deletion(-) diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java index 587f4a2f3..ef86af13a 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java @@ -47,10 +47,25 @@ public class CacheConfig { */ public final static int DEFAULT_MAX_UPDATE_RETRIES = 1; + /** Default setting for heuristic caching + */ + public final static boolean DEFAULT_HEURISTIC_CACHING_ENABLED = false; + + /** Default coefficient used to heuristically determine freshness lifetime from + * cache entry. + */ + public final static float DEFAULT_HEURISTIC_COEFFICIENT = 0.1f; + + /** Default lifetime to be assumed when we cannot calculate freshness heuristically + */ + public final static long DEFAULT_HEURISTIC_LIFETIME = 0; + private int maxObjectSizeBytes = DEFAULT_MAX_OBJECT_SIZE_BYTES; private int maxCacheEntries = DEFAULT_MAX_CACHE_ENTRIES; private int maxUpdateRetries = DEFAULT_MAX_UPDATE_RETRIES; - + private boolean heuristicCachingEnabled = false; + private float heuristicCoefficient = DEFAULT_HEURISTIC_COEFFICIENT; + private long heuristicDefaultLifetime = DEFAULT_HEURISTIC_LIFETIME; private boolean isSharedCache = true; /** @@ -118,4 +133,55 @@ public class CacheConfig { public void setMaxUpdateRetries(int maxUpdateRetries){ this.maxUpdateRetries = maxUpdateRetries; } + + /** + * Returns if heuristic freshness caching is in enabled + * @return + */ + public boolean isHeuristicCachingEnabled() { + return heuristicCachingEnabled; + } + + /** + * Set if heuristic freshness caching is enabled + * @param heursiticCachingEnabled + */ + public void setHeuristicCachingEnabled(boolean heuristicCachingEnabled) { + this.heuristicCachingEnabled = heuristicCachingEnabled; + } + + /** + * Returns coefficient used in heuristic freshness caching + * @return + */ + public float getHeuristicCoefficient() { + return heuristicCoefficient; + } + + /** + * Set coefficient to be used in heuristic freshness caching + * @param heuristicCoefficient + */ + public void setHeuristicCoefficient(float heuristicCoefficient) { + this.heuristicCoefficient = heuristicCoefficient; + } + + /** + * Get the default lifetime to be used if heuristic freshness calculation is + * not possible + * @return + */ + public long getHeuristicDefaultLifetime() { + return heuristicDefaultLifetime; + } + + /** + * Set default lifetime to be used if heuristic freshness calculation is not possible + * @param heuristicDefaultLifetime + */ + public void setHeuristicDefaultLifetime(long heuristicDefaultLifetime) { + this.heuristicDefaultLifetime = heuristicDefaultLifetime; + } + + } diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java index 64b85b5ca..3349e072e 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java @@ -73,6 +73,39 @@ class CacheValidityPolicy { return (getCurrentAgeSecs(entry, now) < getFreshnessLifetimeSecs(entry)); } + /** + * Decides if this response is fresh enough based Last-Modified and Date, if available. + * This entry is meant to be used when isResponseFresh returns false. The algorithm is as follows: + * + * if last-modified and date are defined, freshness lifetime is coefficient*(date-lastModified), + * else freshness lifetime is defaultLifetime + * + * @param entry + * @param now + * @param coefficient + * @param defaultLifetime + * @return + */ + public boolean isResponseHeuristicallyFresh(final HttpCacheEntry entry, + Date now, float coefficient, long defaultLifetime) { + return (getCurrentAgeSecs(entry, now) < getHeuristicFreshnessLifetimeSecs(entry, coefficient, defaultLifetime)); + } + + public long getHeuristicFreshnessLifetimeSecs(HttpCacheEntry entry, + float coefficient, long defaultLifetime) { + Date dateValue = getDateValue(entry); + Date lastModifiedValue = getLastModifiedValue(entry); + + if (dateValue != null && lastModifiedValue != null) { + long diff = dateValue.getTime() - lastModifiedValue.getTime(); + if (diff < 0) + return 0; + return (long)(coefficient * (diff / 1000)); + } + + return defaultLifetime; + } + public boolean isRevalidatable(final HttpCacheEntry entry) { return entry.getFirstHeader(HeaderConstants.ETAG) != null || entry.getFirstHeader(HeaderConstants.LAST_MODIFIED) != null; @@ -98,6 +131,18 @@ class CacheValidityPolicy { return null; } + protected Date getLastModifiedValue(final HttpCacheEntry entry) { + Header dateHdr = entry.getFirstHeader(HeaderConstants.LAST_MODIFIED); + if (dateHdr == null) + return null; + try { + return DateUtils.parseDate(dateHdr.getValue()); + } catch (DateParseException dpe) { + // ignore malformed date + } + return null; + } + protected long getContentLengthValue(final HttpCacheEntry entry) { Header cl = entry.getFirstHeader(HTTP.CONTENT_LEN); if (cl == null) diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachedResponseSuitabilityChecker.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachedResponseSuitabilityChecker.java index c23281499..2a3980e31 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachedResponseSuitabilityChecker.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachedResponseSuitabilityChecker.java @@ -52,6 +52,9 @@ class CachedResponseSuitabilityChecker { private final Log log = LogFactory.getLog(getClass()); private final boolean sharedCache; + private final boolean useHeuristicCaching; + private final float heuristicCoefficient; + private final long heuristicDefaultLifetime; private final CacheValidityPolicy validityStrategy; CachedResponseSuitabilityChecker(final CacheValidityPolicy validityStrategy, @@ -59,6 +62,9 @@ class CachedResponseSuitabilityChecker { super(); this.validityStrategy = validityStrategy; this.sharedCache = config.isSharedCache(); + this.useHeuristicCaching = config.isHeuristicCachingEnabled(); + this.heuristicCoefficient = config.getHeuristicCoefficient(); + this.heuristicDefaultLifetime = config.getHeuristicDefaultLifetime(); } CachedResponseSuitabilityChecker(CacheConfig config) { @@ -67,6 +73,9 @@ class CachedResponseSuitabilityChecker { private boolean isFreshEnough(HttpCacheEntry entry, HttpRequest request, Date now) { if (validityStrategy.isResponseFresh(entry, now)) return true; + if (useHeuristicCaching && + validityStrategy.isResponseHeuristicallyFresh(entry, now, heuristicCoefficient, heuristicDefaultLifetime)) + return true; if (originInsistsOnFreshness(entry)) return false; long maxstale = getMaxStale(request); if (maxstale == -1) return false; diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java index f2aad49f1..49e209043 100644 --- a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java @@ -250,6 +250,48 @@ public class TestCacheValidityPolicy { Assert.assertEquals(4, impl.getFreshnessLifetimeSecs(entry)); } + @Test + public void testHeuristicFreshnessLifetime() { + Date now = new Date(); + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date elevenSecondsAgo = new Date(now.getTime() - 11 * 1000L); + + Header[] headers = new Header[] { + new BasicHeader("Date", DateUtils.formatDate(oneSecondAgo)), + new BasicHeader("Last-Modified", DateUtils.formatDate(elevenSecondsAgo)) + }; + + HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(headers); + CacheValidityPolicy impl = new CacheValidityPolicy(); + Assert.assertEquals(1, impl.getHeuristicFreshnessLifetimeSecs(entry, 0.1f, 0)); + } + + @Test + public void testHeuristicFreshnessLifetimeDefaultsProperly() { + long defaultFreshness = 10; + + HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(); + + CacheValidityPolicy impl = new CacheValidityPolicy(); + Assert.assertEquals(defaultFreshness, impl.getHeuristicFreshnessLifetimeSecs(entry, 0.1f, defaultFreshness)); + } + + @Test + public void testHeuristicFreshnessLifetimeIsNonNegative() { + Date now = new Date(); + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date elevenSecondsAgo = new Date(now.getTime() - 1 * 1000L); + + Header[] headers = new Header[] { + new BasicHeader("Date", DateUtils.formatDate(elevenSecondsAgo)), + new BasicHeader("Last-Modified", DateUtils.formatDate(oneSecondAgo)) + }; + + HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(headers); + CacheValidityPolicy impl = new CacheValidityPolicy(); + Assert.assertTrue(impl.getHeuristicFreshnessLifetimeSecs(entry, 0.1f, 10) >= 0); + } + @Test public void testResponseIsFreshIfFreshnessLifetimeExceedsCurrentAge() { final HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(); diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachedResponseSuitabilityChecker.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachedResponseSuitabilityChecker.java index a317f8cb2..782990bcb 100644 --- a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachedResponseSuitabilityChecker.java +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachedResponseSuitabilityChecker.java @@ -223,4 +223,41 @@ public class TestCachedResponseSuitabilityChecker { Assert.assertFalse(impl.canCachedResponseBeUsed(host, request, entry, now)); } + @Test + public void testSuitableIfCacheEntryIsHeuristicallyFreshEnough() { + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date twentyOneSecondsAgo = new Date(now.getTime() - 21 * 1000L); + + Header[] headers = { + new BasicHeader("Date", DateUtils.formatDate(oneSecondAgo)), + new BasicHeader("Last-Modified", DateUtils.formatDate(twentyOneSecondsAgo)), + new BasicHeader("Content-Length", "128") + }; + + entry = HttpTestUtils.makeCacheEntry(oneSecondAgo, oneSecondAgo, headers); + + CacheConfig config = new CacheConfig(); + config.setHeuristicCachingEnabled(true); + config.setHeuristicCoefficient(0.1f); + impl = new CachedResponseSuitabilityChecker(config); + + Assert.assertTrue(impl.canCachedResponseBeUsed(host, request, entry, now)); + } + + @Test + public void testSuitableIfCacheEntryIsHeuristicallyFreshEnoughByDefault() { + Header[] headers = { + new BasicHeader("Date", DateUtils.formatDate(tenSecondsAgo)), + new BasicHeader("Content-Length", "128") + }; + + entry = getEntry(headers); + + CacheConfig config = new CacheConfig(); + config.setHeuristicCachingEnabled(true); + config.setHeuristicDefaultLifetime(20); + impl = new CachedResponseSuitabilityChecker(config); + + Assert.assertTrue(impl.canCachedResponseBeUsed(host, request, entry, now)); + } } \ No newline at end of file