HTTPCLIENT-990: Allow heuristic freshness caching

Contributed by Michajlo Matijkiw <michajlo_matijkiw at comcast.com>


git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk@1024393 13f79535-47bb-0310-9956-ffa450edef68

httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheConfig.java

@@ -47,10 +47,25 @@ public class CacheConfig {
      */
     public final static int DEFAULT_MAX_UPDATE_RETRIES = 1;
 
+    /** Default setting for heuristic caching
+     */
+    public final static boolean DEFAULT_HEURISTIC_CACHING_ENABLED = false;
+
+    /** Default coefficient used to heuristically determine freshness lifetime from
+     * cache entry.
+     */
+    public final static float DEFAULT_HEURISTIC_COEFFICIENT = 0.1f;
+
+    /** Default lifetime to be assumed when we cannot calculate freshness heuristically
+     */
+    public final static long DEFAULT_HEURISTIC_LIFETIME = 0;
+
     private int maxObjectSizeBytes = DEFAULT_MAX_OBJECT_SIZE_BYTES;
     private int maxCacheEntries = DEFAULT_MAX_CACHE_ENTRIES;
     private int maxUpdateRetries = DEFAULT_MAX_UPDATE_RETRIES;
-
+    private boolean heuristicCachingEnabled = false;
+    private float heuristicCoefficient = DEFAULT_HEURISTIC_COEFFICIENT;
+    private long heuristicDefaultLifetime = DEFAULT_HEURISTIC_LIFETIME;
     private boolean isSharedCache = true;
 
     /**
@@ -118,4 +133,55 @@ public class CacheConfig {
     public void setMaxUpdateRetries(int maxUpdateRetries){
         this.maxUpdateRetries = maxUpdateRetries;
     }
+
+    /**
+     * Returns if heuristic freshness caching is in enabled
+     * @return
+     */
+    public boolean isHeuristicCachingEnabled() {
+        return heuristicCachingEnabled;
+    }
+
+    /**
+     * Set if heuristic freshness caching is enabled
+     * @param heursiticCachingEnabled
+     */
+    public void setHeuristicCachingEnabled(boolean heuristicCachingEnabled) {
+        this.heuristicCachingEnabled = heuristicCachingEnabled;
+    }
+
+    /**
+     * Returns coefficient used in heuristic freshness caching
+     * @return
+     */
+    public float getHeuristicCoefficient() {
+        return heuristicCoefficient;
+    }
+
+    /**
+     * Set coefficient to be used in heuristic freshness caching
+     * @param heuristicCoefficient
+     */
+    public void setHeuristicCoefficient(float heuristicCoefficient) {
+        this.heuristicCoefficient = heuristicCoefficient;
+    }
+
+    /**
+     * Get the default lifetime to be used if heuristic freshness calculation is
+     * not possible
+     * @return
+     */
+    public long getHeuristicDefaultLifetime() {
+        return heuristicDefaultLifetime;
+    }
+
+    /**
+     * Set default lifetime to be used if heuristic freshness calculation is not possible
+     * @param heuristicDefaultLifetime
+     */
+    public void setHeuristicDefaultLifetime(long heuristicDefaultLifetime) {
+        this.heuristicDefaultLifetime = heuristicDefaultLifetime;
+    }
+
+
 }

httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CacheValidityPolicy.java

@@ -73,6 +73,39 @@ class CacheValidityPolicy {
         return (getCurrentAgeSecs(entry, now) < getFreshnessLifetimeSecs(entry));
     }
 
+    /**
+     * Decides if this response is fresh enough based Last-Modified and Date, if available.
+     * This entry is meant to be used when isResponseFresh returns false.  The algorithm is as follows:
+     *
+     * if last-modified and date are defined, freshness lifetime is coefficient*(date-lastModified),
+     * else freshness lifetime is defaultLifetime
+     *
+     * @param entry
+     * @param now
+     * @param coefficient
+     * @param defaultLifetime
+     * @return
+     */
+    public boolean isResponseHeuristicallyFresh(final HttpCacheEntry entry,
+            Date now, float coefficient, long defaultLifetime) {
+        return (getCurrentAgeSecs(entry, now) < getHeuristicFreshnessLifetimeSecs(entry, coefficient, defaultLifetime));
+    }
+
+    public long getHeuristicFreshnessLifetimeSecs(HttpCacheEntry entry,
+            float coefficient, long defaultLifetime) {
+        Date dateValue = getDateValue(entry);
+        Date lastModifiedValue = getLastModifiedValue(entry);
+
+        if (dateValue != null && lastModifiedValue != null) {
+            long diff = dateValue.getTime() - lastModifiedValue.getTime();
+            if (diff < 0)
+                return 0;
+            return (long)(coefficient * (diff / 1000));
+        }
+
+        return defaultLifetime;
+    }
+
     public boolean isRevalidatable(final HttpCacheEntry entry) {
         return entry.getFirstHeader(HeaderConstants.ETAG) != null
                 || entry.getFirstHeader(HeaderConstants.LAST_MODIFIED) != null;
@@ -98,6 +131,18 @@ class CacheValidityPolicy {
         return null;
     }
 
+    protected Date getLastModifiedValue(final HttpCacheEntry entry) {
+        Header dateHdr = entry.getFirstHeader(HeaderConstants.LAST_MODIFIED);
+        if (dateHdr == null)
+            return null;
+        try {
+            return DateUtils.parseDate(dateHdr.getValue());
+        } catch (DateParseException dpe) {
+            // ignore malformed date
+        }
+        return null;
+    }
+
     protected long getContentLengthValue(final HttpCacheEntry entry) {
         Header cl = entry.getFirstHeader(HTTP.CONTENT_LEN);
         if (cl == null)

httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachedResponseSuitabilityChecker.java

@@ -52,6 +52,9 @@ class CachedResponseSuitabilityChecker {
     private final Log log = LogFactory.getLog(getClass());
 
     private final boolean sharedCache;
+    private final boolean useHeuristicCaching;
+    private final float heuristicCoefficient;
+    private final long heuristicDefaultLifetime;
     private final CacheValidityPolicy validityStrategy;
 
     CachedResponseSuitabilityChecker(final CacheValidityPolicy validityStrategy,
@@ -59,6 +62,9 @@ class CachedResponseSuitabilityChecker {
         super();
         this.validityStrategy = validityStrategy;
         this.sharedCache = config.isSharedCache();
+        this.useHeuristicCaching = config.isHeuristicCachingEnabled();
+        this.heuristicCoefficient = config.getHeuristicCoefficient();
+        this.heuristicDefaultLifetime = config.getHeuristicDefaultLifetime();
     }
 
     CachedResponseSuitabilityChecker(CacheConfig config) {
@@ -67,6 +73,9 @@ class CachedResponseSuitabilityChecker {
 
     private boolean isFreshEnough(HttpCacheEntry entry, HttpRequest request, Date now) {
         if (validityStrategy.isResponseFresh(entry, now)) return true;
+        if (useHeuristicCaching &&
+                validityStrategy.isResponseHeuristicallyFresh(entry, now, heuristicCoefficient, heuristicDefaultLifetime))
+            return true;
         if (originInsistsOnFreshness(entry)) return false;
         long maxstale = getMaxStale(request);
         if (maxstale == -1) return false;

httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCacheValidityPolicy.java

@@ -250,6 +250,48 @@ public class TestCacheValidityPolicy {
         Assert.assertEquals(4, impl.getFreshnessLifetimeSecs(entry));
     }
 
+    @Test
+    public void testHeuristicFreshnessLifetime() {
+        Date now = new Date();
+        Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L);
+        Date elevenSecondsAgo = new Date(now.getTime() - 11 * 1000L);
+
+        Header[] headers = new Header[] {
+                new BasicHeader("Date", DateUtils.formatDate(oneSecondAgo)),
+                new BasicHeader("Last-Modified", DateUtils.formatDate(elevenSecondsAgo))
+        };
+
+        HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(headers);
+        CacheValidityPolicy impl = new CacheValidityPolicy();
+        Assert.assertEquals(1, impl.getHeuristicFreshnessLifetimeSecs(entry, 0.1f, 0));
+    }
+
+    @Test
+    public void testHeuristicFreshnessLifetimeDefaultsProperly() {
+        long defaultFreshness = 10;
+
+        HttpCacheEntry entry = HttpTestUtils.makeCacheEntry();
+
+        CacheValidityPolicy impl = new CacheValidityPolicy();
+        Assert.assertEquals(defaultFreshness, impl.getHeuristicFreshnessLifetimeSecs(entry, 0.1f, defaultFreshness));
+    }
+
+    @Test
+    public void testHeuristicFreshnessLifetimeIsNonNegative() {
+        Date now = new Date();
+        Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L);
+        Date elevenSecondsAgo = new Date(now.getTime() - 1 * 1000L);
+
+        Header[] headers = new Header[] {
+                new BasicHeader("Date", DateUtils.formatDate(elevenSecondsAgo)),
+                new BasicHeader("Last-Modified", DateUtils.formatDate(oneSecondAgo))
+        };
+
+        HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(headers);
+        CacheValidityPolicy impl = new CacheValidityPolicy();
+        Assert.assertTrue(impl.getHeuristicFreshnessLifetimeSecs(entry, 0.1f, 10) >= 0);
+    }
+
     @Test
     public void testResponseIsFreshIfFreshnessLifetimeExceedsCurrentAge() {
         final HttpCacheEntry entry = HttpTestUtils.makeCacheEntry();

httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachedResponseSuitabilityChecker.java

@@ -223,4 +223,41 @@ public class TestCachedResponseSuitabilityChecker {
         Assert.assertFalse(impl.canCachedResponseBeUsed(host, request, entry, now));
     }
 
+    @Test
+    public void testSuitableIfCacheEntryIsHeuristicallyFreshEnough() {
+        Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L);
+        Date twentyOneSecondsAgo = new Date(now.getTime() - 21 * 1000L);
+
+        Header[] headers = {
+                new BasicHeader("Date", DateUtils.formatDate(oneSecondAgo)),
+                new BasicHeader("Last-Modified", DateUtils.formatDate(twentyOneSecondsAgo)),
+                new BasicHeader("Content-Length", "128")
+        };
+
+        entry = HttpTestUtils.makeCacheEntry(oneSecondAgo, oneSecondAgo, headers);
+
+        CacheConfig config = new CacheConfig();
+        config.setHeuristicCachingEnabled(true);
+        config.setHeuristicCoefficient(0.1f);
+        impl = new CachedResponseSuitabilityChecker(config);
+
+        Assert.assertTrue(impl.canCachedResponseBeUsed(host, request, entry, now));
+    }
+
+    @Test
+    public void testSuitableIfCacheEntryIsHeuristicallyFreshEnoughByDefault() {
+        Header[] headers = {
+                new BasicHeader("Date", DateUtils.formatDate(tenSecondsAgo)),
+                new BasicHeader("Content-Length", "128")
+        };
+
+        entry = getEntry(headers);
+
+        CacheConfig config = new CacheConfig();
+        config.setHeuristicCachingEnabled(true);
+        config.setHeuristicDefaultLifetime(20);
+        impl = new CachedResponseSuitabilityChecker(config);
+
+        Assert.assertTrue(impl.canCachedResponseBeUsed(host, request, entry, now));
+    }
 }