HTTPCLIENT-985: cache module should populate Via header to capture upstream and downstream protocols

Contributed by Jonathan Moore <jonathan_moore at comcast.com>


git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk@992117 13f79535-47bb-0310-9956-ffa450edef68

httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java

@@ -35,11 +35,13 @@ import java.util.concurrent.atomic.AtomicLong;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpHost;
+import org.apache.http.HttpMessage;
 import org.apache.http.HttpRequest;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
 import org.apache.http.HttpVersion;
 import org.apache.http.ProtocolException;
+import org.apache.http.ProtocolVersion;
 import org.apache.http.RequestLine;
 import org.apache.http.annotation.ThreadSafe;
 import org.apache.http.client.ClientProtocolException;
@@ -55,6 +57,7 @@ import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.message.BasicHttpResponse;
 import org.apache.http.params.HttpParams;
 import org.apache.http.protocol.HttpContext;
+import org.apache.http.util.VersionInfo;
 
 /**
  * @since 4.1
@@ -358,6 +361,8 @@ public class CachingHttpClient implements HttpClient {
         // default response context
         setResponseStatus(context, CacheResponseStatus.CACHE_MISS);
 
+        String via = generateViaHeader(request);
+
         if (clientRequestsOurOptions(request)) {
             setResponseStatus(context, CacheResponseStatus.CACHE_MODULE_RESPONSE);
             return new OptionsHttp11Response();
@@ -375,6 +380,7 @@ public class CachingHttpClient implements HttpClient {
         } catch (ProtocolException e) {
             throw new ClientProtocolException(e);
         }
+        request.addHeader("Via",via);
 
         responseCache.flushInvalidatedCacheEntriesFor(target, request);
 
@@ -429,6 +435,19 @@ public class CachingHttpClient implements HttpClient {
         return callBackend(target, request, context);
     }
 
+    private String generateViaHeader(HttpMessage msg) {
+        final VersionInfo vi = VersionInfo.loadVersionInfo("org.apache.http.client", getClass().getClassLoader());
+        final String release = (vi != null) ? vi.getRelease() : VersionInfo.UNAVAILABLE;
+        final ProtocolVersion pv = msg.getProtocolVersion();
+        if ("http".equalsIgnoreCase(pv.getProtocol())) {
+            return String.format("%d.%d localhost (Apache-HttpClient/%s (cache))",
+                pv.getMajor(), pv.getMinor(), release);
+        } else {
+            return String.format("%s/%d.%d localhost (Apache-HttpClient/%s (cache))",
+                    pv.getProtocol(), pv.getMajor(), pv.getMinor(), release);
+        }
+    }
+
     private void setResponseStatus(final HttpContext context, final CacheResponseStatus value) {
         if (context != null) {
             context.setAttribute(CACHE_RESPONSE_STATUS, value);
@@ -469,6 +488,7 @@ public class CachingHttpClient implements HttpClient {
 
         log.debug("Calling the backend");
         HttpResponse backendResponse = backend.execute(target, request, context);
+        backendResponse.addHeader("Via", generateViaHeader(backendResponse));
         return handleBackendResponse(target, request, requestDate, getCurrentDate(),
                 backendResponse);
 

httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java

@@ -55,6 +55,7 @@ import org.apache.http.params.BasicHttpParams;
 import org.apache.http.params.HttpParams;
 import org.apache.http.protocol.BasicHttpContext;
 import org.apache.http.protocol.HttpContext;
+import org.easymock.Capture;
 import org.easymock.classextension.EasyMock;
 import org.junit.Assert;
 import org.junit.Before;
@@ -371,10 +372,11 @@ public class TestCachingHttpClient {
     @Test
     public void testCallBackendMakesBackEndRequestAndHandlesResponse() throws Exception {
         mockImplMethods(GET_CURRENT_DATE, HANDLE_BACKEND_RESPONSE);
+        HttpResponse resp = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK");
         getCurrentDateReturns(requestDate);
-        backendCallWasMadeWithRequest(request);
+        backendCallWasMade(request, resp);
         getCurrentDateReturns(responseDate);
-        handleBackendResponseReturnsResponse(request, mockBackendResponse);
+        handleBackendResponseReturnsResponse(request, resp);
 
         replayMocks();
 
@@ -755,6 +757,30 @@ public class TestCachingHttpClient {
                 context.getAttribute(CachingHttpClient.CACHE_RESPONSE_STATUS));
     }
 
+    @Test
+    public void testRecordsClientProtocolInViaHeaderIfRequestNotServableFromCache()
+        throws Exception {
+        impl = new CachingHttpClient(mockBackend);
+        HttpRequest req = new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_0);
+        req.setHeader("Cache-Control","no-cache");
+        HttpResponse resp = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_NO_CONTENT, "No Content");
+        Capture<HttpRequest> cap = new Capture<HttpRequest>();
+
+        EasyMock.expect(mockBackend.execute(EasyMock.isA(HttpHost.class),
+                EasyMock.capture(cap), EasyMock.isA(HttpContext.class)))
+            .andReturn(resp);
+
+        replayMocks();
+        impl.execute(host, req, context);
+        verifyMocks();
+
+        HttpRequest captured = cap.getValue();
+        String via = captured.getFirstHeader("Via").getValue();
+        String proto = via.split("\\s+")[0];
+        Assert.assertTrue("http/1.0".equalsIgnoreCase(proto) ||
+                "1.0".equalsIgnoreCase(proto));
+    }
+
     @Test
     public void testSetsCacheMissContextIfRequestNotServableFromCache()
         throws Exception {
@@ -774,6 +800,46 @@ public class TestCachingHttpClient {
                 context.getAttribute(CachingHttpClient.CACHE_RESPONSE_STATUS));
     }
 
+    @Test
+    public void testSetsViaHeaderOnResponseIfRequestNotServableFromCache()
+            throws Exception {
+        impl = new CachingHttpClient(mockBackend);
+        HttpRequest req = new HttpGet("http://foo.example.com/");
+        req.setHeader("Cache-Control","no-cache");
+        HttpResponse resp = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_NO_CONTENT, "No Content");
+
+        EasyMock.expect(mockBackend.execute(EasyMock.isA(HttpHost.class),
+                EasyMock.isA(HttpRequest.class), (HttpContext)EasyMock.isNull()))
+                .andReturn(resp);
+
+        replayMocks();
+        HttpResponse result = impl.execute(host, req);
+        verifyMocks();
+        Assert.assertNotNull(result.getFirstHeader("Via"));
+    }
+
+    @Test
+    public void testSetsViaHeaderOnResponseForCacheMiss()
+        throws Exception {
+        impl = new CachingHttpClient(mockBackend);
+        HttpRequest req1 = new HttpGet("http://foo.example.com/");
+        HttpResponse resp1 = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK");
+        resp1.setEntity(HttpTestUtils.makeBody(128));
+        resp1.setHeader("Content-Length","128");
+        resp1.setHeader("ETag","\"etag\"");
+        resp1.setHeader("Date", DateUtils.formatDate(new Date()));
+        resp1.setHeader("Cache-Control","public, max-age=3600");
+
+        EasyMock.expect(mockBackend.execute(EasyMock.isA(HttpHost.class),
+                EasyMock.isA(HttpRequest.class), EasyMock.isA(HttpContext.class)))
+            .andReturn(resp1);
+
+        replayMocks();
+        HttpResponse result = impl.execute(host, req1, new BasicHttpContext());
+        verifyMocks();
+        Assert.assertNotNull(result.getFirstHeader("Via"));
+    }
+
     @Test
     public void testSetsCacheHitContextIfRequestServedFromCache()
         throws Exception {
@@ -799,6 +865,30 @@ public class TestCachingHttpClient {
                 context.getAttribute(CachingHttpClient.CACHE_RESPONSE_STATUS));
     }
 
+    @Test
+    public void testSetsViaHeaderOnResponseIfRequestServedFromCache()
+        throws Exception {
+        impl = new CachingHttpClient(mockBackend);
+        HttpRequest req1 = new HttpGet("http://foo.example.com/");
+        HttpRequest req2 = new HttpGet("http://foo.example.com/");
+        HttpResponse resp1 = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK");
+        resp1.setEntity(HttpTestUtils.makeBody(128));
+        resp1.setHeader("Content-Length","128");
+        resp1.setHeader("ETag","\"etag\"");
+        resp1.setHeader("Date", DateUtils.formatDate(new Date()));
+        resp1.setHeader("Cache-Control","public, max-age=3600");
+
+        EasyMock.expect(mockBackend.execute(EasyMock.isA(HttpHost.class),
+                EasyMock.isA(HttpRequest.class), (HttpContext)EasyMock.isNull()))
+            .andReturn(resp1);
+
+        replayMocks();
+        impl.execute(host, req1);
+        HttpResponse result = impl.execute(host, req2);
+        verifyMocks();
+        Assert.assertNotNull(result.getFirstHeader("Via"));
+    }
+
     @Test
     public void testSetsValidatedContextIfRequestWasSuccessfullyValidated()
         throws Exception {
@@ -838,6 +928,45 @@ public class TestCachingHttpClient {
                 context.getAttribute(CachingHttpClient.CACHE_RESPONSE_STATUS));
     }
 
+    @Test
+    public void testSetsViaHeaderIfRequestWasSuccessfullyValidated()
+        throws Exception {
+        Date now = new Date();
+        Date tenSecondsAgo = new Date(now.getTime() - 10 * 1000L);
+
+        impl = new CachingHttpClient(mockBackend);
+        HttpRequest req1 = new HttpGet("http://foo.example.com/");
+        HttpRequest req2 = new HttpGet("http://foo.example.com/");
+
+        HttpResponse resp1 = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK");
+        resp1.setEntity(HttpTestUtils.makeBody(128));
+        resp1.setHeader("Content-Length","128");
+        resp1.setHeader("ETag","\"etag\"");
+        resp1.setHeader("Date", DateUtils.formatDate(tenSecondsAgo));
+        resp1.setHeader("Cache-Control","public, max-age=5");
+
+        HttpResponse resp2 = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK");
+        resp2.setEntity(HttpTestUtils.makeBody(128));
+        resp2.setHeader("Content-Length","128");
+        resp2.setHeader("ETag","\"etag\"");
+        resp2.setHeader("Date", DateUtils.formatDate(tenSecondsAgo));
+        resp2.setHeader("Cache-Control","public, max-age=5");
+
+        EasyMock.expect(mockBackend.execute(EasyMock.isA(HttpHost.class),
+                EasyMock.isA(HttpRequest.class), EasyMock.isA(HttpContext.class)))
+            .andReturn(resp1);
+        EasyMock.expect(mockBackend.execute(EasyMock.isA(HttpHost.class),
+                EasyMock.isA(HttpRequest.class), EasyMock.isA(HttpContext.class)))
+            .andReturn(resp2);
+
+        replayMocks();
+        impl.execute(host, req1, new BasicHttpContext());
+        HttpResponse result = impl.execute(host, req2, context);
+        verifyMocks();
+        Assert.assertNotNull(result.getFirstHeader("Via"));
+    }
+
+
     @Test
     public void testSetsModuleResponseContextIfValidationRequiredButFailed()
         throws Exception {
@@ -902,6 +1031,38 @@ public class TestCachingHttpClient {
                 context.getAttribute(CachingHttpClient.CACHE_RESPONSE_STATUS));
     }
 
+    @Test
+    public void testSetViaHeaderIfValidationFailsButNotRequired()
+        throws Exception {
+        Date now = new Date();
+        Date tenSecondsAgo = new Date(now.getTime() - 10 * 1000L);
+
+        impl = new CachingHttpClient(mockBackend);
+        HttpRequest req1 = new HttpGet("http://foo.example.com/");
+        HttpRequest req2 = new HttpGet("http://foo.example.com/");
+
+        HttpResponse resp1 = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK");
+        resp1.setEntity(HttpTestUtils.makeBody(128));
+        resp1.setHeader("Content-Length","128");
+        resp1.setHeader("ETag","\"etag\"");
+        resp1.setHeader("Date", DateUtils.formatDate(tenSecondsAgo));
+        resp1.setHeader("Cache-Control","public, max-age=5");
+
+        EasyMock.expect(mockBackend.execute(EasyMock.isA(HttpHost.class),
+                EasyMock.isA(HttpRequest.class), EasyMock.isA(HttpContext.class)))
+            .andReturn(resp1);
+        EasyMock.expect(mockBackend.execute(EasyMock.isA(HttpHost.class),
+                EasyMock.isA(HttpRequest.class), EasyMock.isA(HttpContext.class)))
+            .andThrow(new IOException());
+
+        replayMocks();
+        impl.execute(host, req1, new BasicHttpContext());
+        HttpResponse result = impl.execute(host, req2, context);
+        verifyMocks();
+        Assert.assertNotNull(result.getFirstHeader("Via"));
+    }
+
+
     @Test
     public void testIsSharedCache() {
         Assert.assertTrue(impl.isSharedCache());
@@ -967,6 +1128,13 @@ public class TestCachingHttpClient {
                 EasyMock.<HttpContext>anyObject())).andReturn(mockBackendResponse);
     }
 
+    private void backendCallWasMade(HttpRequest request, HttpResponse response) throws IOException {
+        EasyMock.expect(mockBackend.execute(
+                EasyMock.<HttpHost>anyObject(),
+                EasyMock.same(request),
+                EasyMock.<HttpContext>anyObject())).andReturn(response);
+    }
+
     private void responsePolicyAllowsCaching(boolean allow) {
         EasyMock.expect(
                 mockResponsePolicy.isResponseCacheable(

httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestProtocolRequirements.java

@@ -31,6 +31,8 @@ import java.io.InputStream;
 import java.net.SocketTimeoutException;
 import java.util.Date;
 import java.util.Random;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.http.Header;
 import org.apache.http.HeaderElement;
@@ -5463,5 +5465,147 @@ public class TestProtocolRequirements extends AbstractProtocolTest {
         }
     }
 
+    /* "The Via general-header field MUST be used by gateways and proxies
+     * to indicate the intermediate protocols and recipients between the
+     * user agent and the server on requests, and between the origin server
+     * and the client on responses."
+     *
+     * http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.45
+     */
+    @Test
+    public void testProperlyFormattedViaHeaderIsAddedToRequests() throws Exception {
+        Capture<HttpRequest> cap = new Capture<HttpRequest>();
+        request.removeHeaders("Via");
+        EasyMock.expect(mockBackend.execute(EasyMock.isA(HttpHost.class),
+                EasyMock.capture(cap), (HttpContext)EasyMock.isNull()))
+            .andReturn(originResponse);
 
+        replayMocks();
+        impl.execute(host, request);
+        verifyMocks();
+
+        HttpRequest captured = cap.getValue();
+        String via = captured.getFirstHeader("Via").getValue();
+        assertValidViaHeader(via);
+    }
+
+    @Test
+    public void testProperlyFormattedViaHeaderIsAddedToResponses() throws Exception {
+        originResponse.removeHeaders("Via");
+        backendExpectsAnyRequest().andReturn(originResponse);
+        replayMocks();
+        HttpResponse result = impl.execute(host, request);
+        verifyMocks();
+        assertValidViaHeader(result.getFirstHeader("Via").getValue());
+    }
+
+
+    private void assertValidViaHeader(String via) {
+        //    	Via =  "Via" ":" 1#( received-protocol received-by [ comment ] )
+        //        received-protocol = [ protocol-name "/" ] protocol-version
+        //        protocol-name     = token
+        //        protocol-version  = token
+        //        received-by       = ( host [ ":" port ] ) | pseudonym
+        //        pseudonym         = token
+
+        String[] parts = via.split("\\s+");
+        Assert.assertTrue(parts.length >= 2);
+
+        // received protocol
+        String receivedProtocol = parts[0];
+        String[] protocolParts = receivedProtocol.split("/");
+        Assert.assertTrue(protocolParts.length >= 1);
+        Assert.assertTrue(protocolParts.length <= 2);
+
+        final String tokenRegexp = "[^\\p{Cntrl}()<>@,;:\\\\\"/\\[\\]?={} \\t]+";
+        for(String protocolPart : protocolParts) {
+            Assert.assertTrue(Pattern.matches(tokenRegexp, protocolPart));
+        }
+
+        // received-by
+        if (!Pattern.matches(tokenRegexp, parts[1])) {
+            // host : port
+            new HttpHost(parts[1]);
+        }
+
+        // comment
+        if (parts.length > 2) {
+            StringBuilder buf = new StringBuilder(parts[2]);
+            for(int i=3; i<parts.length; i++) {
+                buf.append(" "); buf.append(parts[i]);
+            }
+            Assert.assertTrue(isValidComment(buf.toString()));
+        }
+    }
+
+    private boolean isValidComment(String s) {
+        final String leafComment = "^\\(([^\\p{Cntrl}()]|\\\\\\p{ASCII})*\\)$";
+        final String nestedPrefix = "^\\(([^\\p{Cntrl}()]|\\\\\\p{ASCII})*\\(";
+        final String nestedSuffix = "\\)([^\\p{Cntrl}()]|\\\\\\p{ASCII})*\\)$";
+
+        if (Pattern.matches(leafComment,s)) return true;
+        Matcher pref = Pattern.compile(nestedPrefix).matcher(s);
+        Matcher suff = Pattern.compile(nestedSuffix).matcher(s);
+        if (!pref.find()) return false;
+        if (!suff.find()) return false;
+        return isValidComment(s.substring(pref.end() - 1, suff.start() + 1));
+    }
+
+
+    /*
+     * "The received-protocol indicates the protocol version of the message
+     * received by the server or client along each segment of the request/
+     * response chain. The received-protocol version is appended to the Via
+     * field value when the message is forwarded so that information about
+     * the protocol capabilities of upstream applications remains visible
+     * to all recipients."
+     *
+     * http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.45
+     */
+    @Test
+    public void testViaHeaderOnRequestProperlyRecordsClientProtocol()
+            throws Exception {
+        request = new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_0);
+        request.removeHeaders("Via");
+        Capture<HttpRequest> cap = new Capture<HttpRequest>();
+        EasyMock.expect(mockBackend.execute(EasyMock.isA(HttpHost.class),
+                EasyMock.capture(cap), (HttpContext)EasyMock.isNull()))
+                .andReturn(originResponse);
+
+        replayMocks();
+        impl.execute(host, request);
+        verifyMocks();
+
+        HttpRequest captured = cap.getValue();
+        String via = captured.getFirstHeader("Via").getValue();
+        String protocol = via.split("\\s+")[0];
+        String[] protoParts = protocol.split("/");
+        if (protoParts.length > 1) {
+            Assert.assertTrue("http".equalsIgnoreCase(protoParts[0]));
+        }
+        Assert.assertEquals("1.0",protoParts[protoParts.length-1]);
+    }
+
+    @Test
+    public void testViaHeaderOnResponseProperlyRecordsOriginProtocol()
+        throws Exception {
+
+        originResponse = new BasicHttpResponse(HttpVersion.HTTP_1_0, HttpStatus.SC_NO_CONTENT, "No Content");
+
+        backendExpectsAnyRequest().andReturn(originResponse);
+
+        replayMocks();
+        HttpResponse result = impl.execute(host, request);
+        verifyMocks();
+
+        String via = result.getFirstHeader("Via").getValue();
+        String protocol = via.split("\\s+")[0];
+        String[] protoParts = protocol.split("/");
+        Assert.assertTrue(protoParts.length >= 1);
+        Assert.assertTrue(protoParts.length <= 2);
+        if (protoParts.length > 1) {
+            Assert.assertTrue("http".equalsIgnoreCase(protoParts[0]));
+        }
+        Assert.assertEquals("1.0", protoParts[protoParts.length - 1]);
+    }
 }