diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 6c64d4153..704a1c74b 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,6 +1,10 @@ Changes since 4.1 ALPHA2 ------------------- +* [HTTPCLIENT-958] Client cache no longer allows incomplete responses to be + passed on to the client. + Contributed by Jonathan Moore + * [HTTPCLIENT-951] Non-repeatable entity enclosing requests are not correctly retried when 'expect-continue' handshake is active. Contributed by Oleg Kalnichevski diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java index 340dc6aad..12c723308 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/CachingHttpClient.java @@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; @@ -51,6 +52,7 @@ import org.apache.http.client.cache.HttpCacheOperationException; import org.apache.http.client.cache.HttpCacheUpdateCallback; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.conn.ClientConnectionManager; +import org.apache.http.entity.ByteArrayEntity; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.message.BasicHttpResponse; import org.apache.http.message.BasicStatusLine; @@ -533,6 +535,32 @@ public class CachingHttpClient implements HttpClient { } } + protected HttpResponse correctIncompleteResponse(HttpResponse resp, + byte[] bodyBytes) { + int status = resp.getStatusLine().getStatusCode(); + if (status != HttpStatus.SC_OK + && status != HttpStatus.SC_PARTIAL_CONTENT) { + return resp; + } + Header hdr = resp.getFirstHeader("Content-Length"); + if (hdr == null) return resp; + int contentLength; + try { + contentLength = Integer.parseInt(hdr.getValue()); + } catch (NumberFormatException nfe) { + return resp; + } + if (bodyBytes.length >= contentLength) return resp; + HttpResponse error = + new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_BAD_GATEWAY, "Bad Gateway"); + error.setHeader("Content-Type","text/plain;charset=UTF-8"); + String msg = String.format("Received incomplete response with Content-Length %d but actual body length %d", contentLength, bodyBytes.length); + byte[] msgBytes = msg.getBytes(); + error.setHeader("Content-Length", String.format("%d",msgBytes.length)); + error.setEntity(new ByteArrayEntity(msgBytes)); + return error; + } + protected HttpResponse handleBackendResponse( HttpHost target, HttpRequest request, @@ -545,6 +573,7 @@ public class CachingHttpClient implements HttpClient { boolean cacheable = responseCachingPolicy.isResponseCacheable(request, backendResponse); + HttpResponse corrected = backendResponse; if (cacheable) { SizeLimitedResponseReader responseReader = getResponseReader(backendResponse); @@ -553,13 +582,17 @@ public class CachingHttpClient implements HttpClient { return responseReader.getReconstructedResponse(); } - CacheEntry entry = cacheEntryGenerator.generateEntry( - requestDate, - responseDate, - backendResponse, - responseReader.getResponseBytes()); - storeInCache(target, request, entry); - return responseGenerator.generateResponse(entry); + byte[] responseBytes = responseReader.getResponseBytes(); + corrected = correctIncompleteResponse(backendResponse, + responseBytes); + int correctedStatus = corrected.getStatusLine().getStatusCode(); + if (HttpStatus.SC_BAD_GATEWAY != correctedStatus) { + CacheEntry entry = cacheEntryGenerator + .generateEntry(requestDate, responseDate, corrected, + responseBytes); + storeInCache(target, request, entry); + return responseGenerator.generateResponse(entry); + } } String uri = uriExtractor.getURI(target, request); @@ -568,7 +601,7 @@ public class CachingHttpClient implements HttpClient { } catch (HttpCacheOperationException ex) { log.debug("Was unable to remove an entry from the cache based on the uri provided", ex); } - return backendResponse; + return corrected; } protected SizeLimitedResponseReader getResponseReader(HttpResponse backEndResponse) { diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/HttpTestUtils.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/HttpTestUtils.java index 7ebcd1b16..2975b49f5 100644 --- a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/HttpTestUtils.java +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/HttpTestUtils.java @@ -27,6 +27,7 @@ package org.apache.http.impl.client.cache; import java.io.InputStream; +import java.util.Random; import org.apache.http.Header; import org.apache.http.HttpEntity; @@ -35,6 +36,7 @@ import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; import org.apache.http.RequestLine; import org.apache.http.StatusLine; +import org.apache.http.entity.ByteArrayEntity; public class HttpTestUtils { @@ -203,4 +205,19 @@ public class HttpTestUtils { return (equivalent(r1.getRequestLine(), r2.getRequestLine()) && isEndToEndHeaderSubset(r1, r2)); } + + public static byte[] getRandomBytes(int nbytes) { + byte[] bytes = new byte[nbytes]; + (new Random()).nextBytes(bytes); + return bytes; + } + + /** Generates a response body with random content. + * @param nbytes length of the desired response body + * @return an {@link HttpEntity} + */ + public static HttpEntity makeBody(int nbytes) { + return new ByteArrayEntity(getRandomBytes(nbytes)); + } + } \ No newline at end of file diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java index fc7ed21f0..bda1dda99 100644 --- a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestCachingHttpClient.java @@ -26,7 +26,26 @@ */ package org.apache.http.impl.client.cache; -import org.apache.http.*; +import static junit.framework.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.ProtocolException; +import org.apache.http.ProtocolVersion; +import org.apache.http.RequestLine; +import org.apache.http.StatusLine; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient; import org.apache.http.client.ResponseHandler; @@ -41,6 +60,8 @@ import org.apache.http.conn.ssl.SSLSocketFactory; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; +import org.apache.http.message.BasicHttpResponse; +import org.apache.http.message.BasicStatusLine; import org.apache.http.params.HttpParams; import org.apache.http.protocol.HttpContext; import org.easymock.classextension.EasyMock; @@ -49,15 +70,6 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import static junit.framework.Assert.assertTrue; - public class TestCachingHttpClient { private static final ProtocolVersion HTTP_1_1 = new ProtocolVersion("HTTP",1,1); @@ -246,6 +258,8 @@ public class TestCachingHttpClient { generateCacheEntry(requestDate, responseDate, buf); storeInCacheWasCalled(); responseIsGeneratedFromCache(); + responseStatusLineIsInspectable(); + responseDoesNotHaveExplicitContentLength(); replayMocks(); HttpResponse result = impl.handleBackendResponse(host, mockRequest, requestDate, @@ -597,6 +611,8 @@ public class TestCachingHttpClient { generateCacheEntry(requestDate, responseDate, buf); storeInCacheWasCalled(); responseIsGeneratedFromCache(); + responseStatusLineIsInspectable(); + responseDoesNotHaveExplicitContentLength(); replayMocks(); @@ -927,6 +943,117 @@ public class TestCachingHttpClient { Assert.assertTrue(gotException); } + @Test + public void testCorrectIncompleteResponseDoesNotCorrectComplete200Response() + throws Exception { + HttpResponse resp = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + byte[] bytes = HttpTestUtils.getRandomBytes(128); + resp.setEntity(new ByteArrayEntity(bytes)); + resp.setHeader("Content-Length","128"); + + HttpResponse result = impl.correctIncompleteResponse(resp, bytes); + Assert.assertTrue(HttpTestUtils.semanticallyTransparent(resp, result)); + } + + @Test + public void testCorrectIncompleteResponseDoesNotCorrectComplete206Response() + throws Exception { + HttpResponse resp = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + byte[] bytes = HttpTestUtils.getRandomBytes(128); + resp.setEntity(new ByteArrayEntity(bytes)); + resp.setHeader("Content-Length","128"); + resp.setHeader("Content-Range","bytes 0-127/255"); + + HttpResponse result = impl.correctIncompleteResponse(resp, bytes); + Assert.assertTrue(HttpTestUtils.semanticallyTransparent(resp, result)); + } + + @Test + public void testCorrectIncompleteResponseGenerates502ForIncomplete200Response() + throws Exception { + HttpResponse resp = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + byte[] bytes = HttpTestUtils.getRandomBytes(128); + resp.setEntity(new ByteArrayEntity(bytes)); + resp.setHeader("Content-Length","256"); + + HttpResponse result = impl.correctIncompleteResponse(resp, bytes); + Assert.assertTrue(HttpStatus.SC_BAD_GATEWAY == result.getStatusLine().getStatusCode()); + } + + @Test + public void testCorrectIncompleteResponseDoesNotCorrectIncompleteNon200Or206Responses() + throws Exception { + HttpResponse resp = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_FORBIDDEN, "Forbidden"); + byte[] bytes = HttpTestUtils.getRandomBytes(128); + resp.setEntity(new ByteArrayEntity(bytes)); + resp.setHeader("Content-Length","256"); + + HttpResponse result = impl.correctIncompleteResponse(resp, bytes); + Assert.assertTrue(HttpTestUtils.semanticallyTransparent(resp, result)); + } + + @Test + public void testCorrectIncompleteResponseDoesNotCorrectResponsesWithoutExplicitContentLength() + throws Exception { + HttpResponse resp = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + byte[] bytes = HttpTestUtils.getRandomBytes(128); + resp.setEntity(new ByteArrayEntity(bytes)); + + HttpResponse result = impl.correctIncompleteResponse(resp, bytes); + Assert.assertTrue(HttpTestUtils.semanticallyTransparent(resp, result)); + } + + @Test + public void testCorrectIncompleteResponseDoesNotCorrectResponsesWithUnparseableContentLengthHeader() + throws Exception { + HttpResponse resp = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + byte[] bytes = HttpTestUtils.getRandomBytes(128); + resp.setHeader("Content-Length","foo"); + resp.setEntity(new ByteArrayEntity(bytes)); + + HttpResponse result = impl.correctIncompleteResponse(resp, bytes); + Assert.assertTrue(HttpTestUtils.semanticallyTransparent(resp, result)); + } + + @Test + public void testCorrectIncompleteResponseProvidesPlainTextErrorMessage() + throws Exception { + HttpResponse resp = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + byte[] bytes = HttpTestUtils.getRandomBytes(128); + resp.setEntity(new ByteArrayEntity(bytes)); + resp.setHeader("Content-Length","256"); + + HttpResponse result = impl.correctIncompleteResponse(resp, bytes); + Header ctype = result.getFirstHeader("Content-Type"); + Assert.assertEquals("text/plain;charset=UTF-8", ctype.getValue()); + } + + @Test + public void testCorrectIncompleteResponseProvidesNonEmptyErrorMessage() + throws Exception { + HttpResponse resp = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + byte[] bytes = HttpTestUtils.getRandomBytes(128); + resp.setEntity(new ByteArrayEntity(bytes)); + resp.setHeader("Content-Length","256"); + + HttpResponse result = impl.correctIncompleteResponse(resp, bytes); + int clen = Integer.parseInt(result.getFirstHeader("Content-Length").getValue()); + Assert.assertTrue(clen > 0); + HttpEntity body = result.getEntity(); + if (body.getContentLength() < 0) { + InputStream is = body.getContent(); + int bytes_read = 0; + while((is.read()) != -1) { + bytes_read++; + } + is.close(); + Assert.assertEquals(clen, bytes_read); + } else { + Assert.assertTrue(body.getContentLength() == clen); + } + } + + private byte[] readResponse(HttpResponse response) { try { ByteArrayOutputStream s1 = new ByteArrayOutputStream(); @@ -998,6 +1125,11 @@ public class TestCachingHttpClient { allow); } + private void responseDoesNotHaveExplicitContentLength() { + EasyMock.expect(mockBackendResponse.getFirstHeader("Content-Length")) + .andReturn(null).anyTimes(); + } + private byte[] responseReaderReturnsBufferOfSize(int bufferSize) { byte[] buffer = new byte[bufferSize]; org.easymock.EasyMock.expect(mockResponseReader.getResponseBytes()).andReturn(buffer); @@ -1051,8 +1183,14 @@ public class TestCachingHttpClient { } private void responseIsGeneratedFromCache() { - org.easymock.EasyMock.expect(mockResponseGenerator.generateResponse(mockCacheEntry)) - .andReturn(mockCachedResponse); + EasyMock.expect(mockResponseGenerator.generateResponse(mockCacheEntry)) + .andReturn(mockCachedResponse); + } + + private void responseStatusLineIsInspectable() { + StatusLine statusLine = new BasicStatusLine(HTTP_1_1, HttpStatus.SC_OK, "OK"); + EasyMock.expect(mockBackendResponse.getStatusLine()) + .andReturn(statusLine).anyTimes(); } private void responseIsGeneratedFromCache(CacheEntry entry) { diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestProtocolRequirements.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestProtocolRequirements.java index e5afe3f24..798cba430 100644 --- a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestProtocolRequirements.java +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/TestProtocolRequirements.java @@ -126,9 +126,7 @@ public class TestProtocolRequirements { } private HttpEntity makeBody(int nbytes) { - byte[] bytes = new byte[nbytes]; - (new Random()).nextBytes(bytes); - return new ByteArrayEntity(bytes); + return HttpTestUtils.makeBody(nbytes); } private IExpectationSetters backendExpectsAnyRequest() throws Exception { @@ -3663,6 +3661,613 @@ public class TestProtocolRequirements { HttpTestUtils.getCanonicalHeaderValue(result2, h)); } + /* "If a cache has a stored non-empty set of subranges for an + * entity, and an incoming response transfers another subrange, + * the cache MAY combine the new subrange with the existing set if + * both the following conditions are met: + * + * - Both the incoming response and the cache entry have a cache + * validator. + * + * - The two cache validators match using the strong comparison + * function (see section 13.3.3). + * + * If either requirement is not met, the cache MUST use only the + * most recent partial response (based on the Date values + * transmitted with every response, and using the incoming + * response if these values are equal or missing), and MUST + * discard the other partial information." + * + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.4 + */ + @Test + public void testCannotCombinePartialResponseIfIncomingResponseDoesNotHaveACacheValidator() + throws Exception { + + HttpRequest req1 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req1.setHeader("Range","bytes=0-49"); + + Date now = new Date(); + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date twoSecondsAgo = new Date(now.getTime() - 2 * 1000L); + + HttpResponse resp1 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp1.setEntity(makeBody(50)); + resp1.setHeader("Server","MockServer/1.0"); + resp1.setHeader("Date", DateUtils.formatDate(twoSecondsAgo)); + resp1.setHeader("Cache-Control","max-age=3600"); + resp1.setHeader("Content-Range","bytes 0-49/128"); + resp1.setHeader("ETag","\"etag1\""); + + backendExpectsAnyRequest().andReturn(resp1); + + HttpRequest req2 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req2.setHeader("Range","bytes=50-127"); + + HttpResponse resp2 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp2.setEntity(makeBody(78)); + resp2.setHeader("Cache-Control","max-age=3600"); + resp2.setHeader("Content-Range","bytes 50-127/128"); + resp2.setHeader("Server","MockServer/1.0"); + resp2.setHeader("Date", DateUtils.formatDate(oneSecondAgo)); + + backendExpectsAnyRequest().andReturn(resp2); + + HttpRequest req3 = new BasicHttpRequest("GET", "/", HTTP_1_1); + + HttpResponse resp3 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + resp3.setEntity(makeBody(128)); + resp3.setHeader("Server","MockServer/1.0"); + resp3.setHeader("Date", DateUtils.formatDate(now)); + + backendExpectsAnyRequest().andReturn(resp3); + + replayMocks(); + impl.execute(host, req1); + impl.execute(host, req2); + impl.execute(host, req3); + verifyMocks(); + } + + @Test + public void testCannotCombinePartialResponseIfCacheEntryDoesNotHaveACacheValidator() + throws Exception { + + Date now = new Date(); + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date twoSecondsAgo = new Date(now.getTime() - 2 * 1000L); + + HttpRequest req1 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req1.setHeader("Range","bytes=0-49"); + + HttpResponse resp1 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp1.setEntity(makeBody(50)); + resp1.setHeader("Cache-Control","max-age=3600"); + resp1.setHeader("Content-Range","bytes 0-49/128"); + resp1.setHeader("Server","MockServer/1.0"); + resp1.setHeader("Date", DateUtils.formatDate(twoSecondsAgo)); + + backendExpectsAnyRequest().andReturn(resp1); + + HttpRequest req2 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req2.setHeader("Range","bytes=50-127"); + + HttpResponse resp2 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp2.setEntity(makeBody(78)); + resp2.setHeader("Cache-Control","max-age=3600"); + resp2.setHeader("Content-Range","bytes 50-127/128"); + resp2.setHeader("ETag","\"etag1\""); + resp2.setHeader("Server","MockServer/1.0"); + resp2.setHeader("Date", DateUtils.formatDate(oneSecondAgo)); + + backendExpectsAnyRequest().andReturn(resp2); + + HttpRequest req3 = new BasicHttpRequest("GET", "/", HTTP_1_1); + + HttpResponse resp3 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + resp3.setEntity(makeBody(128)); + resp3.setHeader("Server","MockServer/1.0"); + resp3.setHeader("Date", DateUtils.formatDate(now)); + + backendExpectsAnyRequest().andReturn(resp3); + + replayMocks(); + impl.execute(host, req1); + impl.execute(host, req2); + impl.execute(host, req3); + verifyMocks(); + } + + @Test + public void testCannotCombinePartialResponseIfCacheValidatorsDoNotStronglyMatch() + throws Exception { + + Date now = new Date(); + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date twoSecondsAgo = new Date(now.getTime() - 2 * 1000L); + + HttpRequest req1 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req1.setHeader("Range","bytes=0-49"); + + HttpResponse resp1 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp1.setEntity(makeBody(50)); + resp1.setHeader("Cache-Control","max-age=3600"); + resp1.setHeader("Content-Range","bytes 0-49/128"); + resp1.setHeader("ETag","\"etag1\""); + resp1.setHeader("Server","MockServer/1.0"); + resp1.setHeader("Date", DateUtils.formatDate(twoSecondsAgo)); + + backendExpectsAnyRequest().andReturn(resp1); + + HttpRequest req2 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req2.setHeader("Range","bytes=50-127"); + + HttpResponse resp2 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp2.setEntity(makeBody(78)); + resp2.setHeader("Cache-Control","max-age=3600"); + resp2.setHeader("Content-Range","bytes 50-127/128"); + resp2.setHeader("ETag","\"etag2\""); + resp2.setHeader("Server","MockServer/1.0"); + resp2.setHeader("Date", DateUtils.formatDate(oneSecondAgo)); + + backendExpectsAnyRequest().andReturn(resp2); + + HttpRequest req3 = new BasicHttpRequest("GET", "/", HTTP_1_1); + + HttpResponse resp3 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + resp3.setEntity(makeBody(128)); + resp3.setHeader("Server","MockServer/1.0"); + resp3.setHeader("Date", DateUtils.formatDate(now)); + + backendExpectsAnyRequest().andReturn(resp3); + + replayMocks(); + impl.execute(host, req1); + impl.execute(host, req2); + impl.execute(host, req3); + verifyMocks(); + } + + @Test + public void testMustDiscardLeastRecentPartialResponseIfIncomingRequestDoesNotHaveCacheValidator() + throws Exception { + + Date now = new Date(); + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date twoSecondsAgo = new Date(now.getTime() - 2 * 1000L); + + HttpRequest req1 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req1.setHeader("Range","bytes=0-49"); + + HttpResponse resp1 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp1.setEntity(makeBody(50)); + resp1.setHeader("Cache-Control","max-age=3600"); + resp1.setHeader("Content-Range","bytes 0-49/128"); + resp1.setHeader("ETag","\"etag1\""); + resp1.setHeader("Server","MockServer/1.0"); + resp1.setHeader("Date", DateUtils.formatDate(twoSecondsAgo)); + + backendExpectsAnyRequest().andReturn(resp1); + + HttpRequest req2 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req2.setHeader("Range","bytes=50-127"); + + HttpResponse resp2 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp2.setEntity(makeBody(78)); + resp2.setHeader("Cache-Control","max-age=3600"); + resp2.setHeader("Content-Range","bytes 50-127/128"); + resp2.setHeader("Server","MockServer/1.0"); + resp2.setHeader("Date", DateUtils.formatDate(oneSecondAgo)); + + backendExpectsAnyRequest().andReturn(resp2); + + HttpRequest req3 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req3.setHeader("Range","bytes=0-49"); + + HttpResponse resp3 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + resp3.setEntity(makeBody(128)); + resp3.setHeader("Server","MockServer/1.0"); + resp3.setHeader("Date", DateUtils.formatDate(now)); + + // must make this request; cannot serve from cache + backendExpectsAnyRequest().andReturn(resp3); + + replayMocks(); + impl.execute(host, req1); + impl.execute(host, req2); + impl.execute(host, req3); + verifyMocks(); + } + + @Test + public void testMustDiscardLeastRecentPartialResponseIfCachedResponseDoesNotHaveCacheValidator() + throws Exception { + + Date now = new Date(); + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date twoSecondsAgo = new Date(now.getTime() - 2 * 1000L); + + HttpRequest req1 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req1.setHeader("Range","bytes=0-49"); + + HttpResponse resp1 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp1.setEntity(makeBody(50)); + resp1.setHeader("Cache-Control","max-age=3600"); + resp1.setHeader("Content-Range","bytes 0-49/128"); + resp1.setHeader("Server","MockServer/1.0"); + resp1.setHeader("Date", DateUtils.formatDate(twoSecondsAgo)); + + backendExpectsAnyRequest().andReturn(resp1); + + HttpRequest req2 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req2.setHeader("Range","bytes=50-127"); + + HttpResponse resp2 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp2.setEntity(makeBody(78)); + resp2.setHeader("Cache-Control","max-age=3600"); + resp2.setHeader("Content-Range","bytes 50-127/128"); + resp2.setHeader("ETag","\"etag1\""); + resp2.setHeader("Server","MockServer/1.0"); + resp2.setHeader("Date", DateUtils.formatDate(oneSecondAgo)); + + backendExpectsAnyRequest().andReturn(resp2); + + HttpRequest req3 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req3.setHeader("Range","bytes=0-49"); + + HttpResponse resp3 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + resp3.setEntity(makeBody(128)); + resp3.setHeader("Server","MockServer/1.0"); + resp3.setHeader("Date", DateUtils.formatDate(now)); + + // must make this request; cannot serve from cache + backendExpectsAnyRequest().andReturn(resp3); + + replayMocks(); + impl.execute(host, req1); + impl.execute(host, req2); + impl.execute(host, req3); + verifyMocks(); + } + + @Test + public void testMustDiscardLeastRecentPartialResponseIfCacheValidatorsDoNotStronglyMatch() + throws Exception { + + Date now = new Date(); + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date twoSecondsAgo = new Date(now.getTime() - 2 * 1000L); + + HttpRequest req1 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req1.setHeader("Range","bytes=0-49"); + + HttpResponse resp1 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp1.setEntity(makeBody(50)); + resp1.setHeader("Cache-Control","max-age=3600"); + resp1.setHeader("Content-Range","bytes 0-49/128"); + resp1.setHeader("Etag","\"etag1\""); + resp1.setHeader("Server","MockServer/1.0"); + resp1.setHeader("Date", DateUtils.formatDate(twoSecondsAgo)); + + backendExpectsAnyRequest().andReturn(resp1); + + HttpRequest req2 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req2.setHeader("Range","bytes=50-127"); + + HttpResponse resp2 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp2.setEntity(makeBody(78)); + resp2.setHeader("Cache-Control","max-age=3600"); + resp2.setHeader("Content-Range","bytes 50-127/128"); + resp2.setHeader("ETag","\"etag2\""); + resp2.setHeader("Server","MockServer/1.0"); + resp2.setHeader("Date", DateUtils.formatDate(oneSecondAgo)); + + backendExpectsAnyRequest().andReturn(resp2); + + HttpRequest req3 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req3.setHeader("Range","bytes=0-49"); + + HttpResponse resp3 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + resp3.setEntity(makeBody(128)); + resp3.setHeader("Server","MockServer/1.0"); + resp3.setHeader("Date", DateUtils.formatDate(now)); + + // must make this request; cannot serve from cache + backendExpectsAnyRequest().andReturn(resp3); + + replayMocks(); + impl.execute(host, req1); + impl.execute(host, req2); + impl.execute(host, req3); + verifyMocks(); + } + + @Test + public void testMustDiscardLeastRecentPartialResponseIfCacheValidatorsDoNotStronglyMatchEvenIfResponsesOutOfOrder() + throws Exception { + + Date now = new Date(); + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + Date twoSecondsAgo = new Date(now.getTime() - 2 * 1000L); + + HttpRequest req1 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req1.setHeader("Range","bytes=0-49"); + + HttpResponse resp1 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp1.setEntity(makeBody(50)); + resp1.setHeader("Cache-Control","max-age=3600"); + resp1.setHeader("Content-Range","bytes 0-49/128"); + resp1.setHeader("Etag","\"etag1\""); + resp1.setHeader("Server","MockServer/1.0"); + resp1.setHeader("Date", DateUtils.formatDate(oneSecondAgo)); + + backendExpectsAnyRequest().andReturn(resp1); + + HttpRequest req2 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req2.setHeader("Range","bytes=50-127"); + + HttpResponse resp2 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp2.setEntity(makeBody(78)); + resp2.setHeader("Cache-Control","max-age=3600"); + resp2.setHeader("Content-Range","bytes 50-127/128"); + resp2.setHeader("ETag","\"etag2\""); + resp2.setHeader("Server","MockServer/1.0"); + resp2.setHeader("Date", DateUtils.formatDate(twoSecondsAgo)); + + backendExpectsAnyRequest().andReturn(resp2); + + HttpRequest req3 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req3.setHeader("Range","bytes=50-127"); + + HttpResponse resp3 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + resp3.setEntity(makeBody(128)); + resp3.setHeader("Server","MockServer/1.0"); + resp3.setHeader("Date", DateUtils.formatDate(now)); + + // must make this request; cannot serve from cache + backendExpectsAnyRequest().andReturn(resp3); + + replayMocks(); + impl.execute(host, req1); + impl.execute(host, req2); + impl.execute(host, req3); + verifyMocks(); + } + + @Test + public void testMustDiscardCachedPartialResponseIfCacheValidatorsDoNotStronglyMatchAndDateHeadersAreEqual() + throws Exception { + + Date now = new Date(); + Date oneSecondAgo = new Date(now.getTime() - 1 * 1000L); + + HttpRequest req1 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req1.setHeader("Range","bytes=0-49"); + + HttpResponse resp1 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp1.setEntity(makeBody(50)); + resp1.setHeader("Cache-Control","max-age=3600"); + resp1.setHeader("Content-Range","bytes 0-49/128"); + resp1.setHeader("Etag","\"etag1\""); + resp1.setHeader("Server","MockServer/1.0"); + resp1.setHeader("Date", DateUtils.formatDate(oneSecondAgo)); + + backendExpectsAnyRequest().andReturn(resp1); + + HttpRequest req2 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req2.setHeader("Range","bytes=50-127"); + + HttpResponse resp2 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_PARTIAL_CONTENT, "Partial Content"); + resp2.setEntity(makeBody(78)); + resp2.setHeader("Cache-Control","max-age=3600"); + resp2.setHeader("Content-Range","bytes 50-127/128"); + resp2.setHeader("ETag","\"etag2\""); + resp2.setHeader("Server","MockServer/1.0"); + resp2.setHeader("Date", DateUtils.formatDate(oneSecondAgo)); + + backendExpectsAnyRequest().andReturn(resp2); + + HttpRequest req3 = new BasicHttpRequest("GET", "/", HTTP_1_1); + req3.setHeader("Range","bytes=0-49"); + + HttpResponse resp3 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_OK, "OK"); + resp3.setEntity(makeBody(128)); + resp3.setHeader("Server","MockServer/1.0"); + resp3.setHeader("Date", DateUtils.formatDate(now)); + + // must make this request; cannot serve from cache + backendExpectsAnyRequest().andReturn(resp3); + + replayMocks(); + impl.execute(host, req1); + impl.execute(host, req2); + impl.execute(host, req3); + verifyMocks(); + } + + /* "When the cache receives a subsequent request whose Request-URI + * specifies one or more cache entries including a Vary header + * field, the cache MUST NOT use such a cache entry to construct a + * response to the new request unless all of the selecting + * request-headers present in the new request match the + * corresponding stored request-headers in the original request." + * + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 + */ + @Test + public void testCannotUseVariantCacheEntryIfNotAllSelectingRequestHeadersMatch() + throws Exception { + + HttpRequest req1 = new BasicHttpRequest("GET","/",HTTP_1_1); + req1.setHeader("Accept-Encoding","gzip"); + + HttpResponse resp1 = make200Response(); + resp1.setHeader("ETag","\"etag1\""); + resp1.setHeader("Cache-Control","max-age=3600"); + resp1.setHeader("Vary","Accept-Encoding"); + + backendExpectsAnyRequest().andReturn(resp1); + + HttpRequest req2 = new BasicHttpRequest("GET","/",HTTP_1_1); + req2.removeHeaders("Accept-Encoding"); + + HttpResponse resp2 = make200Response(); + resp2.setHeader("ETag","\"etag1\""); + resp2.setHeader("Cache-Control","max-age=3600"); + + // not allowed to have a cache hit; must forward request + backendExpectsAnyRequest().andReturn(resp2); + + replayMocks(); + impl.execute(host, req1); + impl.execute(host, req2); + verifyMocks(); + } + + /* "A Vary header field-value of "*" always fails to match and + * subsequent requests on that resource can only be properly + * interpreted by the origin server." + * + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 + */ + @Test + public void testCannotServeFromCacheForVaryStar() throws Exception { + HttpRequest req1 = new BasicHttpRequest("GET","/",HTTP_1_1); + + HttpResponse resp1 = make200Response(); + resp1.setHeader("ETag","\"etag1\""); + resp1.setHeader("Cache-Control","max-age=3600"); + resp1.setHeader("Vary","*"); + + backendExpectsAnyRequest().andReturn(resp1); + + HttpRequest req2 = new BasicHttpRequest("GET","/",HTTP_1_1); + + HttpResponse resp2 = make200Response(); + resp2.setHeader("ETag","\"etag1\""); + resp2.setHeader("Cache-Control","max-age=3600"); + + // not allowed to have a cache hit; must forward request + backendExpectsAnyRequest().andReturn(resp2); + + replayMocks(); + impl.execute(host, req1); + impl.execute(host, req2); + verifyMocks(); + } + + /* " If the selecting request header fields for the cached entry + * do not match the selecting request header fields of the new + * request, then the cache MUST NOT use a cached entry to satisfy + * the request unless it first relays the new request to the + * origin server in a conditional request and the server responds + * with 304 (Not Modified), including an entity tag or + * Content-Location that indicates the entity to be used. + * + * If an entity tag was assigned to a cached representation, the + * forwarded request SHOULD be conditional and include the entity + * tags in an If-None-Match header field from all its cache + * entries for the resource. This conveys to the server the set of + * entities currently held by the cache, so that if any one of + * these entities matches the requested entity, the server can use + * the ETag header field in its 304 (Not Modified) response to + * tell the cache which entry is appropriate. If the entity-tag of + * the new response matches that of an existing entry, the new + * response SHOULD be used to update the header fields of the + * existing entry, and the result MUST be returned to the client. + * + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6 + */ + @Test + public void testNonmatchingVariantCannotBeServedFromCacheUnlessConditionallyValidated() + throws Exception { + + HttpRequest req1 = new BasicHttpRequest("GET","/",HTTP_1_1); + req1.setHeader("User-Agent","MyBrowser/1.0"); + + HttpResponse resp1 = make200Response(); + resp1.setHeader("ETag","\"etag1\""); + resp1.setHeader("Cache-Control","max-age=3600"); + resp1.setHeader("Vary","User-Agent"); + resp1.setHeader("Content-Type","application/octet-stream"); + + backendExpectsAnyRequest().andReturn(resp1); + + HttpRequest req2 = new BasicHttpRequest("GET","/",HTTP_1_1); + req2.setHeader("User-Agent","MyBrowser/1.5"); + + HttpRequest conditional = new BasicHttpRequest("GET","/",HTTP_1_1); + conditional.setHeader("User-Agent","MyBrowser/1.5"); + conditional.setHeader("If-None-Match","\"etag1\""); + + HttpResponse resp200 = make200Response(); + resp200.setHeader("ETag","\"etag1\""); + resp200.setHeader("Vary","User-Agent"); + + HttpResponse resp304 = new BasicHttpResponse(HTTP_1_1, HttpStatus.SC_NOT_MODIFIED, "Not Modified"); + resp304.setHeader("ETag","\"etag1\""); + resp304.setHeader("Vary","User-Agent"); + + Capture condCap = new Capture(); + Capture uncondCap = new Capture(); + + EasyMock.expect(mockBackend.execute(EasyMock.isA(HttpHost.class), + EasyMock.and(eqRequest(conditional), + EasyMock.capture(condCap)), + (HttpContext)EasyMock.isNull())) + .andReturn(resp304).times(0,1); + EasyMock.expect(mockBackend.execute(EasyMock.isA(HttpHost.class), + EasyMock.and(eqRequest(req2), + EasyMock.capture(uncondCap)), + (HttpContext)EasyMock.isNull())) + .andReturn(resp200).times(0,1); + + replayMocks(); + impl.execute(host, req1); + HttpResponse result = impl.execute(host, req2); + verifyMocks(); + + if (HttpStatus.SC_OK == result.getStatusLine().getStatusCode()) { + Assert.assertTrue(condCap.hasCaptured() + || uncondCap.hasCaptured()); + if (uncondCap.hasCaptured()) { + Assert.assertTrue(HttpTestUtils.semanticallyTransparent(resp200, result)); + } + } + } + + /* "A cache that receives an incomplete response (for example, + * with fewer bytes of data than specified in a Content-Length + * header) MAY store the response. However, the cache MUST treat + * this as a partial response. Partial responses MAY be combined + * as described in section 13.5.4; the result might be a full + * response or might still be partial. A cache MUST NOT return a + * partial response to a client without explicitly marking it as + * such, using the 206 (Partial Content) status code. A cache MUST + * NOT return a partial response using a status code of 200 (OK)." + * + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.8 + */ + @Test + public void testIncompleteResponseMustNotBeReturnedToClientWithoutMarkingItAs206() throws Exception { + originResponse.setEntity(makeBody(128)); + originResponse.setHeader("Content-Length","256"); + + backendExpectsAnyRequest().andReturn(originResponse); + + replayMocks(); + HttpResponse result = impl.execute(host, request); + verifyMocks(); + + int status = result.getStatusLine().getStatusCode(); + Assert.assertFalse(HttpStatus.SC_OK == status); + if (status > 200 && status <= 299 + && HttpTestUtils.equivalent(originResponse.getEntity(), + result.getEntity())) { + Assert.assertTrue(HttpStatus.SC_PARTIAL_CONTENT == status); + } + } + private class FakeHeaderGroup extends HeaderGroup{ public void addHeader(String name, String value){