HTTPCLIENT-1164: Implemented a new CompressionDecorator aimed at replacing

the ContentEncodingHttpClient. This reuses the same request/response
interceptors with some minor additions to make the resulting response
semantically consistent (i.e. if a response is uncompressed, we mark the
entity as having an unknown length and remove the Content-Length header
before passing the response upstream). This allows the CompressionDecorator
and CachingHttpClient to be wired around a DefaultHttpClient in either order
and still get cache hits, depending on whether you want to cache compressed
or uncompressed bodies.

Minor change to ResponseContentEncoding to set a context variable that the
uncompression was applied. Only other change was to deprecate the
ContentEncodingHttpClient.


git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk@1301090 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Moore 2012-03-15 17:02:07 +00:00
parent cef44fa283
commit 54c7e992fb
6 changed files with 618 additions and 0 deletions

View File

@ -21,6 +21,8 @@ notable enhancements in HttpClient:
Changelog
-------------------
* [HTTPCLIENT-1164] Compressed entities are not being cached properly.
Contributed by Jon Moore <jonm at apache dot org>.
* [HTTPCLIENT-1154] MemcachedHttpCacheStorage should allow client to
specify custom prefix string for keys.

View File

@ -52,6 +52,8 @@
@Immutable
public class ResponseContentEncoding implements HttpResponseInterceptor {
public static final String UNCOMPRESSED = "http.client.response.uncompressed";
/**
* Handles the following {@code Content-Encoding}s by
* using the appropriate decompressor to wrap the response Entity:
@ -80,9 +82,11 @@ public void process(
String codecname = codec.getName().toLowerCase(Locale.US);
if ("gzip".equals(codecname) || "x-gzip".equals(codecname)) {
response.setEntity(new GzipDecompressingEntity(response.getEntity()));
if (context != null) context.setAttribute(UNCOMPRESSED, true);
return;
} else if ("deflate".equals(codecname)) {
response.setEntity(new DeflateDecompressingEntity(response.getEntity()));
if (context != null) context.setAttribute(UNCOMPRESSED, true);
return;
} else if ("identity".equals(codecname)) {

View File

@ -0,0 +1,168 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package org.apache.http.impl.client;
import java.io.IOException;
import java.net.URI;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.RequestAcceptEncoding;
import org.apache.http.client.protocol.ResponseContentEncoding;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
/*
* <p>Decorator adding support for compressed responses. This class sets
* the <code>Accept-Encoding</code> header on requests to indicate
* support for the <code>gzip</code> and <code>deflate</code>
* compression schemes; it then checks the <code>Content-Encoding</code>
* header on the response to uncompress any compressed response bodies.
* The {@link InputStream} of the entity will contain the uncompressed
* content.</p>
*
* <p><b>N.B.</b> Any upstream clients of this class need to be aware that
* this effectively obscures visibility into the length of a server
* response body, since the <code>Content-Length</code> header will
* correspond to the compressed entity length received from the server,
* but the content length experienced by reading the response body may
* be different (hopefully higher!).</p>
*
* <p>That said, this decorator is compatible with the {@link CachingHttpClient}
* in that the two decorators can be added in either order and still have
* cacheable responses be cached.</p>
*/
public class CompressionDecorator implements HttpClient {
private HttpClient backend;
private HttpRequestInterceptor acceptEncodingInterceptor;
private HttpResponseInterceptor contentEncodingInterceptor;
/*
* Constructs a decorator to ask for and handle compressed
* entities on the fly.
* @param backend the {@link HttpClient} to use for actually
* issuing requests
*/
public CompressionDecorator(HttpClient backend) {
this(backend, new RequestAcceptEncoding(), new ResponseContentEncoding());
}
CompressionDecorator(HttpClient backend, HttpRequestInterceptor requestInterceptor, HttpResponseInterceptor responseInterceptor) {
this.backend = backend;
this.acceptEncodingInterceptor = requestInterceptor;
this.contentEncodingInterceptor = responseInterceptor;
}
public HttpParams getParams() {
return backend.getParams();
}
public ClientConnectionManager getConnectionManager() {
return backend.getConnectionManager();
}
public HttpResponse execute(HttpUriRequest request) throws IOException,
ClientProtocolException {
return execute(getHttpHost(request), request, (HttpContext)null);
}
HttpHost getHttpHost(HttpUriRequest request) {
URI uri = request.getURI();
return new HttpHost(uri.getAuthority());
}
public HttpResponse execute(HttpUriRequest request, HttpContext context)
throws IOException, ClientProtocolException {
return execute(getHttpHost(request), request, context);
}
public HttpResponse execute(HttpHost target, HttpRequest request)
throws IOException, ClientProtocolException {
return execute(target, request, (HttpContext)null);
}
public HttpResponse execute(HttpHost target, HttpRequest request,
HttpContext context) throws IOException, ClientProtocolException {
try {
if (context == null) context = new BasicHttpContext();
HttpRequest wrapped = new RequestWrapper(request);
acceptEncodingInterceptor.process(wrapped, context);
HttpResponse response = backend.execute(target, wrapped, context);
contentEncodingInterceptor.process(response, context);
if (Boolean.TRUE.equals(context.getAttribute(ResponseContentEncoding.UNCOMPRESSED))) {
response.removeHeaders("Content-Length");
response.removeHeaders("Content-Encoding");
}
return response;
} catch (HttpException e) {
throw new RuntimeException(e);
}
}
public <T> T execute(HttpUriRequest request,
ResponseHandler<? extends T> responseHandler) throws IOException,
ClientProtocolException {
return execute(getHttpHost(request), request, responseHandler);
}
public <T> T execute(HttpUriRequest request,
ResponseHandler<? extends T> responseHandler, HttpContext context)
throws IOException, ClientProtocolException {
return execute(getHttpHost(request), request, responseHandler, context);
}
public <T> T execute(HttpHost target, HttpRequest request,
ResponseHandler<? extends T> responseHandler) throws IOException,
ClientProtocolException {
return execute(target, request, responseHandler, null);
}
public <T> T execute(HttpHost target, HttpRequest request,
ResponseHandler<? extends T> responseHandler, HttpContext context)
throws IOException, ClientProtocolException {
HttpResponse response = execute(target, request, context);
try {
return responseHandler.handleResponse(response);
} finally {
HttpEntity entity = response.getEntity();
if (entity != null) EntityUtils.consume(entity);
}
}
}

View File

@ -36,9 +36,18 @@
/**
* {@link DefaultHttpClient} sub-class which includes a {@link RequestAcceptEncoding}
* for the request and response.
*
* <b>Deprecation note:</b> due to the way this class modifies a response body
* without changing the response headers to reflect the entity changes, it cannot
* be used as the &quot;backend&quot; for a {@link CachingHttpClient} and still
* have uncompressed responses be cached. Users are encouraged to use the
* {@link CompressionDecorator} instead of this class, which can be wired in
* either before or after caching, depending on whether you want to cache
* responses in compressed or uncompressed form.
*
* @since 4.1
*/
@Deprecated
@ThreadSafe // since DefaultHttpClient is
public class ContentEncodingHttpClient extends DefaultHttpClient {

View File

@ -0,0 +1,105 @@
package org.apache.http.impl.client;
import java.io.IOException;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.ProtocolVersion;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.impl.conn.SingleClientConnManager;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.HttpContext;
@SuppressWarnings("deprecation")
public class DummyHttpClient implements HttpClient {
private HttpParams params = new BasicHttpParams();
private ClientConnectionManager connManager = new SingleClientConnManager();
private HttpRequest request;
private HttpResponse response = new BasicHttpResponse(new ProtocolVersion("HTTP",1,1), HttpStatus.SC_OK, "OK");
public void setParams(HttpParams params) {
this.params = params;
}
public HttpParams getParams() {
return params;
}
public ClientConnectionManager getConnectionManager() {
return connManager;
}
public void setConnectionManager(ClientConnectionManager ccm) {
connManager = ccm;
}
public void setResponse(HttpResponse resp) {
response = resp;
}
public HttpRequest getCapturedRequest() {
return request;
}
public HttpResponse execute(HttpUriRequest request) throws IOException,
ClientProtocolException {
this.request = request;
return response;
}
public HttpResponse execute(HttpUriRequest request, HttpContext context)
throws IOException, ClientProtocolException {
this.request = request;
return response;
}
public HttpResponse execute(HttpHost target, HttpRequest request)
throws IOException, ClientProtocolException {
this.request = request;
return response;
}
public HttpResponse execute(HttpHost target, HttpRequest request,
HttpContext context) throws IOException, ClientProtocolException {
this.request = request;
return response;
}
public <T> T execute(HttpUriRequest request,
ResponseHandler<? extends T> responseHandler) throws IOException,
ClientProtocolException {
this.request = request;
return responseHandler.handleResponse(response);
}
public <T> T execute(HttpUriRequest request,
ResponseHandler<? extends T> responseHandler, HttpContext context)
throws IOException, ClientProtocolException {
this.request = request;
return responseHandler.handleResponse(response);
}
public <T> T execute(HttpHost target, HttpRequest request,
ResponseHandler<? extends T> responseHandler) throws IOException,
ClientProtocolException {
this.request = request;
return responseHandler.handleResponse(response);
}
public <T> T execute(HttpHost target, HttpRequest request,
ResponseHandler<? extends T> responseHandler, HttpContext context)
throws IOException, ClientProtocolException {
this.request = request;
return responseHandler.handleResponse(response);
}
}

View File

@ -0,0 +1,330 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package org.apache.http.impl.client;
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.GZIPOutputStream;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.HttpVersion;
import org.apache.http.client.HttpClient;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.message.BasicHttpResponse;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class TestCompressionDecorator {
private DummyHttpClient backend;
@Mock private ClientConnectionManager mockConnManager;
@Mock private ResponseHandler<Object> mockHandler;
private CompressionDecorator impl;
private HttpUriRequest request;
private HttpContext ctx;
private HttpHost host;
@Mock private HttpResponse mockResponse;
@Mock private HttpEntity mockEntity;
private Object handled;
@Before
public void canCreate() {
handled = new Object();
backend = new DummyHttpClient();
impl = new CompressionDecorator(backend);
request = new HttpGet("http://localhost:8080");
ctx = new BasicHttpContext();
host = new HttpHost("www.example.com");
}
@Test
public void isAnHttpClient() {
assertTrue(impl instanceof HttpClient);
}
@Test
public void usesParamsFromBackend() {
HttpParams params = new BasicHttpParams();
backend.setParams(params);
assertSame(params, impl.getParams());
}
@Test
public void extractsHostNameFromUriRequest() {
assertEquals(new HttpHost("www.example.com"),
impl.getHttpHost(new HttpGet("http://www.example.com/")));
}
@Test
public void extractsHostNameAndPortFromUriRequest() {
assertEquals(new HttpHost("www.example.com:8080"),
impl.getHttpHost(new HttpGet("http://www.example.com:8080/")));
}
@Test
public void extractsIPAddressFromUriRequest() {
assertEquals(new HttpHost("10.0.0.1"),
impl.getHttpHost(new HttpGet("http://10.0.0.1/")));
}
@Test
public void extractsIPAddressAndPortFromUriRequest() {
assertEquals(new HttpHost("10.0.0.1:8080"),
impl.getHttpHost(new HttpGet("http://10.0.0.1:8080/")));
}
@Test
public void extractsLocalhostFromUriRequest() {
assertEquals(new HttpHost("localhost"),
impl.getHttpHost(new HttpGet("http://localhost/")));
}
@Test
public void extractsLocalhostAndPortFromUriRequest() {
assertEquals(new HttpHost("localhost:8080"),
impl.getHttpHost(new HttpGet("http://localhost:8080/")));
}
@Test
public void usesConnectionManagerFromBackend() {
backend.setConnectionManager(mockConnManager);
assertSame(mockConnManager, impl.getConnectionManager());
}
private void assertAcceptEncodingGzipAndDeflateWereAddedToRequest(HttpRequest captured) {
boolean foundGzip = false;
boolean foundDeflate = false;
for(Header h : captured.getHeaders("Accept-Encoding")) {
for(HeaderElement elt : h.getElements()) {
if ("gzip".equals(elt.getName())) foundGzip = true;
if ("deflate".equals(elt.getName())) foundDeflate = true;
}
}
assertTrue(foundGzip);
assertTrue(foundDeflate);
}
@Test
public void addsAcceptEncodingHeaderToHttpUriRequest() throws Exception {
impl.execute(request);
assertAcceptEncodingGzipAndDeflateWereAddedToRequest(backend.getCapturedRequest());
}
@Test
public void addsAcceptEncodingHeaderToHttpUriRequestWithContext() throws Exception {
impl.execute(request, ctx);
assertAcceptEncodingGzipAndDeflateWereAddedToRequest(backend.getCapturedRequest());
}
@Test
public void addsAcceptEncodingHeaderToHostAndHttpRequest() throws Exception {
impl.execute(host, request);
assertAcceptEncodingGzipAndDeflateWereAddedToRequest(backend.getCapturedRequest());
}
@Test
public void addsAcceptEncodingHeaderToHostAndHttpRequestWithContext() throws Exception {
impl.execute(host, request, ctx);
assertAcceptEncodingGzipAndDeflateWereAddedToRequest(backend.getCapturedRequest());
}
@Test
public void addsAcceptEncodingHeaderToUriRequestWithHandler() throws Exception {
when(mockHandler.handleResponse(isA(HttpResponse.class))).thenReturn(new Object());
impl.execute(request, mockHandler);
assertAcceptEncodingGzipAndDeflateWereAddedToRequest(backend.getCapturedRequest());
}
@Test
public void addsAcceptEncodingHeaderToUriRequestWithHandlerAndContext() throws Exception {
when(mockHandler.handleResponse(isA(HttpResponse.class))).thenReturn(new Object());
impl.execute(request, mockHandler, ctx);
assertAcceptEncodingGzipAndDeflateWereAddedToRequest(backend.getCapturedRequest());
}
@Test
public void addsAcceptEncodingHeaderToRequestWithHostAndHandler() throws Exception {
when(mockHandler.handleResponse(isA(HttpResponse.class))).thenReturn(new Object());
impl.execute(host, request, mockHandler);
assertAcceptEncodingGzipAndDeflateWereAddedToRequest(backend.getCapturedRequest());
}
@Test
public void addsAcceptEncodingHeaderToRequestWithHostAndContextAndHandler() throws Exception {
when(mockHandler.handleResponse(isA(HttpResponse.class))).thenReturn(new Object());
impl.execute(host, request, mockHandler, ctx);
assertAcceptEncodingGzipAndDeflateWereAddedToRequest(backend.getCapturedRequest());
}
private void mockResponseHasNoContentEncodingHeaders() {
backend.setResponse(mockResponse);
when(mockResponse.getAllHeaders()).thenReturn(new Header[]{});
when(mockResponse.getHeaders("Content-Encoding")).thenReturn(new Header[]{});
when(mockResponse.getFirstHeader("Content-Encoding")).thenReturn(null);
when(mockResponse.getLastHeader("Content-Encoding")).thenReturn(null);
when(mockResponse.getEntity()).thenReturn(mockEntity);
}
@Test
public void doesNotModifyResponseBodyIfNoContentEncoding() throws Exception {
mockResponseHasNoContentEncodingHeaders();
assertSame(mockResponse, impl.execute(request));
verify(mockResponse, never()).setEntity(any(HttpEntity.class));
}
@Test
public void doesNotModifyResponseBodyIfNoContentEncodingWithContext() throws Exception {
mockResponseHasNoContentEncodingHeaders();
assertSame(mockResponse, impl.execute(request, ctx));
verify(mockResponse, never()).setEntity(any(HttpEntity.class));
}
@Test
public void doesNotModifyResponseBodyIfNoContentEncodingForHostRequest() throws Exception {
mockResponseHasNoContentEncodingHeaders();
assertSame(mockResponse, impl.execute(host, request));
verify(mockResponse, never()).setEntity(any(HttpEntity.class));
}
@Test
public void doesNotModifyResponseBodyIfNoContentEncodingForHostRequestWithContext() throws Exception {
mockResponseHasNoContentEncodingHeaders();
assertSame(mockResponse, impl.execute(host, request, ctx));
verify(mockResponse, never()).setEntity(any(HttpEntity.class));
}
@Test
public void doesNotModifyResponseBodyWithHandlerIfNoContentEncoding() throws Exception {
mockResponseHasNoContentEncodingHeaders();
when(mockHandler.handleResponse(mockResponse)).thenReturn(handled);
assertSame(handled, impl.execute(request, mockHandler));
verify(mockResponse, never()).setEntity(any(HttpEntity.class));
}
@Test
public void doesNotModifyResponseBodyWithHandlerAndContextIfNoContentEncoding() throws Exception {
mockResponseHasNoContentEncodingHeaders();
when(mockHandler.handleResponse(mockResponse)).thenReturn(handled);
assertSame(handled, impl.execute(request, mockHandler, ctx));
verify(mockResponse, never()).setEntity(any(HttpEntity.class));
}
@Test
public void doesNotModifyResponseBodyWithHostAndHandlerIfNoContentEncoding() throws Exception {
mockResponseHasNoContentEncodingHeaders();
when(mockHandler.handleResponse(mockResponse)).thenReturn(handled);
assertSame(handled, impl.execute(host, request, mockHandler));
verify(mockResponse, never()).setEntity(any(HttpEntity.class));
}
@Test
public void doesNotModifyResponseBodyWithHostAndHandlerAndContextIfNoContentEncoding() throws Exception {
mockResponseHasNoContentEncodingHeaders();
when(mockHandler.handleResponse(mockResponse)).thenReturn(handled);
assertSame(handled, impl.execute(host, request, mockHandler, ctx));
verify(mockResponse, never()).setEntity(any(HttpEntity.class));
}
@Test
public void successfullyUncompressesContent() throws Exception {
final String plainText = "hello\n";
HttpResponse response = getGzippedResponse(plainText);
backend.setResponse(response);
HttpResponse result = impl.execute(request);
ByteArrayOutputStream resultBuf = new ByteArrayOutputStream();
InputStream is = result.getEntity().getContent();
int b;
while((b = is.read()) != -1) {
resultBuf.write(b);
}
is.close();
assertEquals(plainText, new String(resultBuf.toByteArray()));
}
@Test
public void uncompressedResponseHasUnknownLength() throws Exception {
final String plainText = "hello\n";
HttpResponse response = getGzippedResponse(plainText);
backend.setResponse(response);
HttpResponse result = impl.execute(request);
HttpEntity entity = result.getEntity();
assertEquals(-1, entity.getContentLength());
EntityUtils.consume(entity);
assertNull(result.getFirstHeader("Content-Length"));
}
@Test
public void uncompressedResponseIsNotEncoded() throws Exception {
final String plainText = "hello\n";
HttpResponse response = getGzippedResponse(plainText);
backend.setResponse(response);
HttpResponse result = impl.execute(request);
assertNull(result.getFirstHeader("Content-Encoding"));
}
private HttpResponse getGzippedResponse(final String plainText)
throws IOException {
HttpResponse response = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK");
response.setHeader("Content-Encoding","gzip");
response.setHeader("Content-Type","text/plain");
ByteArrayOutputStream buf = new ByteArrayOutputStream();
GZIPOutputStream gos = new GZIPOutputStream(buf);
gos.write(plainText.getBytes());
gos.close();
ByteArrayEntity body = new ByteArrayEntity(buf.toByteArray());
body.setContentEncoding("gzip");
body.setContentType("text/plain");
response.setHeader("Content-Length", "" + (int)body.getContentLength());
response.setEntity(body);
return response;
}
}