From 5d11a3e751fe0c02a7a4539d3436b06e0be35876 Mon Sep 17 00:00:00 2001 From: Oleg Kalnichevski Date: Tue, 25 Feb 2014 09:49:47 +0000 Subject: [PATCH] HTTPCLIENT-1403: Pluggable content decoders git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk@1571635 13f79535-47bb-0310-9956-ffa450edef68 --- RELEASE_NOTES.txt | 6 +- .../client/entity/DecompressingEntity.java | 37 +++++--- .../entity/DeflateDecompressingEntity.java | 38 ++------ .../entity/GzipDecompressingEntity.java | 31 ++----- .../client/entity/InputStreamFactory.java | 41 +++++++++ .../entity/LazyDecompressingInputStream.java | 13 ++- .../protocol/RequestAcceptEncoding.java | 26 +++++- .../protocol/ResponseContentEncoding.java | 92 ++++++++++++------- .../http/impl/client/HttpClientBuilder.java | 33 ++++++- .../entity/TestDecompressingEntity.java | 15 ++- .../protocol/TestResponseContentEncoding.java | 8 +- 11 files changed, 211 insertions(+), 129 deletions(-) create mode 100644 httpclient/src/main/java/org/apache/http/client/entity/InputStreamFactory.java diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 607fc3e86..edc80a61e 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -4,7 +4,11 @@ Changes for 4.4-alpha1 Changelog: ------------------- -* [HTTPCLIENT-1470] CachingExec(ClientExecChain, HttpCache, CacheConfig, AsynchronousValidator) throws NPE if config is null +* [HTTPCLIENT-1403] Pluggable content decoders. + Contributed by Oleg Kalnichevski + +* [HTTPCLIENT-1470] CachingExec(ClientExecChain, HttpCache, CacheConfig, AsynchronousValidator) + throws NPE if config is null. * [HTTPCLIENT-1466] FileBodyPart#generateContentType() ignores custom ContentType values. Contributed by Oleg Kalnichevski diff --git a/httpclient/src/main/java/org/apache/http/client/entity/DecompressingEntity.java b/httpclient/src/main/java/org/apache/http/client/entity/DecompressingEntity.java index f0e16db91..08d7c6d27 100644 --- a/httpclient/src/main/java/org/apache/http/client/entity/DecompressingEntity.java +++ b/httpclient/src/main/java/org/apache/http/client/entity/DecompressingEntity.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.entity.HttpEntityWrapper; import org.apache.http.util.Args; @@ -37,15 +38,16 @@ import org.apache.http.util.Args; /** * Common base class for decompressing {@link HttpEntity} implementations. * - * @since 4.1 + * @since 4.4 */ -abstract class DecompressingEntity extends HttpEntityWrapper { +public class DecompressingEntity extends HttpEntityWrapper { /** * Default buffer size. */ private static final int BUFFER_SIZE = 1024 * 2; + private final InputStreamFactory inputStreamFactory; /** * {@link #getContent()} method must return the same {@link InputStream} * instance when DecompressingEntity is wrapping a streaming entity. @@ -55,23 +57,21 @@ abstract class DecompressingEntity extends HttpEntityWrapper { /** * Creates a new {@link DecompressingEntity}. * - * @param wrapped - * the non-null {@link HttpEntity} to be wrapped + * @param wrapped the non-null {@link HttpEntity} to be wrapped + * @param inputStreamFactory factory to create decompressing stream. */ - public DecompressingEntity(final HttpEntity wrapped) { + public DecompressingEntity( + final HttpEntity wrapped, + final InputStreamFactory inputStreamFactory) { super(wrapped); + this.inputStreamFactory = inputStreamFactory; } - abstract InputStream decorate(final InputStream wrapped) throws IOException; - private InputStream getDecompressingStream() throws IOException { final InputStream in = wrappedEntity.getContent(); - return new LazyDecompressingInputStream(in, this); + return new LazyDecompressingInputStream(in, inputStreamFactory); } - /** - * {@inheritDoc} - */ @Override public InputStream getContent() throws IOException { if (wrappedEntity.isStreaming()) { @@ -84,9 +84,6 @@ abstract class DecompressingEntity extends HttpEntityWrapper { } } - /** - * {@inheritDoc} - */ @Override public void writeTo(final OutputStream outstream) throws IOException { Args.notNull(outstream, "Output stream"); @@ -102,4 +99,16 @@ abstract class DecompressingEntity extends HttpEntityWrapper { } } + @Override + public Header getContentEncoding() { + /* Content encoding is now 'identity' */ + return null; + } + + @Override + public long getContentLength() { + /* length of decompressed content is not known */ + return -1; + } + } diff --git a/httpclient/src/main/java/org/apache/http/client/entity/DeflateDecompressingEntity.java b/httpclient/src/main/java/org/apache/http/client/entity/DeflateDecompressingEntity.java index 4089d002c..e9fddcb52 100644 --- a/httpclient/src/main/java/org/apache/http/client/entity/DeflateDecompressingEntity.java +++ b/httpclient/src/main/java/org/apache/http/client/entity/DeflateDecompressingEntity.java @@ -29,7 +29,6 @@ package org.apache.http.client.entity; import java.io.IOException; import java.io.InputStream; -import org.apache.http.Header; import org.apache.http.HttpEntity; /** @@ -58,39 +57,14 @@ public class DeflateDecompressingEntity extends DecompressingEntity { * a non-null {@link HttpEntity} to be wrapped */ public DeflateDecompressingEntity(final HttpEntity entity) { - super(entity); - } + super(entity, new InputStreamFactory() { - /** - * Returns the non-null InputStream that should be returned to by all requests to - * {@link #getContent()}. - * - * @return a non-null InputStream - * @throws IOException if there was a problem - */ - @Override - InputStream decorate(final InputStream wrapped) throws IOException { - return new DeflateInputStream(wrapped); - } + @Override + public InputStream create(final InputStream instream) throws IOException { + return new DeflateInputStream(instream); + } - /** - * {@inheritDoc} - */ - @Override - public Header getContentEncoding() { - - /* This HttpEntityWrapper has dealt with the Content-Encoding. */ - return null; - } - - /** - * {@inheritDoc} - */ - @Override - public long getContentLength() { - - /* Length of inflated content is unknown. */ - return -1; + }); } } diff --git a/httpclient/src/main/java/org/apache/http/client/entity/GzipDecompressingEntity.java b/httpclient/src/main/java/org/apache/http/client/entity/GzipDecompressingEntity.java index a0198efd6..ca45b59bb 100644 --- a/httpclient/src/main/java/org/apache/http/client/entity/GzipDecompressingEntity.java +++ b/httpclient/src/main/java/org/apache/http/client/entity/GzipDecompressingEntity.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.zip.GZIPInputStream; -import org.apache.http.Header; import org.apache.http.HttpEntity; /** @@ -49,32 +48,14 @@ public class GzipDecompressingEntity extends DecompressingEntity { * the non-null {@link HttpEntity} to be wrapped */ public GzipDecompressingEntity(final HttpEntity entity) { - super(entity); - } + super(entity, new InputStreamFactory() { - @Override - InputStream decorate(final InputStream wrapped) throws IOException { - return new GZIPInputStream(wrapped); - } + @Override + public InputStream create(final InputStream instream) throws IOException { + return new GZIPInputStream(instream); + } - /** - * {@inheritDoc} - */ - @Override - public Header getContentEncoding() { - - /* This HttpEntityWrapper has dealt with the Content-Encoding. */ - return null; - } - - /** - * {@inheritDoc} - */ - @Override - public long getContentLength() { - - /* length of ungzipped content is not known */ - return -1; + }); } } diff --git a/httpclient/src/main/java/org/apache/http/client/entity/InputStreamFactory.java b/httpclient/src/main/java/org/apache/http/client/entity/InputStreamFactory.java new file mode 100644 index 000000000..4753cf602 --- /dev/null +++ b/httpclient/src/main/java/org/apache/http/client/entity/InputStreamFactory.java @@ -0,0 +1,41 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.http.client.entity; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Factory for decorated {@link java.io.InputStream}s. + * + * @since 4.4 + */ +public interface InputStreamFactory { + + InputStream create(InputStream instream) throws IOException; + +} diff --git a/httpclient/src/main/java/org/apache/http/client/entity/LazyDecompressingInputStream.java b/httpclient/src/main/java/org/apache/http/client/entity/LazyDecompressingInputStream.java index fb0e43b17..db95ce8b4 100644 --- a/httpclient/src/main/java/org/apache/http/client/entity/LazyDecompressingInputStream.java +++ b/httpclient/src/main/java/org/apache/http/client/entity/LazyDecompressingInputStream.java @@ -26,11 +26,11 @@ */ package org.apache.http.client.entity; -import org.apache.http.annotation.NotThreadSafe; - import java.io.IOException; import java.io.InputStream; +import org.apache.http.annotation.NotThreadSafe; + /** * Lazy init InputStream wrapper. */ @@ -38,21 +38,20 @@ import java.io.InputStream; class LazyDecompressingInputStream extends InputStream { private final InputStream wrappedStream; - - private final DecompressingEntity decompressingEntity; + private final InputStreamFactory inputStreamFactory; private InputStream wrapperStream; public LazyDecompressingInputStream( final InputStream wrappedStream, - final DecompressingEntity decompressingEntity) { + final InputStreamFactory inputStreamFactory) { this.wrappedStream = wrappedStream; - this.decompressingEntity = decompressingEntity; + this.inputStreamFactory = inputStreamFactory; } private void initWrapper() throws IOException { if (wrapperStream == null) { - wrapperStream = decompressingEntity.decorate(wrappedStream); + wrapperStream = inputStreamFactory.create(wrappedStream); } } diff --git a/httpclient/src/main/java/org/apache/http/client/protocol/RequestAcceptEncoding.java b/httpclient/src/main/java/org/apache/http/client/protocol/RequestAcceptEncoding.java index ed9920821..9209ac8b7 100644 --- a/httpclient/src/main/java/org/apache/http/client/protocol/RequestAcceptEncoding.java +++ b/httpclient/src/main/java/org/apache/http/client/protocol/RequestAcceptEncoding.java @@ -27,6 +27,7 @@ package org.apache.http.client.protocol; import java.io.IOException; +import java.util.List; import org.apache.http.HttpException; import org.apache.http.HttpRequest; @@ -46,9 +47,30 @@ import org.apache.http.protocol.HttpContext; @Immutable public class RequestAcceptEncoding implements HttpRequestInterceptor { + private final String acceptEncoding; + /** - * Adds the header {@code "Accept-Encoding: gzip,deflate"} to the request. + * @since 4.4 */ + public RequestAcceptEncoding(final List encodings) { + if (encodings != null && !encodings.isEmpty()) { + final StringBuilder buf = new StringBuilder(); + for (int i = 0; i < encodings.size(); i++) { + if (i > 0) { + buf.append(","); + } + buf.append(encodings.get(i)); + } + this.acceptEncoding = buf.toString(); + } else { + this.acceptEncoding = "gzip,deflate"; + } + } + + public RequestAcceptEncoding() { + this(null); + } + @Override public void process( final HttpRequest request, @@ -56,7 +78,7 @@ public class RequestAcceptEncoding implements HttpRequestInterceptor { /* Signal support for Accept-Encoding transfer encodings. */ if (!request.containsHeader("Accept-Encoding")) { - request.addHeader("Accept-Encoding", "gzip,deflate"); + request.addHeader("Accept-Encoding", acceptEncoding); } } diff --git a/httpclient/src/main/java/org/apache/http/client/protocol/ResponseContentEncoding.java b/httpclient/src/main/java/org/apache/http/client/protocol/ResponseContentEncoding.java index 9cce4b50a..91a4441de 100644 --- a/httpclient/src/main/java/org/apache/http/client/protocol/ResponseContentEncoding.java +++ b/httpclient/src/main/java/org/apache/http/client/protocol/ResponseContentEncoding.java @@ -27,7 +27,9 @@ package org.apache.http.client.protocol; import java.io.IOException; +import java.io.InputStream; import java.util.Locale; +import java.util.zip.GZIPInputStream; import org.apache.http.Header; import org.apache.http.HeaderElement; @@ -37,8 +39,11 @@ import org.apache.http.HttpResponse; import org.apache.http.HttpResponseInterceptor; import org.apache.http.annotation.Immutable; import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.DeflateDecompressingEntity; -import org.apache.http.client.entity.GzipDecompressingEntity; +import org.apache.http.client.entity.DecompressingEntity; +import org.apache.http.client.entity.DeflateInputStream; +import org.apache.http.client.entity.InputStreamFactory; +import org.apache.http.config.Lookup; +import org.apache.http.config.RegistryBuilder; import org.apache.http.protocol.HttpContext; /** @@ -55,20 +60,49 @@ public class ResponseContentEncoding implements HttpResponseInterceptor { public static final String UNCOMPRESSED = "http.client.response.uncompressed"; + private final static InputStreamFactory GZIP = new InputStreamFactory() { + + @Override + public InputStream create(final InputStream instream) throws IOException { + return new GZIPInputStream(instream); + } + }; + + private final static InputStreamFactory DEFLATE = new InputStreamFactory() { + + @Override + public InputStream create(final InputStream instream) throws IOException { + return new DeflateInputStream(instream); + } + + }; + + private final Lookup decoderRegistry; + /** - * Handles the following {@code Content-Encoding}s by - * using the appropriate decompressor to wrap the response Entity: - *
    - *
  • gzip - see {@link GzipDecompressingEntity}
    • - *
  • deflate - see {@link DeflateDecompressingEntity}
    • - *
  • identity - no action needed
    • - *
- * - * @param response the response which contains the entity - * @param context not currently used - * - * @throws HttpException if the {@code Content-Encoding} is none of the above + * @since 4.4 */ + public ResponseContentEncoding(final Lookup decoderRegistry) { + this.decoderRegistry = decoderRegistry != null ? decoderRegistry : + RegistryBuilder.create() + .register("gzip", GZIP) + .register("x-gzip", GZIP) + .register("deflate", DEFLATE) + .build(); + } + + /** + * Handles gzip and deflate compressed entities by using the following + * decoders: + *
    + *
  • gzip - see {@link GZIPInputStream}
    • + *
  • deflate - see {@link DeflateInputStream}
    • + *
+ */ + public ResponseContentEncoding() { + this(null); + } + @Override public void process( final HttpResponse response, @@ -83,30 +117,20 @@ public class ResponseContentEncoding implements HttpResponseInterceptor { final Header ceheader = entity.getContentEncoding(); if (ceheader != null) { final HeaderElement[] codecs = ceheader.getElements(); - boolean uncompressed = false; for (final HeaderElement codec : codecs) { - final String codecname = codec.getName().toLowerCase(Locale.US); - if ("gzip".equals(codecname) || "x-gzip".equals(codecname)) { - response.setEntity(new GzipDecompressingEntity(response.getEntity())); - uncompressed = true; - break; - } else if ("deflate".equals(codecname)) { - response.setEntity(new DeflateDecompressingEntity(response.getEntity())); - uncompressed = true; - break; - } else if ("identity".equals(codecname)) { - - /* Don't need to transform the content - no-op */ - return; + final String codecname = codec.getName().toLowerCase(Locale.ROOT); + final InputStreamFactory decoderFactory = decoderRegistry.lookup(codecname); + if (decoderFactory != null) { + response.setEntity(new DecompressingEntity(response.getEntity(), decoderFactory)); + response.removeHeaders("Content-Length"); + response.removeHeaders("Content-Encoding"); + response.removeHeaders("Content-MD5"); } else { - throw new HttpException("Unsupported Content-Coding: " + codec.getName()); + if (!"identity".equals(codecname)) { + throw new HttpException("Unsupported Content-Coding: " + codec.getName()); + } } } - if (uncompressed) { - response.removeHeaders("Content-Length"); - response.removeHeaders("Content-Encoding"); - response.removeHeaders("Content-MD5"); - } } } } diff --git a/httpclient/src/main/java/org/apache/http/impl/client/HttpClientBuilder.java b/httpclient/src/main/java/org/apache/http/impl/client/HttpClientBuilder.java index 1493f6942..8c480ae21 100644 --- a/httpclient/src/main/java/org/apache/http/impl/client/HttpClientBuilder.java +++ b/httpclient/src/main/java/org/apache/http/impl/client/HttpClientBuilder.java @@ -31,8 +31,10 @@ import java.io.Closeable; import java.net.ProxySelector; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Map; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSocketFactory; @@ -56,6 +58,7 @@ import org.apache.http.client.UserTokenHandler; import org.apache.http.client.config.AuthSchemes; import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.InputStreamFactory; import org.apache.http.client.protocol.RequestAcceptEncoding; import org.apache.http.client.protocol.RequestAddCookies; import org.apache.http.client.protocol.RequestAuthCache; @@ -177,6 +180,7 @@ public class HttpClientBuilder { private ServiceUnavailableRetryStrategy serviceUnavailStrategy; private Lookup authSchemeRegistry; private Lookup cookieSpecRegistry; + private Map contentDecoderMap; private CookieStore cookieStore; private CredentialsProvider credentialsProvider; private String userAgent; @@ -636,6 +640,17 @@ public class HttpClientBuilder { return this; } + + /** + * Assigns a map of {@link org.apache.http.client.entity.InputStreamFactory}s + * to be used for automatic content decompression. + */ + public final HttpClientBuilder setContentDecoderRegistry( + final Map contentDecoderMap) { + this.contentDecoderMap = contentDecoderMap; + return this; + } + /** * Assigns default {@link RequestConfig} instance which will be used * for request execution if not explicitly set in the client execution @@ -831,7 +846,13 @@ public class HttpClientBuilder { b.add(new RequestAddCookies()); } if (!contentCompressionDisabled) { - b.add(new RequestAcceptEncoding()); + if (contentDecoderMap != null) { + final List encodings = new ArrayList(contentDecoderMap.keySet()); + Collections.sort(encodings); + b.add(new RequestAcceptEncoding(encodings)); + } else { + b.add(new RequestAcceptEncoding()); + } } if (!authCachingDisabled) { b.add(new RequestAuthCache()); @@ -840,7 +861,15 @@ public class HttpClientBuilder { b.add(new ResponseProcessCookies()); } if (!contentCompressionDisabled) { - b.add(new ResponseContentEncoding()); + if (contentDecoderMap != null) { + final RegistryBuilder b2 = RegistryBuilder.create(); + for (Map.Entry entry: contentDecoderMap.entrySet()) { + b2.register(entry.getKey(), entry.getValue()); + } + b.add(new ResponseContentEncoding(b2.build())); + } else { + b.add(new ResponseContentEncoding()); + } } if (requestLast != null) { for (final HttpRequestInterceptor i: requestLast) { diff --git a/httpclient/src/test/java/org/apache/http/client/entity/TestDecompressingEntity.java b/httpclient/src/test/java/org/apache/http/client/entity/TestDecompressingEntity.java index 954bbb0ce..347d99d74 100644 --- a/httpclient/src/test/java/org/apache/http/client/entity/TestDecompressingEntity.java +++ b/httpclient/src/test/java/org/apache/http/client/entity/TestDecompressingEntity.java @@ -93,16 +93,15 @@ public class TestDecompressingEntity { static class ChecksumEntity extends DecompressingEntity { - private final Checksum checksum; - public ChecksumEntity(final HttpEntity wrapped, final Checksum checksum) { - super(wrapped); - this.checksum = checksum; - } + super(wrapped, new InputStreamFactory() { - @Override - InputStream decorate(final InputStream wrapped) throws IOException { - return new CheckedInputStream(wrapped, this.checksum); + @Override + public InputStream create(final InputStream instream) throws IOException { + return new CheckedInputStream(instream, checksum); + } + + }); } } diff --git a/httpclient/src/test/java/org/apache/http/client/protocol/TestResponseContentEncoding.java b/httpclient/src/test/java/org/apache/http/client/protocol/TestResponseContentEncoding.java index 494d664c8..07735df8c 100644 --- a/httpclient/src/test/java/org/apache/http/client/protocol/TestResponseContentEncoding.java +++ b/httpclient/src/test/java/org/apache/http/client/protocol/TestResponseContentEncoding.java @@ -32,7 +32,7 @@ import org.apache.http.HttpResponse; import org.apache.http.HttpResponseInterceptor; import org.apache.http.HttpVersion; import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.DeflateDecompressingEntity; +import org.apache.http.client.entity.DecompressingEntity; import org.apache.http.client.entity.GzipDecompressingEntity; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHttpResponse; @@ -80,7 +80,7 @@ public class TestResponseContentEncoding { interceptor.process(response, context); final HttpEntity entity = response.getEntity(); Assert.assertNotNull(entity); - Assert.assertTrue(entity instanceof GzipDecompressingEntity); + Assert.assertTrue(entity instanceof DecompressingEntity); } @Test @@ -110,7 +110,7 @@ public class TestResponseContentEncoding { interceptor.process(response, context); final HttpEntity entity = response.getEntity(); Assert.assertNotNull(entity); - Assert.assertTrue(entity instanceof GzipDecompressingEntity); + Assert.assertTrue(entity instanceof DecompressingEntity); } @Test @@ -125,7 +125,7 @@ public class TestResponseContentEncoding { interceptor.process(response, context); final HttpEntity entity = response.getEntity(); Assert.assertNotNull(entity); - Assert.assertTrue(entity instanceof DeflateDecompressingEntity); + Assert.assertTrue(entity instanceof DecompressingEntity); } @Test