Apache
/
httpcomponents-client
mirror of https://github.com/apache/httpcomponents-client.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

HTTPCLIENT-1403: Pluggable content decoders 

git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk@1571635 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Oleg Kalnichevski 2014-02-25 09:49:47 +00:00
parent 5c89f7b4e4
commit 5d11a3e751
11 changed files with 211 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
RELEASE_NOTES.txt
View File

@ -4,7 +4,11 @@ Changes for 4.4-alpha1
Changelog:
-------------------
* [HTTPCLIENT-1470] CachingExec(ClientExecChain, HttpCache, CacheConfig, AsynchronousValidator) throws NPE if config is null
* [HTTPCLIENT-1403] Pluggable content decoders.
Contributed by Oleg Kalnichevski <olegk at apache.org>
* [HTTPCLIENT-1470] CachingExec(ClientExecChain, HttpCache, CacheConfig, AsynchronousValidator)
throws NPE if config is null.
* [HTTPCLIENT-1466] FileBodyPart#generateContentType() ignores custom ContentType values.
Contributed by Oleg Kalnichevski <olegk at apache.org>

37
httpclient/src/main/java/org/apache/http/client/entity/DecompressingEntity.java
View File

@ -30,6 +30,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.entity.HttpEntityWrapper;
import org.apache.http.util.Args;
@ -37,15 +38,16 @@ import org.apache.http.util.Args;
/**
* Common base class for decompressing {@link HttpEntity} implementations.
*
* @since 4.1
* @since 4.4
*/
abstract class DecompressingEntity extends HttpEntityWrapper {
public class DecompressingEntity extends HttpEntityWrapper {
/**
* Default buffer size.
*/
private static final int BUFFER_SIZE = 1024 * 2;
private final InputStreamFactory inputStreamFactory;
/**
* {@link #getContent()} method must return the same {@link InputStream}
* instance when DecompressingEntity is wrapping a streaming entity.
@ -55,23 +57,21 @@ abstract class DecompressingEntity extends HttpEntityWrapper {
/**
* Creates a new {@link DecompressingEntity}.
*
* @param wrapped
* the non-null {@link HttpEntity} to be wrapped
* @param wrapped the non-null {@link HttpEntity} to be wrapped
* @param inputStreamFactory factory to create decompressing stream.
*/
public DecompressingEntity(final HttpEntity wrapped) {
public DecompressingEntity(
final HttpEntity wrapped,
final InputStreamFactory inputStreamFactory) {
super(wrapped);
this.inputStreamFactory = inputStreamFactory;
}
abstract InputStream decorate(final InputStream wrapped) throws IOException;
private InputStream getDecompressingStream() throws IOException {
final InputStream in = wrappedEntity.getContent();
return new LazyDecompressingInputStream(in, this);
return new LazyDecompressingInputStream(in, inputStreamFactory);
}
/**
* {@inheritDoc}
*/
@Override
public InputStream getContent() throws IOException {
if (wrappedEntity.isStreaming()) {
@ -84,9 +84,6 @@ abstract class DecompressingEntity extends HttpEntityWrapper {
}
}
/**
* {@inheritDoc}
*/
@Override
public void writeTo(final OutputStream outstream) throws IOException {
Args.notNull(outstream, "Output stream");
@ -102,4 +99,16 @@ abstract class DecompressingEntity extends HttpEntityWrapper {
}
}
@Override
public Header getContentEncoding() {
/* Content encoding is now 'identity' */
return null;
}
@Override
public long getContentLength() {
/* length of decompressed content is not known */
return -1;
}
}

38
httpclient/src/main/java/org/apache/http/client/entity/DeflateDecompressingEntity.java
View File

@ -29,7 +29,6 @@ package org.apache.http.client.entity;
import java.io.IOException;
import java.io.InputStream;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
/**
@ -58,39 +57,14 @@ public class DeflateDecompressingEntity extends DecompressingEntity {
* a non-null {@link HttpEntity} to be wrapped
*/
public DeflateDecompressingEntity(final HttpEntity entity) {
super(entity);
}
super(entity, new InputStreamFactory() {
/**
* Returns the non-null InputStream that should be returned to by all requests to
* {@link #getContent()}.
*
* @return a non-null InputStream
* @throws IOException if there was a problem
*/
@Override
InputStream decorate(final InputStream wrapped) throws IOException {
return new DeflateInputStream(wrapped);
}
@Override
public InputStream create(final InputStream instream) throws IOException {
return new DeflateInputStream(instream);
}
/**
* {@inheritDoc}
*/
@Override
public Header getContentEncoding() {
/* This HttpEntityWrapper has dealt with the Content-Encoding. */
return null;
}
/**
* {@inheritDoc}
*/
@Override
public long getContentLength() {
/* Length of inflated content is unknown. */
return -1;
});
}
}

31
httpclient/src/main/java/org/apache/http/client/entity/GzipDecompressingEntity.java
View File

@ -30,7 +30,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.zip.GZIPInputStream;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
/**
@ -49,32 +48,14 @@ public class GzipDecompressingEntity extends DecompressingEntity {
* the non-null {@link HttpEntity} to be wrapped
*/
public GzipDecompressingEntity(final HttpEntity entity) {
super(entity);
}
super(entity, new InputStreamFactory() {
@Override
InputStream decorate(final InputStream wrapped) throws IOException {
return new GZIPInputStream(wrapped);
}
@Override
public InputStream create(final InputStream instream) throws IOException {
return new GZIPInputStream(instream);
}
/**
* {@inheritDoc}
*/
@Override
public Header getContentEncoding() {
/* This HttpEntityWrapper has dealt with the Content-Encoding. */
return null;
}
/**
* {@inheritDoc}
*/
@Override
public long getContentLength() {
/* length of ungzipped content is not known */
return -1;
});
}
}

41
httpclient/src/main/java/org/apache/http/client/entity/InputStreamFactory.java Normal file
View File

@ -0,0 +1,41 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.client.entity;
import java.io.IOException;
import java.io.InputStream;
/**
* Factory for decorated {@link java.io.InputStream}s.
*
* @since 4.4
*/
public interface InputStreamFactory {
InputStream create(InputStream instream) throws IOException;
}

13
httpclient/src/main/java/org/apache/http/client/entity/LazyDecompressingInputStream.java
View File

@ -26,11 +26,11 @@
*/
package org.apache.http.client.entity;
import org.apache.http.annotation.NotThreadSafe;
import java.io.IOException;
import java.io.InputStream;
import org.apache.http.annotation.NotThreadSafe;
/**
* Lazy init InputStream wrapper.
*/
@ -38,21 +38,20 @@ import java.io.InputStream;
class LazyDecompressingInputStream extends InputStream {
private final InputStream wrappedStream;
private final DecompressingEntity decompressingEntity;
private final InputStreamFactory inputStreamFactory;
private InputStream wrapperStream;
public LazyDecompressingInputStream(
final InputStream wrappedStream,
final DecompressingEntity decompressingEntity) {
final InputStreamFactory inputStreamFactory) {
this.wrappedStream = wrappedStream;
this.decompressingEntity = decompressingEntity;
this.inputStreamFactory = inputStreamFactory;
}
private void initWrapper() throws IOException {
if (wrapperStream == null) {
wrapperStream = decompressingEntity.decorate(wrappedStream);
wrapperStream = inputStreamFactory.create(wrappedStream);
}
}

26
httpclient/src/main/java/org/apache/http/client/protocol/RequestAcceptEncoding.java
View File

@ -27,6 +27,7 @@
package org.apache.http.client.protocol;
import java.io.IOException;
import java.util.List;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
@ -46,9 +47,30 @@ import org.apache.http.protocol.HttpContext;
@Immutable
public class RequestAcceptEncoding implements HttpRequestInterceptor {
private final String acceptEncoding;
/**
* Adds the header {@code "Accept-Encoding: gzip,deflate"} to the request.
* @since 4.4
*/
public RequestAcceptEncoding(final List<String> encodings) {
if (encodings != null && !encodings.isEmpty()) {
final StringBuilder buf = new StringBuilder();
for (int i = 0; i < encodings.size(); i++) {
if (i > 0) {
buf.append(",");
}
buf.append(encodings.get(i));
}
this.acceptEncoding = buf.toString();
} else {
this.acceptEncoding = "gzip,deflate";
}
}
public RequestAcceptEncoding() {
this(null);
}
@Override
public void process(
final HttpRequest request,
@ -56,7 +78,7 @@ public class RequestAcceptEncoding implements HttpRequestInterceptor {
/* Signal support for Accept-Encoding transfer encodings. */
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip,deflate");
request.addHeader("Accept-Encoding", acceptEncoding);
}
}

92
httpclient/src/main/java/org/apache/http/client/protocol/ResponseContentEncoding.java
View File

@ -27,7 +27,9 @@
package org.apache.http.client.protocol;
import java.io.IOException;
import java.io.InputStream;
import java.util.Locale;
import java.util.zip.GZIPInputStream;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
@ -37,8 +39,11 @@ import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.annotation.Immutable;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.DeflateDecompressingEntity;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.entity.DecompressingEntity;
import org.apache.http.client.entity.DeflateInputStream;
import org.apache.http.client.entity.InputStreamFactory;
import org.apache.http.config.Lookup;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.protocol.HttpContext;
/**
@ -55,20 +60,49 @@ public class ResponseContentEncoding implements HttpResponseInterceptor {
public static final String UNCOMPRESSED = "http.client.response.uncompressed";
private final static InputStreamFactory GZIP = new InputStreamFactory() {
@Override
public InputStream create(final InputStream instream) throws IOException {
return new GZIPInputStream(instream);
}
};
private final static InputStreamFactory DEFLATE = new InputStreamFactory() {
@Override
public InputStream create(final InputStream instream) throws IOException {
return new DeflateInputStream(instream);
}
};
private final Lookup<InputStreamFactory> decoderRegistry;
/**
* Handles the following {@code Content-Encoding}s by
* using the appropriate decompressor to wrap the response Entity:
* <ul>
* <li>gzip - see {@link GzipDecompressingEntity}</li>
* <li>deflate - see {@link DeflateDecompressingEntity}</li>
* <li>identity - no action needed</li>
* </ul>
*
* @param response the response which contains the entity
* @param context not currently used
*
* @throws HttpException if the {@code Content-Encoding} is none of the above
* @since 4.4
*/
public ResponseContentEncoding(final Lookup<InputStreamFactory> decoderRegistry) {
this.decoderRegistry = decoderRegistry != null ? decoderRegistry :
RegistryBuilder.<InputStreamFactory>create()
.register("gzip", GZIP)
.register("x-gzip", GZIP)
.register("deflate", DEFLATE)
.build();
}
/**
* Handles <tt>gzip</tt> and <tt>deflate</tt> compressed entities by using the following
* decoders:
* <ul>
* <li>gzip - see {@link GZIPInputStream}</li>
* <li>deflate - see {@link DeflateInputStream}</li>
* </ul>
*/
public ResponseContentEncoding() {
this(null);
}
@Override
public void process(
final HttpResponse response,
@ -83,30 +117,20 @@ public class ResponseContentEncoding implements HttpResponseInterceptor {
final Header ceheader = entity.getContentEncoding();
if (ceheader != null) {
final HeaderElement[] codecs = ceheader.getElements();
boolean uncompressed = false;
for (final HeaderElement codec : codecs) {
final String codecname = codec.getName().toLowerCase(Locale.US);
if ("gzip".equals(codecname) || "x-gzip".equals(codecname)) {
response.setEntity(new GzipDecompressingEntity(response.getEntity()));
uncompressed = true;
break;
} else if ("deflate".equals(codecname)) {
response.setEntity(new DeflateDecompressingEntity(response.getEntity()));
uncompressed = true;
break;
} else if ("identity".equals(codecname)) {
/* Don't need to transform the content - no-op */
return;
final String codecname = codec.getName().toLowerCase(Locale.ROOT);
final InputStreamFactory decoderFactory = decoderRegistry.lookup(codecname);
if (decoderFactory != null) {
response.setEntity(new DecompressingEntity(response.getEntity(), decoderFactory));
response.removeHeaders("Content-Length");
response.removeHeaders("Content-Encoding");
response.removeHeaders("Content-MD5");
} else {
throw new HttpException("Unsupported Content-Coding: " + codec.getName());
if (!"identity".equals(codecname)) {
throw new HttpException("Unsupported Content-Coding: " + codec.getName());
}
}
}
if (uncompressed) {
response.removeHeaders("Content-Length");
response.removeHeaders("Content-Encoding");
response.removeHeaders("Content-MD5");
}
}
}
}

33
httpclient/src/main/java/org/apache/http/impl/client/HttpClientBuilder.java
View File

@ -31,8 +31,10 @@ import java.io.Closeable;
import java.net.ProxySelector;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
@ -56,6 +58,7 @@ import org.apache.http.client.UserTokenHandler;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.InputStreamFactory;
import org.apache.http.client.protocol.RequestAcceptEncoding;
import org.apache.http.client.protocol.RequestAddCookies;
import org.apache.http.client.protocol.RequestAuthCache;
@ -177,6 +180,7 @@ public class HttpClientBuilder {
private ServiceUnavailableRetryStrategy serviceUnavailStrategy;
private Lookup<AuthSchemeProvider> authSchemeRegistry;
private Lookup<CookieSpecProvider> cookieSpecRegistry;
private Map<String, InputStreamFactory> contentDecoderMap;
private CookieStore cookieStore;
private CredentialsProvider credentialsProvider;
private String userAgent;
@ -636,6 +640,17 @@ public class HttpClientBuilder {
return this;
}
/**
* Assigns a map of {@link org.apache.http.client.entity.InputStreamFactory}s
* to be used for automatic content decompression.
*/
public final HttpClientBuilder setContentDecoderRegistry(
final Map<String, InputStreamFactory> contentDecoderMap) {
this.contentDecoderMap = contentDecoderMap;
return this;
}
/**
* Assigns default {@link RequestConfig} instance which will be used
* for request execution if not explicitly set in the client execution
@ -831,7 +846,13 @@ public class HttpClientBuilder {
b.add(new RequestAddCookies());
}
if (!contentCompressionDisabled) {
b.add(new RequestAcceptEncoding());
if (contentDecoderMap != null) {
final List<String> encodings = new ArrayList<String>(contentDecoderMap.keySet());
Collections.sort(encodings);
b.add(new RequestAcceptEncoding(encodings));
} else {
b.add(new RequestAcceptEncoding());
}
}
if (!authCachingDisabled) {
b.add(new RequestAuthCache());
@ -840,7 +861,15 @@ public class HttpClientBuilder {
b.add(new ResponseProcessCookies());
}
if (!contentCompressionDisabled) {
b.add(new ResponseContentEncoding());
if (contentDecoderMap != null) {
final RegistryBuilder<InputStreamFactory> b2 = RegistryBuilder.create();
for (Map.Entry<String, InputStreamFactory> entry: contentDecoderMap.entrySet()) {
b2.register(entry.getKey(), entry.getValue());
}
b.add(new ResponseContentEncoding(b2.build()));
} else {
b.add(new ResponseContentEncoding());
}
}
if (requestLast != null) {
for (final HttpRequestInterceptor i: requestLast) {

15
httpclient/src/test/java/org/apache/http/client/entity/TestDecompressingEntity.java
View File

@ -93,16 +93,15 @@ public class TestDecompressingEntity {
static class ChecksumEntity extends DecompressingEntity {
private final Checksum checksum;
public ChecksumEntity(final HttpEntity wrapped, final Checksum checksum) {
super(wrapped);
this.checksum = checksum;
}
super(wrapped, new InputStreamFactory() {
@Override
InputStream decorate(final InputStream wrapped) throws IOException {
return new CheckedInputStream(wrapped, this.checksum);
@Override
public InputStream create(final InputStream instream) throws IOException {
return new CheckedInputStream(instream, checksum);
}
});
}
}

8
httpclient/src/test/java/org/apache/http/client/protocol/TestResponseContentEncoding.java
View File

@ -32,7 +32,7 @@ import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.HttpVersion;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.DeflateDecompressingEntity;
import org.apache.http.client.entity.DecompressingEntity;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.entity.StringEntity;
import org.apache.http.message.BasicHttpResponse;
@ -80,7 +80,7 @@ public class TestResponseContentEncoding {
interceptor.process(response, context);
final HttpEntity entity = response.getEntity();
Assert.assertNotNull(entity);
Assert.assertTrue(entity instanceof GzipDecompressingEntity);
Assert.assertTrue(entity instanceof DecompressingEntity);
}
@Test
@ -110,7 +110,7 @@ public class TestResponseContentEncoding {
interceptor.process(response, context);
final HttpEntity entity = response.getEntity();
Assert.assertNotNull(entity);
Assert.assertTrue(entity instanceof GzipDecompressingEntity);
Assert.assertTrue(entity instanceof DecompressingEntity);
}
@Test
@ -125,7 +125,7 @@ public class TestResponseContentEncoding {
interceptor.process(response, context);
final HttpEntity entity = response.getEntity();
Assert.assertNotNull(entity);
Assert.assertTrue(entity instanceof DeflateDecompressingEntity);
Assert.assertTrue(entity instanceof DecompressingEntity);
}
@Test