From e9d0960e8d468148039be9b76d99cc7e8f603077 Mon Sep 17 00:00:00 2001 From: Simone Bordet Date: Tue, 18 Sep 2012 15:44:08 +0200 Subject: [PATCH] jetty-9 - HTTP client: implemented gzip decoding. --- .../eclipse/jetty/client/ContentDecoder.java | 86 +++++ .../jetty/client/GZIPContentDecoder.java | 361 ++++++++++++++++++ .../org/eclipse/jetty/client/HttpClient.java | 11 + .../eclipse/jetty/client/HttpConnection.java | 43 ++- .../eclipse/jetty/client/HttpReceiver.java | 80 ++-- .../jetty/client/api/ContentDecoder.java | 24 -- .../org/eclipse/jetty/client/api/Request.java | 2 - .../util/InputStreamContentProvider.java | 82 ---- .../jetty/client/GZIPContentDecoderTest.java | 287 ++++++++++++++ .../eclipse/jetty/client/HttpClientTest.java | 27 ++ .../jetty/client/HttpReceiverTest.java | 43 +++ 11 files changed, 895 insertions(+), 151 deletions(-) create mode 100644 jetty-client/src/main/java/org/eclipse/jetty/client/ContentDecoder.java create mode 100644 jetty-client/src/main/java/org/eclipse/jetty/client/GZIPContentDecoder.java delete mode 100644 jetty-client/src/main/java/org/eclipse/jetty/client/api/ContentDecoder.java delete mode 100644 jetty-client/src/main/java/org/eclipse/jetty/client/util/InputStreamContentProvider.java create mode 100644 jetty-client/src/test/java/org/eclipse/jetty/client/GZIPContentDecoderTest.java diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/ContentDecoder.java b/jetty-client/src/main/java/org/eclipse/jetty/client/ContentDecoder.java new file mode 100644 index 00000000000..49e48c88d47 --- /dev/null +++ b/jetty-client/src/main/java/org/eclipse/jetty/client/ContentDecoder.java @@ -0,0 +1,86 @@ +// +// ======================================================================== +// Copyright (c) 1995-2012 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.client; + +import java.nio.ByteBuffer; + +/** + * {@link ContentDecoder} decodes content bytes of a response. + * + * @see Factory + */ +public interface ContentDecoder +{ + /** + *

Decodes the bytes in the given {@code buffer} and returns decoded bytes, if any.

+ * + * @param buffer the buffer containing encoded bytes + * @return a buffer containing decoded bytes, if any + */ + public abstract ByteBuffer decode(ByteBuffer buffer); + + /** + * Factory for {@link ContentDecoder}s; subclasses must implement {@link #newContentDecoder()}. + *

+ * {@link Factory} have an {@link #getEncoding() encoding}, which is the string used in + * {@code Accept-Encoding} request header and in {@code Content-Encoding} response headers. + *

+ * {@link Factory} instances are configured in {@link HttpClient} via + * {@link HttpClient#getContentDecoderFactories()}. + */ + public static abstract class Factory + { + private final String encoding; + + protected Factory(String encoding) + { + this.encoding = encoding; + } + + /** + * @return the type of the decoders created by this factory + */ + public String getEncoding() + { + return encoding; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) return true; + if (!(obj instanceof Factory)) return false; + Factory that = (Factory)obj; + return encoding.equals(that.encoding); + } + + @Override + public int hashCode() + { + return encoding.hashCode(); + } + + /** + * Factory method for {@link ContentDecoder}s + * + * @return a new instance of a {@link ContentDecoder} + */ + public abstract ContentDecoder newContentDecoder(); + } +} diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/GZIPContentDecoder.java b/jetty-client/src/main/java/org/eclipse/jetty/client/GZIPContentDecoder.java new file mode 100644 index 00000000000..ce01bfbf715 --- /dev/null +++ b/jetty-client/src/main/java/org/eclipse/jetty/client/GZIPContentDecoder.java @@ -0,0 +1,361 @@ +// +// ======================================================================== +// Copyright (c) 1995-2012 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.client; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; +import java.util.zip.ZipException; + +import org.eclipse.jetty.util.BufferUtil; + +/** + * {@link ContentDecoder} for the "gzip" encoding. + */ +public class GZIPContentDecoder implements ContentDecoder +{ + private final Inflater inflater = new Inflater(true); + private final byte[] bytes; + private byte[] output; + private State state; + private int size; + private int value; + private byte flags; + + public GZIPContentDecoder() + { + this(2048); + } + + public GZIPContentDecoder(int bufferSize) + { + this.bytes = new byte[bufferSize]; + reset(); + } + + /** + * {@inheritDoc} + *

If the decoding did not produce any output, for example because it consumed gzip header + * or trailer bytes, it returns a buffer with zero capacity.

+ *

This method never returns null.

+ *

The given {@code buffer}'s position will be modified to reflect the bytes consumed during + * the decoding.

+ *

The decoding may be finished without consuming the buffer completely if the buffer contains + * gzip bytes plus other bytes (either plain or gzipped).

+ */ + @Override + public ByteBuffer decode(ByteBuffer buffer) + { + try + { + while (buffer.hasRemaining()) + { + byte currByte = buffer.get(); + switch (state) + { + case INITIAL: + { + buffer.position(buffer.position() - 1); + state = State.ID; + break; + } + case ID: + { + value += (currByte & 0xFF) << 8 * size; + ++size; + if (size == 2) + { + if (value != 0x8B1F) + throw new ZipException("Invalid gzip bytes"); + state = State.CM; + } + break; + } + case CM: + { + if ((currByte & 0xFF) != 0x08) + throw new ZipException("Invalid gzip compression method"); + state = State.FLG; + break; + } + case FLG: + { + flags = currByte; + state = State.MTIME; + size = 0; + value = 0; + break; + } + case MTIME: + { + // Skip the 4 MTIME bytes + ++size; + if (size == 4) + state = State.XFL; + break; + } + case XFL: + { + // Skip XFL + state = State.OS; + break; + } + case OS: + { + // Skip OS + state = State.FLAGS; + break; + } + case FLAGS: + { + buffer.position(buffer.position() - 1); + if ((flags & 0x04) == 0x04) + { + state = State.EXTRA_LENGTH; + size = 0; + value = 0; + } + else if ((flags & 0x08) == 0x08) + state = State.NAME; + else if ((flags & 0x10) == 0x10) + state = State.COMMENT; + else if ((flags & 0x2) == 0x2) + { + state = State.HCRC; + size = 0; + value = 0; + } + else + state = State.DATA; + break; + } + case EXTRA_LENGTH: + { + value += (currByte & 0xFF) << 8 * size; + ++size; + if (size == 2) + state = State.EXTRA; + break; + } + case EXTRA: + { + // Skip EXTRA bytes + --value; + if (value == 0) + { + // Clear the EXTRA flag and loop on the flags + flags &= ~0x04; + state = State.FLAGS; + } + break; + } + case NAME: + { + // Skip NAME bytes + if (currByte == 0) + { + // Clear the NAME flag and loop on the flags + flags &= ~0x08; + state = State.FLAGS; + } + break; + } + case COMMENT: + { + // Skip COMMENT bytes + if (currByte == 0) + { + // Clear the COMMENT flag and loop on the flags + flags &= ~0x10; + state = State.FLAGS; + } + break; + } + case HCRC: + { + // Skip HCRC + ++size; + if (size == 2) + { + // Clear the HCRC flag and loop on the flags + flags &= ~0x02; + state = State.FLAGS; + } + break; + } + case DATA: + { + buffer.position(buffer.position() - 1); + while (true) + { + int decoded = inflate(bytes); + if (decoded == 0) + { + if (inflater.needsInput()) + { + if (buffer.hasRemaining()) + { + byte[] input = new byte[buffer.remaining()]; + buffer.get(input); + inflater.setInput(input); + } + else + { + if (output != null) + { + ByteBuffer result = ByteBuffer.wrap(output); + output = null; + return result; + } + break; + } + } + else if (inflater.finished()) + { + int remaining = inflater.getRemaining(); + buffer.position(buffer.limit() - remaining); + state = State.CRC; + size = 0; + value = 0; + break; + } + else + { + throw new ZipException("Invalid inflater state"); + } + } + else + { + if (output == null) + { + // Save the inflated bytes and loop to see if we have finished + output = new byte[decoded]; + System.arraycopy(bytes, 0, output, 0, decoded); + } + else + { + // Accumulate inflated bytes and loop to see if we have finished + byte[] newOutput = new byte[output.length + decoded]; + System.arraycopy(output, 0, newOutput, 0, output.length); + System.arraycopy(bytes, 0, newOutput, output.length, decoded); + output = newOutput; + } + } + } + break; + } + case CRC: + { + value += (currByte & 0xFF) << 8 * size; + ++size; + if (size == 4) + { + // From RFC 1952, compliant decoders need not to verify the CRC + state = State.ISIZE; + size = 0; + value = 0; + } + break; + } + case ISIZE: + { + value += (currByte & 0xFF) << 8 * size; + ++size; + if (size == 4) + { + if (value != inflater.getBytesWritten()) + throw new ZipException("Invalid input size"); + + ByteBuffer result = output == null ? BufferUtil.EMPTY_BUFFER : ByteBuffer.wrap(output); + reset(); + return result; + } + break; + } + default: + throw new ZipException(); + } + } + return BufferUtil.EMPTY_BUFFER; + } + catch (ZipException x) + { + throw new RuntimeException(x); + } + } + + private int inflate(byte[] bytes) throws ZipException + { + try + { + return inflater.inflate(bytes); + } + catch (DataFormatException x) + { + throw new ZipException(x.getMessage()); + } + } + + private void reset() + { + inflater.reset(); + Arrays.fill(bytes, (byte)0); + output = null; + state = State.INITIAL; + size = 0; + value = 0; + flags = 0; + } + + protected boolean finished() + { + return state == State.INITIAL; + } + + /** + * Specialized {@link ContentDecoder.Factory} for the "gzip" encoding. + */ + public static class Factory extends ContentDecoder.Factory + { + private final int bufferSize; + + public Factory() + { + this(2048); + } + + public Factory(int bufferSize) + { + super("gzip"); + this.bufferSize = bufferSize; + } + + @Override + public ContentDecoder newContentDecoder() + { + return new GZIPContentDecoder(bufferSize); + } + } + + private enum State + { + INITIAL, ID, CM, FLG, MTIME, XFL, OS, FLAGS, EXTRA_LENGTH, EXTRA, NAME, COMMENT, HCRC, DATA, CRC, ISIZE + } +} diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java index 2dea4dbf9c2..70bfdfa62a2 100644 --- a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java +++ b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java @@ -27,7 +27,9 @@ import java.nio.channels.SelectionKey; import java.nio.channels.SocketChannel; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -102,6 +104,7 @@ public class HttpClient extends AggregateLifeCycle private final List requestListeners = new CopyOnWriteArrayList<>(); private final CookieStore cookieStore = new HttpCookieStore(); private final AuthenticationStore authenticationStore = new HttpAuthenticationStore(); + private final Set decoderFactories = Collections.newSetFromMap(new ConcurrentHashMap()); private final SslContextFactory sslContextFactory; private volatile Executor executor; private volatile ByteBufferPool byteBufferPool; @@ -156,6 +159,8 @@ public class HttpClient extends AggregateLifeCycle handlers.add(new RedirectProtocolHandler(this)); handlers.add(new AuthenticationProtocolHandler(this)); + decoderFactories.add(new GZIPContentDecoder.Factory()); + super.doStart(); LOG.info("Started {}", this); @@ -181,6 +186,7 @@ public class HttpClient extends AggregateLifeCycle cookieStore.clear(); authenticationStore.clearAuthentications(); authenticationStore.clearAuthenticationResults(); + decoderFactories.clear(); super.doStop(); @@ -202,6 +208,11 @@ public class HttpClient extends AggregateLifeCycle return authenticationStore; } + public Set getContentDecoderFactories() + { + return decoderFactories; + } + public Future GET(String uri) { return GET(URI.create(uri)); diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpConnection.java b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpConnection.java index fe2f6a47d48..6fe17b7ad18 100644 --- a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpConnection.java +++ b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpConnection.java @@ -18,8 +18,10 @@ package org.eclipse.jetty.client; -import java.util.Collection; +import java.util.Enumeration; +import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import org.eclipse.jetty.client.api.Authentication; @@ -118,8 +120,6 @@ public class HttpConnection extends AbstractConnection implements Connection if (request.idleTimeout() <= 0) request.idleTimeout(client.getIdleTimeout()); - // TODO: follow redirects - HttpVersion version = request.version(); HttpFields headers = request.headers(); ContentProvider content = request.content(); @@ -165,7 +165,22 @@ public class HttpConnection extends AbstractConnection implements Connection if (authnResult != null) authnResult.apply(request); - // TODO: decoder headers + if (!headers.containsKey(HttpHeader.ACCEPT_ENCODING.asString())) + { + Set decoderFactories = client.getContentDecoderFactories(); + if (!decoderFactories.isEmpty()) + { + StringBuilder value = new StringBuilder(); + for (Iterator iterator = decoderFactories.iterator(); iterator.hasNext();) + { + ContentDecoder.Factory decoderFactory = iterator.next(); + value.append(decoderFactory.getEncoding()); + if (iterator.hasNext()) + value.append(","); + } + headers.put(HttpHeader.ACCEPT_ENCODING, value.toString()); + } + } // If we are HTTP 1.1, add the Host header if (version.getVersion() > 10) @@ -217,16 +232,20 @@ public class HttpConnection extends AbstractConnection implements Connection if (success) { HttpFields responseHeaders = exchange.response().headers(); - Collection values = responseHeaders.getValuesCollection(HttpHeader.CONNECTION.asString()); - if (values != null && values.contains("close")) + Enumeration values = responseHeaders.getValues(HttpHeader.CONNECTION.asString(), ","); + if (values != null) { - destination.remove(this); - close(); - } - else - { - destination.release(this); + while (values.hasMoreElements()) + { + if ("close".equalsIgnoreCase(values.nextElement())) + { + destination.remove(this); + close(); + return; + } + } } + destination.release(this); } else { diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java index ebc39d869ae..0ad2d9f9400 100644 --- a/jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java +++ b/jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java @@ -20,6 +20,7 @@ package org.eclipse.jetty.client; import java.io.EOFException; import java.nio.ByteBuffer; +import java.util.Enumeration; import java.util.List; import java.util.concurrent.TimeoutException; @@ -44,6 +45,7 @@ public class HttpReceiver implements HttpParser.ResponseHandler private final ResponseNotifier notifier = new ResponseNotifier(); private final HttpConnection connection; private volatile boolean failed; + private volatile ContentDecoder decoder; public HttpReceiver(HttpConnection connection) { @@ -117,14 +119,14 @@ public class HttpReceiver implements HttpParser.ResponseHandler if (currentListener == initialListener) conversation.listener(initialListener); else - conversation.listener(new MultipleResponseListener(currentListener, initialListener)); + conversation.listener(new DoubleResponseListener(currentListener, initialListener)); } else { if (currentListener == initialListener) conversation.listener(handlerListener); else - conversation.listener(new MultipleResponseListener(currentListener, handlerListener)); + conversation.listener(new DoubleResponseListener(currentListener, handlerListener)); } LOG.debug("Receiving {}", response); @@ -137,7 +139,7 @@ public class HttpReceiver implements HttpParser.ResponseHandler public boolean parsedHeader(HttpHeader header, String name, String value) { HttpExchange exchange = connection.getExchange(); - exchange.response().headers().put(name, value); + exchange.response().headers().add(name, value); switch (name.toLowerCase()) { @@ -168,6 +170,23 @@ public class HttpReceiver implements HttpParser.ResponseHandler HttpResponse response = exchange.response(); LOG.debug("Headers {}", response); notifier.notifyHeaders(conversation.listener(), response); + + Enumeration contentEncodings = response.headers().getValues(HttpHeader.CONTENT_ENCODING.asString(), ","); + if (contentEncodings != null) + { + for (ContentDecoder.Factory factory : connection.getHttpClient().getContentDecoderFactories()) + { + while (contentEncodings.hasMoreElements()) + { + if (factory.getEncoding().equalsIgnoreCase(contentEncodings.nextElement())) + { + this.decoder = factory.newContentDecoder(); + break; + } + } + } + } + return false; } @@ -178,6 +197,14 @@ public class HttpReceiver implements HttpParser.ResponseHandler HttpConversation conversation = exchange.conversation(); HttpResponse response = exchange.response(); LOG.debug("Content {}: {} bytes", response, buffer.remaining()); + + ContentDecoder decoder = this.decoder; + if (decoder != null) + { + buffer = decoder.decode(buffer); + LOG.debug("{} {}: {} bytes", decoder, response, buffer.remaining()); + } + notifier.notifyContent(conversation.listener(), response, buffer); return false; } @@ -195,6 +222,7 @@ public class HttpReceiver implements HttpParser.ResponseHandler protected void success() { parser.reset(); + decoder = null; HttpExchange exchange = connection.getExchange(); HttpResponse response = exchange.response(); @@ -253,68 +281,58 @@ public class HttpReceiver implements HttpParser.ResponseHandler fail(new TimeoutException()); } - private class MultipleResponseListener implements Response.Listener + private class DoubleResponseListener implements Response.Listener { private final ResponseNotifier notifier = new ResponseNotifier(); - private final Response.Listener[] listeners; + private final Response.Listener listener1; + private final Response.Listener listener2; - private MultipleResponseListener(Response.Listener... listeners) + private DoubleResponseListener(Response.Listener listener1, Response.Listener listener2) { - this.listeners = listeners; + this.listener1 = listener1; + this.listener2 = listener2; } @Override public void onBegin(Response response) { - for (Response.Listener listener : listeners) - { - notifier.notifyBegin(listener, response); - } + notifier.notifyBegin(listener1, response); + notifier.notifyBegin(listener2, response); } @Override public void onHeaders(Response response) { - for (Response.Listener listener : listeners) - { - notifier.notifyHeaders(listener, response); - } + notifier.notifyHeaders(listener1, response); + notifier.notifyHeaders(listener2, response); } @Override public void onContent(Response response, ByteBuffer content) { - for (Response.Listener listener : listeners) - { - notifier.notifyContent(listener, response, content); - } + notifier.notifyContent(listener1, response, content); + notifier.notifyContent(listener2, response, content); } @Override public void onSuccess(Response response) { - for (Response.Listener listener : listeners) - { - notifier.notifySuccess(listener, response); - } + notifier.notifySuccess(listener1, response); + notifier.notifySuccess(listener2, response); } @Override public void onFailure(Response response, Throwable failure) { - for (Response.Listener listener : listeners) - { - notifier.notifyFailure(listener, response, failure); - } + notifier.notifyFailure(listener1, response, failure); + notifier.notifyFailure(listener2, response, failure); } @Override public void onComplete(Result result) { - for (Response.Listener listener : listeners) - { - notifier.notifyComplete(listener, result); - } + notifier.notifyComplete(listener1, result); + notifier.notifyComplete(listener2, result); } } } diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/api/ContentDecoder.java b/jetty-client/src/main/java/org/eclipse/jetty/client/api/ContentDecoder.java deleted file mode 100644 index b8dc313e7a0..00000000000 --- a/jetty-client/src/main/java/org/eclipse/jetty/client/api/ContentDecoder.java +++ /dev/null @@ -1,24 +0,0 @@ -// -// ======================================================================== -// Copyright (c) 1995-2012 Mort Bay Consulting Pty. Ltd. -// ------------------------------------------------------------------------ -// All rights reserved. This program and the accompanying materials -// are made available under the terms of the Eclipse Public License v1.0 -// and Apache License v2.0 which accompanies this distribution. -// -// The Eclipse Public License is available at -// http://www.eclipse.org/legal/epl-v10.html -// -// The Apache License v2.0 is available at -// http://www.opensource.org/licenses/apache2.0.php -// -// You may elect to redistribute this code under either of these licenses. -// ======================================================================== -// - -package org.eclipse.jetty.client.api; - -// TODO -interface ContentDecoder -{ -} diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/api/Request.java b/jetty-client/src/main/java/org/eclipse/jetty/client/api/Request.java index 1c1899e3754..f5ce33402c1 100644 --- a/jetty-client/src/main/java/org/eclipse/jetty/client/api/Request.java +++ b/jetty-client/src/main/java/org/eclipse/jetty/client/api/Request.java @@ -159,8 +159,6 @@ public interface Request */ Request file(Path file, String contentType) throws IOException; -// Request decoder(ContentDecoder decoder); - /** * @return the user agent for this request */ diff --git a/jetty-client/src/main/java/org/eclipse/jetty/client/util/InputStreamContentProvider.java b/jetty-client/src/main/java/org/eclipse/jetty/client/util/InputStreamContentProvider.java deleted file mode 100644 index 3d550b857bc..00000000000 --- a/jetty-client/src/main/java/org/eclipse/jetty/client/util/InputStreamContentProvider.java +++ /dev/null @@ -1,82 +0,0 @@ -// -// ======================================================================== -// Copyright (c) 1995-2012 Mort Bay Consulting Pty. Ltd. -// ------------------------------------------------------------------------ -// All rights reserved. This program and the accompanying materials -// are made available under the terms of the Eclipse Public License v1.0 -// and Apache License v2.0 which accompanies this distribution. -// -// The Eclipse Public License is available at -// http://www.eclipse.org/legal/epl-v10.html -// -// The Apache License v2.0 is available at -// http://www.opensource.org/licenses/apache2.0.php -// -// You may elect to redistribute this code under either of these licenses. -// ======================================================================== -// - -package org.eclipse.jetty.client.util; - -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.util.Iterator; - -import org.eclipse.jetty.client.api.ContentProvider; - -public class InputStreamContentProvider implements ContentProvider -{ - private final InputStream input; - private final long length; - private final int capacity; - - public InputStreamContentProvider(InputStream input) - { - this(input, -1); - } - - public InputStreamContentProvider(InputStream input, long length) - { - this(input, length, length <= 0 ? 4096 : (int)Math.min(4096, length)); - } - - public InputStreamContentProvider(InputStream input, long length, int capacity) - { - this.input = input; - this.length = length; - this.capacity = capacity; - } - - @Override - public long length() - { - return length; - } - - @Override - public Iterator iterator() - { - return null; // TODO - } - - private class LazyIterator implements Iterator - { - @Override - public boolean hasNext() - { - return false; - } - - @Override - public ByteBuffer next() - { - return null; - } - - @Override - public void remove() - { - throw new UnsupportedOperationException(); - } - } -} diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/GZIPContentDecoderTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/GZIPContentDecoderTest.java new file mode 100644 index 00000000000..8b710845116 --- /dev/null +++ b/jetty-client/src/test/java/org/eclipse/jetty/client/GZIPContentDecoderTest.java @@ -0,0 +1,287 @@ +// +// ======================================================================== +// Copyright (c) 1995-2012 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.client; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class GZIPContentDecoderTest +{ + @Test + public void testStreamNoBlocks() throws Exception + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.close(); + byte[] bytes = baos.toByteArray(); + + GZIPInputStream input = new GZIPInputStream(new ByteArrayInputStream(bytes), 1); + int read = input.read(); + assertEquals(-1, read); + } + + @Test + public void testStreamBigBlockOneByteAtATime() throws Exception + { + String data = "0123456789ABCDEF"; + for (int i = 0; i < 10; ++i) + data += data; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes("UTF-8")); + output.close(); + byte[] bytes = baos.toByteArray(); + + baos = new ByteArrayOutputStream(); + GZIPInputStream input = new GZIPInputStream(new ByteArrayInputStream(bytes), 1); + int read; + while ((read = input.read()) >= 0) + baos.write(read); + assertEquals(data, new String(baos.toByteArray(), "UTF-8")); + } + + @Test + public void testNoBlocks() throws Exception + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.close(); + byte[] bytes = baos.toByteArray(); + + GZIPContentDecoder decoder = new GZIPContentDecoder(); + ByteBuffer decoded = decoder.decode(ByteBuffer.wrap(bytes)); + assertEquals(0, decoded.remaining()); + } + + @Test + public void testSmallBlock() throws Exception + { + String data = "0"; + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes("UTF-8")); + output.close(); + byte[] bytes = baos.toByteArray(); + + GZIPContentDecoder decoder = new GZIPContentDecoder(); + ByteBuffer decoded = decoder.decode(ByteBuffer.wrap(bytes)); + assertEquals(data, Charset.forName("UTF-8").decode(decoded).toString()); + } + + @Test + public void testSmallBlockWithGZIPChunkedAtBegin() throws Exception + { + String data = "0"; + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes("UTF-8")); + output.close(); + byte[] bytes = baos.toByteArray(); + + // The header is 10 bytes, chunk at 11 bytes + byte[] bytes1 = new byte[11]; + System.arraycopy(bytes, 0, bytes1, 0, bytes1.length); + byte[] bytes2 = new byte[bytes.length - bytes1.length]; + System.arraycopy(bytes, bytes1.length, bytes2, 0, bytes2.length); + + GZIPContentDecoder decoder = new GZIPContentDecoder(); + ByteBuffer decoded = decoder.decode(ByteBuffer.wrap(bytes1)); + assertEquals(0, decoded.capacity()); + decoded = decoder.decode(ByteBuffer.wrap(bytes2)); + assertEquals(data, Charset.forName("UTF-8").decode(decoded).toString()); + } + + @Test + public void testSmallBlockWithGZIPChunkedAtEnd() throws Exception + { + String data = "0"; + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes("UTF-8")); + output.close(); + byte[] bytes = baos.toByteArray(); + + // The trailer is 8 bytes, chunk the last 9 bytes + byte[] bytes1 = new byte[bytes.length - 9]; + System.arraycopy(bytes, 0, bytes1, 0, bytes1.length); + byte[] bytes2 = new byte[bytes.length - bytes1.length]; + System.arraycopy(bytes, bytes1.length, bytes2, 0, bytes2.length); + + GZIPContentDecoder decoder = new GZIPContentDecoder(); + ByteBuffer decoded = decoder.decode(ByteBuffer.wrap(bytes1)); + assertEquals(data, Charset.forName("UTF-8").decode(decoded).toString()); + assertFalse(decoder.finished()); + decoded = decoder.decode(ByteBuffer.wrap(bytes2)); + assertEquals(0, decoded.remaining()); + assertTrue(decoder.finished()); + } + + @Test + public void testSmallBlockWithGZIPTrailerChunked() throws Exception + { + String data = "0"; + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes("UTF-8")); + output.close(); + byte[] bytes = baos.toByteArray(); + + // The trailer is 4+4 bytes, chunk the last 3 bytes + byte[] bytes1 = new byte[bytes.length - 3]; + System.arraycopy(bytes, 0, bytes1, 0, bytes1.length); + byte[] bytes2 = new byte[bytes.length - bytes1.length]; + System.arraycopy(bytes, bytes1.length, bytes2, 0, bytes2.length); + + GZIPContentDecoder decoder = new GZIPContentDecoder(); + ByteBuffer decoded = decoder.decode(ByteBuffer.wrap(bytes1)); + assertEquals(0, decoded.capacity()); + decoded = decoder.decode(ByteBuffer.wrap(bytes2)); + assertEquals(data, Charset.forName("UTF-8").decode(decoded).toString()); + } + + @Test + public void testTwoSmallBlocks() throws Exception + { + String data1 = "0"; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data1.getBytes("UTF-8")); + output.close(); + byte[] bytes1 = baos.toByteArray(); + + String data2 = "1"; + baos = new ByteArrayOutputStream(); + output = new GZIPOutputStream(baos); + output.write(data2.getBytes("UTF-8")); + output.close(); + byte[] bytes2 = baos.toByteArray(); + + byte[] bytes = new byte[bytes1.length + bytes2.length]; + System.arraycopy(bytes1, 0, bytes, 0, bytes1.length); + System.arraycopy(bytes2, 0, bytes, bytes1.length, bytes2.length); + + GZIPContentDecoder decoder = new GZIPContentDecoder(); + ByteBuffer buffer = ByteBuffer.wrap(bytes); + ByteBuffer decoded = decoder.decode(buffer); + assertEquals(data1, Charset.forName("UTF-8").decode(decoded).toString()); + assertTrue(decoder.finished()); + assertTrue(buffer.hasRemaining()); + decoded = decoder.decode(buffer); + assertEquals(data2, Charset.forName("UTF-8").decode(decoded).toString()); + assertTrue(decoder.finished()); + assertFalse(buffer.hasRemaining()); + } + + @Test + public void testBigBlock() throws Exception + { + String data = "0123456789ABCDEF"; + for (int i = 0; i < 10; ++i) + data += data; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes("UTF-8")); + output.close(); + byte[] bytes = baos.toByteArray(); + + String result = ""; + GZIPContentDecoder decoder = new GZIPContentDecoder(); + ByteBuffer buffer = ByteBuffer.wrap(bytes); + while (buffer.hasRemaining()) + { + ByteBuffer decoded = decoder.decode(buffer); + result += Charset.forName("UTF-8").decode(decoded).toString(); + } + assertEquals(data, result); + } + + @Test + public void testBigBlockOneByteAtATime() throws Exception + { + String data = "0123456789ABCDEF"; + for (int i = 0; i < 10; ++i) + data += data; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes("UTF-8")); + output.close(); + byte[] bytes = baos.toByteArray(); + + String result = ""; + GZIPContentDecoder decoder = new GZIPContentDecoder(64); + ByteBuffer buffer = ByteBuffer.wrap(bytes); + while (buffer.hasRemaining()) + { + ByteBuffer decoded = decoder.decode(ByteBuffer.wrap(new byte[]{buffer.get()})); + if (decoded.hasRemaining()) + result += Charset.forName("UTF-8").decode(decoded).toString(); + } + assertEquals(data, result); + assertTrue(decoder.finished()); + } + + @Test + public void testBigBlockWithExtraBytes() throws Exception + { + String data1 = "0123456789ABCDEF"; + for (int i = 0; i < 10; ++i) + data1 += data1; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data1.getBytes("UTF-8")); + output.close(); + byte[] bytes1 = baos.toByteArray(); + + String data2 = "HELLO"; + byte[] bytes2 = data2.getBytes("UTF-8"); + + byte[] bytes = new byte[bytes1.length + bytes2.length]; + System.arraycopy(bytes1, 0, bytes, 0, bytes1.length); + System.arraycopy(bytes2, 0, bytes, bytes1.length, bytes2.length); + + String result = ""; + GZIPContentDecoder decoder = new GZIPContentDecoder(64); + ByteBuffer buffer = ByteBuffer.wrap(bytes); + while (buffer.hasRemaining()) + { + ByteBuffer decoded = decoder.decode(buffer); + if (decoded.hasRemaining()) + result += Charset.forName("UTF-8").decode(decoded).toString(); + if (decoder.finished()) + break; + } + assertEquals(data1, result); + assertTrue(buffer.hasRemaining()); + assertEquals(data2, Charset.forName("UTF-8").decode(buffer).toString()); + } +} diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java index e120e648090..e5270e8cfd5 100644 --- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java +++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java @@ -31,6 +31,7 @@ import java.util.NoSuchElementException; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.zip.GZIPOutputStream; import javax.servlet.ServletException; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; @@ -484,4 +485,30 @@ public class HttpClientTest extends AbstractHttpClientServerTest Assert.assertTrue(latch.await(5, TimeUnit.SECONDS)); } + + @Test + public void test_GZIP_ContentEncoding() throws Exception + { + final byte[] data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + start(new AbstractHandler() + { + @Override + public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException + { + baseRequest.setHandled(true); + response.setHeader("Content-Encoding", "gzip"); + GZIPOutputStream gzipOutput = new GZIPOutputStream(response.getOutputStream()); + gzipOutput.write(data); + gzipOutput.finish(); + } + }); + + ContentResponse response = client.newRequest("localhost", connector.getLocalPort()) + .scheme(scheme) + .send() + .get(5, TimeUnit.SECONDS); + + Assert.assertEquals(200, response.status()); + Assert.assertArrayEquals(data, response.content()); + } } diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpReceiverTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpReceiverTest.java index 66fd82fb61c..637c860b028 100644 --- a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpReceiverTest.java +++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpReceiverTest.java @@ -18,13 +18,17 @@ package org.eclipse.jetty.client; +import java.io.ByteArrayOutputStream; import java.io.EOFException; +import java.nio.ByteBuffer; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; +import java.util.zip.GZIPOutputStream; +import org.eclipse.jetty.client.api.ContentResponse; import org.eclipse.jetty.client.api.Response; import org.eclipse.jetty.client.util.BlockingResponseListener; import org.eclipse.jetty.http.HttpFields; @@ -200,4 +204,43 @@ public class HttpReceiverTest Assert.assertTrue(e.getCause() instanceof HttpResponseException); } } + + @Test + public void test_Receive_GZIPResponseContent_Fragmented() throws Exception + { + byte[] data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (GZIPOutputStream gzipOutput = new GZIPOutputStream(baos)) + { + gzipOutput.write(data); + } + byte[] gzip = baos.toByteArray(); + + endPoint.setInput("" + + "HTTP/1.1 200 OK\r\n" + + "Content-Length: " + gzip.length + "\r\n" + + "Content-Encoding: gzip\r\n" + + "\r\n"); + BlockingResponseListener listener = new BlockingResponseListener(); + HttpExchange exchange = newExchange(listener); + exchange.receive(); + endPoint.reset(); + + ByteBuffer buffer = ByteBuffer.wrap(gzip); + int fragment = buffer.limit() - 1; + buffer.limit(fragment); + endPoint.setInput(buffer); + exchange.receive(); + endPoint.reset(); + + buffer.limit(gzip.length); + buffer.position(fragment); + endPoint.setInput(buffer); + exchange.receive(); + + ContentResponse response = listener.get(5, TimeUnit.SECONDS); + Assert.assertNotNull(response); + Assert.assertEquals(200, response.status()); + Assert.assertArrayEquals(data, response.content()); + } }