jetty-9 - HTTP client: implemented gzip decoding.

This commit is contained in:
Simone Bordet 2012-09-18 15:44:08 +02:00
parent f1d5fd1f9d
commit e9d0960e8d
11 changed files with 895 additions and 151 deletions

View File

@ -0,0 +1,86 @@
//
// ========================================================================
// Copyright (c) 1995-2012 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.client;
import java.nio.ByteBuffer;
/**
* {@link ContentDecoder} decodes content bytes of a response.
*
* @see Factory
*/
public interface ContentDecoder
{
/**
* <p>Decodes the bytes in the given {@code buffer} and returns decoded bytes, if any.</p>
*
* @param buffer the buffer containing encoded bytes
* @return a buffer containing decoded bytes, if any
*/
public abstract ByteBuffer decode(ByteBuffer buffer);
/**
* Factory for {@link ContentDecoder}s; subclasses must implement {@link #newContentDecoder()}.
* <p />
* {@link Factory} have an {@link #getEncoding() encoding}, which is the string used in
* {@code Accept-Encoding} request header and in {@code Content-Encoding} response headers.
* <p />
* {@link Factory} instances are configured in {@link HttpClient} via
* {@link HttpClient#getContentDecoderFactories()}.
*/
public static abstract class Factory
{
private final String encoding;
protected Factory(String encoding)
{
this.encoding = encoding;
}
/**
* @return the type of the decoders created by this factory
*/
public String getEncoding()
{
return encoding;
}
@Override
public boolean equals(Object obj)
{
if (this == obj) return true;
if (!(obj instanceof Factory)) return false;
Factory that = (Factory)obj;
return encoding.equals(that.encoding);
}
@Override
public int hashCode()
{
return encoding.hashCode();
}
/**
* Factory method for {@link ContentDecoder}s
*
* @return a new instance of a {@link ContentDecoder}
*/
public abstract ContentDecoder newContentDecoder();
}
}

View File

@ -0,0 +1,361 @@
//
// ========================================================================
// Copyright (c) 1995-2012 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.client;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;
import java.util.zip.ZipException;
import org.eclipse.jetty.util.BufferUtil;
/**
* {@link ContentDecoder} for the "gzip" encoding.
*/
public class GZIPContentDecoder implements ContentDecoder
{
private final Inflater inflater = new Inflater(true);
private final byte[] bytes;
private byte[] output;
private State state;
private int size;
private int value;
private byte flags;
public GZIPContentDecoder()
{
this(2048);
}
public GZIPContentDecoder(int bufferSize)
{
this.bytes = new byte[bufferSize];
reset();
}
/**
* {@inheritDoc}
* <p>If the decoding did not produce any output, for example because it consumed gzip header
* or trailer bytes, it returns a buffer with zero capacity.</p>
* <p>This method never returns null.</p>
* <p>The given {@code buffer}'s position will be modified to reflect the bytes consumed during
* the decoding.</p>
* <p>The decoding may be finished without consuming the buffer completely if the buffer contains
* gzip bytes plus other bytes (either plain or gzipped).</p>
*/
@Override
public ByteBuffer decode(ByteBuffer buffer)
{
try
{
while (buffer.hasRemaining())
{
byte currByte = buffer.get();
switch (state)
{
case INITIAL:
{
buffer.position(buffer.position() - 1);
state = State.ID;
break;
}
case ID:
{
value += (currByte & 0xFF) << 8 * size;
++size;
if (size == 2)
{
if (value != 0x8B1F)
throw new ZipException("Invalid gzip bytes");
state = State.CM;
}
break;
}
case CM:
{
if ((currByte & 0xFF) != 0x08)
throw new ZipException("Invalid gzip compression method");
state = State.FLG;
break;
}
case FLG:
{
flags = currByte;
state = State.MTIME;
size = 0;
value = 0;
break;
}
case MTIME:
{
// Skip the 4 MTIME bytes
++size;
if (size == 4)
state = State.XFL;
break;
}
case XFL:
{
// Skip XFL
state = State.OS;
break;
}
case OS:
{
// Skip OS
state = State.FLAGS;
break;
}
case FLAGS:
{
buffer.position(buffer.position() - 1);
if ((flags & 0x04) == 0x04)
{
state = State.EXTRA_LENGTH;
size = 0;
value = 0;
}
else if ((flags & 0x08) == 0x08)
state = State.NAME;
else if ((flags & 0x10) == 0x10)
state = State.COMMENT;
else if ((flags & 0x2) == 0x2)
{
state = State.HCRC;
size = 0;
value = 0;
}
else
state = State.DATA;
break;
}
case EXTRA_LENGTH:
{
value += (currByte & 0xFF) << 8 * size;
++size;
if (size == 2)
state = State.EXTRA;
break;
}
case EXTRA:
{
// Skip EXTRA bytes
--value;
if (value == 0)
{
// Clear the EXTRA flag and loop on the flags
flags &= ~0x04;
state = State.FLAGS;
}
break;
}
case NAME:
{
// Skip NAME bytes
if (currByte == 0)
{
// Clear the NAME flag and loop on the flags
flags &= ~0x08;
state = State.FLAGS;
}
break;
}
case COMMENT:
{
// Skip COMMENT bytes
if (currByte == 0)
{
// Clear the COMMENT flag and loop on the flags
flags &= ~0x10;
state = State.FLAGS;
}
break;
}
case HCRC:
{
// Skip HCRC
++size;
if (size == 2)
{
// Clear the HCRC flag and loop on the flags
flags &= ~0x02;
state = State.FLAGS;
}
break;
}
case DATA:
{
buffer.position(buffer.position() - 1);
while (true)
{
int decoded = inflate(bytes);
if (decoded == 0)
{
if (inflater.needsInput())
{
if (buffer.hasRemaining())
{
byte[] input = new byte[buffer.remaining()];
buffer.get(input);
inflater.setInput(input);
}
else
{
if (output != null)
{
ByteBuffer result = ByteBuffer.wrap(output);
output = null;
return result;
}
break;
}
}
else if (inflater.finished())
{
int remaining = inflater.getRemaining();
buffer.position(buffer.limit() - remaining);
state = State.CRC;
size = 0;
value = 0;
break;
}
else
{
throw new ZipException("Invalid inflater state");
}
}
else
{
if (output == null)
{
// Save the inflated bytes and loop to see if we have finished
output = new byte[decoded];
System.arraycopy(bytes, 0, output, 0, decoded);
}
else
{
// Accumulate inflated bytes and loop to see if we have finished
byte[] newOutput = new byte[output.length + decoded];
System.arraycopy(output, 0, newOutput, 0, output.length);
System.arraycopy(bytes, 0, newOutput, output.length, decoded);
output = newOutput;
}
}
}
break;
}
case CRC:
{
value += (currByte & 0xFF) << 8 * size;
++size;
if (size == 4)
{
// From RFC 1952, compliant decoders need not to verify the CRC
state = State.ISIZE;
size = 0;
value = 0;
}
break;
}
case ISIZE:
{
value += (currByte & 0xFF) << 8 * size;
++size;
if (size == 4)
{
if (value != inflater.getBytesWritten())
throw new ZipException("Invalid input size");
ByteBuffer result = output == null ? BufferUtil.EMPTY_BUFFER : ByteBuffer.wrap(output);
reset();
return result;
}
break;
}
default:
throw new ZipException();
}
}
return BufferUtil.EMPTY_BUFFER;
}
catch (ZipException x)
{
throw new RuntimeException(x);
}
}
private int inflate(byte[] bytes) throws ZipException
{
try
{
return inflater.inflate(bytes);
}
catch (DataFormatException x)
{
throw new ZipException(x.getMessage());
}
}
private void reset()
{
inflater.reset();
Arrays.fill(bytes, (byte)0);
output = null;
state = State.INITIAL;
size = 0;
value = 0;
flags = 0;
}
protected boolean finished()
{
return state == State.INITIAL;
}
/**
* Specialized {@link ContentDecoder.Factory} for the "gzip" encoding.
*/
public static class Factory extends ContentDecoder.Factory
{
private final int bufferSize;
public Factory()
{
this(2048);
}
public Factory(int bufferSize)
{
super("gzip");
this.bufferSize = bufferSize;
}
@Override
public ContentDecoder newContentDecoder()
{
return new GZIPContentDecoder(bufferSize);
}
}
private enum State
{
INITIAL, ID, CM, FLG, MTIME, XFL, OS, FLAGS, EXTRA_LENGTH, EXTRA, NAME, COMMENT, HCRC, DATA, CRC, ISIZE
}
}

View File

@ -27,7 +27,9 @@ import java.nio.channels.SelectionKey;
import java.nio.channels.SocketChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CopyOnWriteArrayList;
@ -102,6 +104,7 @@ public class HttpClient extends AggregateLifeCycle
private final List<Request.Listener> requestListeners = new CopyOnWriteArrayList<>();
private final CookieStore cookieStore = new HttpCookieStore();
private final AuthenticationStore authenticationStore = new HttpAuthenticationStore();
private final Set<ContentDecoder.Factory> decoderFactories = Collections.newSetFromMap(new ConcurrentHashMap<ContentDecoder.Factory, Boolean>());
private final SslContextFactory sslContextFactory;
private volatile Executor executor;
private volatile ByteBufferPool byteBufferPool;
@ -156,6 +159,8 @@ public class HttpClient extends AggregateLifeCycle
handlers.add(new RedirectProtocolHandler(this));
handlers.add(new AuthenticationProtocolHandler(this));
decoderFactories.add(new GZIPContentDecoder.Factory());
super.doStart();
LOG.info("Started {}", this);
@ -181,6 +186,7 @@ public class HttpClient extends AggregateLifeCycle
cookieStore.clear();
authenticationStore.clearAuthentications();
authenticationStore.clearAuthenticationResults();
decoderFactories.clear();
super.doStop();
@ -202,6 +208,11 @@ public class HttpClient extends AggregateLifeCycle
return authenticationStore;
}
public Set<ContentDecoder.Factory> getContentDecoderFactories()
{
return decoderFactories;
}
public Future<ContentResponse> GET(String uri)
{
return GET(URI.create(uri));

View File

@ -18,8 +18,10 @@
package org.eclipse.jetty.client;
import java.util.Collection;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import org.eclipse.jetty.client.api.Authentication;
@ -118,8 +120,6 @@ public class HttpConnection extends AbstractConnection implements Connection
if (request.idleTimeout() <= 0)
request.idleTimeout(client.getIdleTimeout());
// TODO: follow redirects
HttpVersion version = request.version();
HttpFields headers = request.headers();
ContentProvider content = request.content();
@ -165,7 +165,22 @@ public class HttpConnection extends AbstractConnection implements Connection
if (authnResult != null)
authnResult.apply(request);
// TODO: decoder headers
if (!headers.containsKey(HttpHeader.ACCEPT_ENCODING.asString()))
{
Set<ContentDecoder.Factory> decoderFactories = client.getContentDecoderFactories();
if (!decoderFactories.isEmpty())
{
StringBuilder value = new StringBuilder();
for (Iterator<ContentDecoder.Factory> iterator = decoderFactories.iterator(); iterator.hasNext();)
{
ContentDecoder.Factory decoderFactory = iterator.next();
value.append(decoderFactory.getEncoding());
if (iterator.hasNext())
value.append(",");
}
headers.put(HttpHeader.ACCEPT_ENCODING, value.toString());
}
}
// If we are HTTP 1.1, add the Host header
if (version.getVersion() > 10)
@ -217,16 +232,20 @@ public class HttpConnection extends AbstractConnection implements Connection
if (success)
{
HttpFields responseHeaders = exchange.response().headers();
Collection<String> values = responseHeaders.getValuesCollection(HttpHeader.CONNECTION.asString());
if (values != null && values.contains("close"))
Enumeration<String> values = responseHeaders.getValues(HttpHeader.CONNECTION.asString(), ",");
if (values != null)
{
destination.remove(this);
close();
}
else
{
destination.release(this);
while (values.hasMoreElements())
{
if ("close".equalsIgnoreCase(values.nextElement()))
{
destination.remove(this);
close();
return;
}
}
}
destination.release(this);
}
else
{

View File

@ -20,6 +20,7 @@ package org.eclipse.jetty.client;
import java.io.EOFException;
import java.nio.ByteBuffer;
import java.util.Enumeration;
import java.util.List;
import java.util.concurrent.TimeoutException;
@ -44,6 +45,7 @@ public class HttpReceiver implements HttpParser.ResponseHandler<ByteBuffer>
private final ResponseNotifier notifier = new ResponseNotifier();
private final HttpConnection connection;
private volatile boolean failed;
private volatile ContentDecoder decoder;
public HttpReceiver(HttpConnection connection)
{
@ -117,14 +119,14 @@ public class HttpReceiver implements HttpParser.ResponseHandler<ByteBuffer>
if (currentListener == initialListener)
conversation.listener(initialListener);
else
conversation.listener(new MultipleResponseListener(currentListener, initialListener));
conversation.listener(new DoubleResponseListener(currentListener, initialListener));
}
else
{
if (currentListener == initialListener)
conversation.listener(handlerListener);
else
conversation.listener(new MultipleResponseListener(currentListener, handlerListener));
conversation.listener(new DoubleResponseListener(currentListener, handlerListener));
}
LOG.debug("Receiving {}", response);
@ -137,7 +139,7 @@ public class HttpReceiver implements HttpParser.ResponseHandler<ByteBuffer>
public boolean parsedHeader(HttpHeader header, String name, String value)
{
HttpExchange exchange = connection.getExchange();
exchange.response().headers().put(name, value);
exchange.response().headers().add(name, value);
switch (name.toLowerCase())
{
@ -168,6 +170,23 @@ public class HttpReceiver implements HttpParser.ResponseHandler<ByteBuffer>
HttpResponse response = exchange.response();
LOG.debug("Headers {}", response);
notifier.notifyHeaders(conversation.listener(), response);
Enumeration<String> contentEncodings = response.headers().getValues(HttpHeader.CONTENT_ENCODING.asString(), ",");
if (contentEncodings != null)
{
for (ContentDecoder.Factory factory : connection.getHttpClient().getContentDecoderFactories())
{
while (contentEncodings.hasMoreElements())
{
if (factory.getEncoding().equalsIgnoreCase(contentEncodings.nextElement()))
{
this.decoder = factory.newContentDecoder();
break;
}
}
}
}
return false;
}
@ -178,6 +197,14 @@ public class HttpReceiver implements HttpParser.ResponseHandler<ByteBuffer>
HttpConversation conversation = exchange.conversation();
HttpResponse response = exchange.response();
LOG.debug("Content {}: {} bytes", response, buffer.remaining());
ContentDecoder decoder = this.decoder;
if (decoder != null)
{
buffer = decoder.decode(buffer);
LOG.debug("{} {}: {} bytes", decoder, response, buffer.remaining());
}
notifier.notifyContent(conversation.listener(), response, buffer);
return false;
}
@ -195,6 +222,7 @@ public class HttpReceiver implements HttpParser.ResponseHandler<ByteBuffer>
protected void success()
{
parser.reset();
decoder = null;
HttpExchange exchange = connection.getExchange();
HttpResponse response = exchange.response();
@ -253,68 +281,58 @@ public class HttpReceiver implements HttpParser.ResponseHandler<ByteBuffer>
fail(new TimeoutException());
}
private class MultipleResponseListener implements Response.Listener
private class DoubleResponseListener implements Response.Listener
{
private final ResponseNotifier notifier = new ResponseNotifier();
private final Response.Listener[] listeners;
private final Response.Listener listener1;
private final Response.Listener listener2;
private MultipleResponseListener(Response.Listener... listeners)
private DoubleResponseListener(Response.Listener listener1, Response.Listener listener2)
{
this.listeners = listeners;
this.listener1 = listener1;
this.listener2 = listener2;
}
@Override
public void onBegin(Response response)
{
for (Response.Listener listener : listeners)
{
notifier.notifyBegin(listener, response);
}
notifier.notifyBegin(listener1, response);
notifier.notifyBegin(listener2, response);
}
@Override
public void onHeaders(Response response)
{
for (Response.Listener listener : listeners)
{
notifier.notifyHeaders(listener, response);
}
notifier.notifyHeaders(listener1, response);
notifier.notifyHeaders(listener2, response);
}
@Override
public void onContent(Response response, ByteBuffer content)
{
for (Response.Listener listener : listeners)
{
notifier.notifyContent(listener, response, content);
}
notifier.notifyContent(listener1, response, content);
notifier.notifyContent(listener2, response, content);
}
@Override
public void onSuccess(Response response)
{
for (Response.Listener listener : listeners)
{
notifier.notifySuccess(listener, response);
}
notifier.notifySuccess(listener1, response);
notifier.notifySuccess(listener2, response);
}
@Override
public void onFailure(Response response, Throwable failure)
{
for (Response.Listener listener : listeners)
{
notifier.notifyFailure(listener, response, failure);
}
notifier.notifyFailure(listener1, response, failure);
notifier.notifyFailure(listener2, response, failure);
}
@Override
public void onComplete(Result result)
{
for (Response.Listener listener : listeners)
{
notifier.notifyComplete(listener, result);
}
notifier.notifyComplete(listener1, result);
notifier.notifyComplete(listener2, result);
}
}
}

View File

@ -1,24 +0,0 @@
//
// ========================================================================
// Copyright (c) 1995-2012 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.client.api;
// TODO
interface ContentDecoder
{
}

View File

@ -159,8 +159,6 @@ public interface Request
*/
Request file(Path file, String contentType) throws IOException;
// Request decoder(ContentDecoder decoder);
/**
* @return the user agent for this request
*/

View File

@ -1,82 +0,0 @@
//
// ========================================================================
// Copyright (c) 1995-2012 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.client.util;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Iterator;
import org.eclipse.jetty.client.api.ContentProvider;
public class InputStreamContentProvider implements ContentProvider
{
private final InputStream input;
private final long length;
private final int capacity;
public InputStreamContentProvider(InputStream input)
{
this(input, -1);
}
public InputStreamContentProvider(InputStream input, long length)
{
this(input, length, length <= 0 ? 4096 : (int)Math.min(4096, length));
}
public InputStreamContentProvider(InputStream input, long length, int capacity)
{
this.input = input;
this.length = length;
this.capacity = capacity;
}
@Override
public long length()
{
return length;
}
@Override
public Iterator<ByteBuffer> iterator()
{
return null; // TODO
}
private class LazyIterator implements Iterator<ByteBuffer>
{
@Override
public boolean hasNext()
{
return false;
}
@Override
public ByteBuffer next()
{
return null;
}
@Override
public void remove()
{
throw new UnsupportedOperationException();
}
}
}

View File

@ -0,0 +1,287 @@
//
// ========================================================================
// Copyright (c) 1995-2012 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.client;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class GZIPContentDecoderTest
{
@Test
public void testStreamNoBlocks() throws Exception
{
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.close();
byte[] bytes = baos.toByteArray();
GZIPInputStream input = new GZIPInputStream(new ByteArrayInputStream(bytes), 1);
int read = input.read();
assertEquals(-1, read);
}
@Test
public void testStreamBigBlockOneByteAtATime() throws Exception
{
String data = "0123456789ABCDEF";
for (int i = 0; i < 10; ++i)
data += data;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data.getBytes("UTF-8"));
output.close();
byte[] bytes = baos.toByteArray();
baos = new ByteArrayOutputStream();
GZIPInputStream input = new GZIPInputStream(new ByteArrayInputStream(bytes), 1);
int read;
while ((read = input.read()) >= 0)
baos.write(read);
assertEquals(data, new String(baos.toByteArray(), "UTF-8"));
}
@Test
public void testNoBlocks() throws Exception
{
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.close();
byte[] bytes = baos.toByteArray();
GZIPContentDecoder decoder = new GZIPContentDecoder();
ByteBuffer decoded = decoder.decode(ByteBuffer.wrap(bytes));
assertEquals(0, decoded.remaining());
}
@Test
public void testSmallBlock() throws Exception
{
String data = "0";
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data.getBytes("UTF-8"));
output.close();
byte[] bytes = baos.toByteArray();
GZIPContentDecoder decoder = new GZIPContentDecoder();
ByteBuffer decoded = decoder.decode(ByteBuffer.wrap(bytes));
assertEquals(data, Charset.forName("UTF-8").decode(decoded).toString());
}
@Test
public void testSmallBlockWithGZIPChunkedAtBegin() throws Exception
{
String data = "0";
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data.getBytes("UTF-8"));
output.close();
byte[] bytes = baos.toByteArray();
// The header is 10 bytes, chunk at 11 bytes
byte[] bytes1 = new byte[11];
System.arraycopy(bytes, 0, bytes1, 0, bytes1.length);
byte[] bytes2 = new byte[bytes.length - bytes1.length];
System.arraycopy(bytes, bytes1.length, bytes2, 0, bytes2.length);
GZIPContentDecoder decoder = new GZIPContentDecoder();
ByteBuffer decoded = decoder.decode(ByteBuffer.wrap(bytes1));
assertEquals(0, decoded.capacity());
decoded = decoder.decode(ByteBuffer.wrap(bytes2));
assertEquals(data, Charset.forName("UTF-8").decode(decoded).toString());
}
@Test
public void testSmallBlockWithGZIPChunkedAtEnd() throws Exception
{
String data = "0";
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data.getBytes("UTF-8"));
output.close();
byte[] bytes = baos.toByteArray();
// The trailer is 8 bytes, chunk the last 9 bytes
byte[] bytes1 = new byte[bytes.length - 9];
System.arraycopy(bytes, 0, bytes1, 0, bytes1.length);
byte[] bytes2 = new byte[bytes.length - bytes1.length];
System.arraycopy(bytes, bytes1.length, bytes2, 0, bytes2.length);
GZIPContentDecoder decoder = new GZIPContentDecoder();
ByteBuffer decoded = decoder.decode(ByteBuffer.wrap(bytes1));
assertEquals(data, Charset.forName("UTF-8").decode(decoded).toString());
assertFalse(decoder.finished());
decoded = decoder.decode(ByteBuffer.wrap(bytes2));
assertEquals(0, decoded.remaining());
assertTrue(decoder.finished());
}
@Test
public void testSmallBlockWithGZIPTrailerChunked() throws Exception
{
String data = "0";
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data.getBytes("UTF-8"));
output.close();
byte[] bytes = baos.toByteArray();
// The trailer is 4+4 bytes, chunk the last 3 bytes
byte[] bytes1 = new byte[bytes.length - 3];
System.arraycopy(bytes, 0, bytes1, 0, bytes1.length);
byte[] bytes2 = new byte[bytes.length - bytes1.length];
System.arraycopy(bytes, bytes1.length, bytes2, 0, bytes2.length);
GZIPContentDecoder decoder = new GZIPContentDecoder();
ByteBuffer decoded = decoder.decode(ByteBuffer.wrap(bytes1));
assertEquals(0, decoded.capacity());
decoded = decoder.decode(ByteBuffer.wrap(bytes2));
assertEquals(data, Charset.forName("UTF-8").decode(decoded).toString());
}
@Test
public void testTwoSmallBlocks() throws Exception
{
String data1 = "0";
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data1.getBytes("UTF-8"));
output.close();
byte[] bytes1 = baos.toByteArray();
String data2 = "1";
baos = new ByteArrayOutputStream();
output = new GZIPOutputStream(baos);
output.write(data2.getBytes("UTF-8"));
output.close();
byte[] bytes2 = baos.toByteArray();
byte[] bytes = new byte[bytes1.length + bytes2.length];
System.arraycopy(bytes1, 0, bytes, 0, bytes1.length);
System.arraycopy(bytes2, 0, bytes, bytes1.length, bytes2.length);
GZIPContentDecoder decoder = new GZIPContentDecoder();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
ByteBuffer decoded = decoder.decode(buffer);
assertEquals(data1, Charset.forName("UTF-8").decode(decoded).toString());
assertTrue(decoder.finished());
assertTrue(buffer.hasRemaining());
decoded = decoder.decode(buffer);
assertEquals(data2, Charset.forName("UTF-8").decode(decoded).toString());
assertTrue(decoder.finished());
assertFalse(buffer.hasRemaining());
}
@Test
public void testBigBlock() throws Exception
{
String data = "0123456789ABCDEF";
for (int i = 0; i < 10; ++i)
data += data;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data.getBytes("UTF-8"));
output.close();
byte[] bytes = baos.toByteArray();
String result = "";
GZIPContentDecoder decoder = new GZIPContentDecoder();
ByteBuffer buffer = ByteBuffer.wrap(bytes);
while (buffer.hasRemaining())
{
ByteBuffer decoded = decoder.decode(buffer);
result += Charset.forName("UTF-8").decode(decoded).toString();
}
assertEquals(data, result);
}
@Test
public void testBigBlockOneByteAtATime() throws Exception
{
String data = "0123456789ABCDEF";
for (int i = 0; i < 10; ++i)
data += data;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data.getBytes("UTF-8"));
output.close();
byte[] bytes = baos.toByteArray();
String result = "";
GZIPContentDecoder decoder = new GZIPContentDecoder(64);
ByteBuffer buffer = ByteBuffer.wrap(bytes);
while (buffer.hasRemaining())
{
ByteBuffer decoded = decoder.decode(ByteBuffer.wrap(new byte[]{buffer.get()}));
if (decoded.hasRemaining())
result += Charset.forName("UTF-8").decode(decoded).toString();
}
assertEquals(data, result);
assertTrue(decoder.finished());
}
@Test
public void testBigBlockWithExtraBytes() throws Exception
{
String data1 = "0123456789ABCDEF";
for (int i = 0; i < 10; ++i)
data1 += data1;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data1.getBytes("UTF-8"));
output.close();
byte[] bytes1 = baos.toByteArray();
String data2 = "HELLO";
byte[] bytes2 = data2.getBytes("UTF-8");
byte[] bytes = new byte[bytes1.length + bytes2.length];
System.arraycopy(bytes1, 0, bytes, 0, bytes1.length);
System.arraycopy(bytes2, 0, bytes, bytes1.length, bytes2.length);
String result = "";
GZIPContentDecoder decoder = new GZIPContentDecoder(64);
ByteBuffer buffer = ByteBuffer.wrap(bytes);
while (buffer.hasRemaining())
{
ByteBuffer decoded = decoder.decode(buffer);
if (decoded.hasRemaining())
result += Charset.forName("UTF-8").decode(decoded).toString();
if (decoder.finished())
break;
}
assertEquals(data1, result);
assertTrue(buffer.hasRemaining());
assertEquals(data2, Charset.forName("UTF-8").decode(buffer).toString());
}
}

View File

@ -31,6 +31,7 @@ import java.util.NoSuchElementException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.GZIPOutputStream;
import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
@ -484,4 +485,30 @@ public class HttpClientTest extends AbstractHttpClientServerTest
Assert.assertTrue(latch.await(5, TimeUnit.SECONDS));
}
@Test
public void test_GZIP_ContentEncoding() throws Exception
{
final byte[] data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
start(new AbstractHandler()
{
@Override
public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
{
baseRequest.setHandled(true);
response.setHeader("Content-Encoding", "gzip");
GZIPOutputStream gzipOutput = new GZIPOutputStream(response.getOutputStream());
gzipOutput.write(data);
gzipOutput.finish();
}
});
ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
.scheme(scheme)
.send()
.get(5, TimeUnit.SECONDS);
Assert.assertEquals(200, response.status());
Assert.assertArrayEquals(data, response.content());
}
}

View File

@ -18,13 +18,17 @@
package org.eclipse.jetty.client;
import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.nio.ByteBuffer;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.zip.GZIPOutputStream;
import org.eclipse.jetty.client.api.ContentResponse;
import org.eclipse.jetty.client.api.Response;
import org.eclipse.jetty.client.util.BlockingResponseListener;
import org.eclipse.jetty.http.HttpFields;
@ -200,4 +204,43 @@ public class HttpReceiverTest
Assert.assertTrue(e.getCause() instanceof HttpResponseException);
}
}
@Test
public void test_Receive_GZIPResponseContent_Fragmented() throws Exception
{
byte[] data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (GZIPOutputStream gzipOutput = new GZIPOutputStream(baos))
{
gzipOutput.write(data);
}
byte[] gzip = baos.toByteArray();
endPoint.setInput("" +
"HTTP/1.1 200 OK\r\n" +
"Content-Length: " + gzip.length + "\r\n" +
"Content-Encoding: gzip\r\n" +
"\r\n");
BlockingResponseListener listener = new BlockingResponseListener();
HttpExchange exchange = newExchange(listener);
exchange.receive();
endPoint.reset();
ByteBuffer buffer = ByteBuffer.wrap(gzip);
int fragment = buffer.limit() - 1;
buffer.limit(fragment);
endPoint.setInput(buffer);
exchange.receive();
endPoint.reset();
buffer.limit(gzip.length);
buffer.position(fragment);
endPoint.setInput(buffer);
exchange.receive();
ContentResponse response = listener.get(5, TimeUnit.SECONDS);
Assert.assertNotNull(response);
Assert.assertEquals(200, response.status());
Assert.assertArrayEquals(data, response.content());
}
}