From c1f6f6b6082dc9ca8687f1928edf02ac9c79829a Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Tue, 12 Mar 2019 18:05:41 +1100 Subject: [PATCH] Implementation of a request customizer to do gzip inflation Signed-off-by: Greg Wilkins --- .../handler/gzip/GZIPContentDecoder.java | 455 ++++++++++++++++++ .../handler/gzip/GzipRequestCustomizer.java | 145 ++++++ .../jetty/servlet/GzipHandlerTest.java | 165 ++++++- 3 files changed, 764 insertions(+), 1 deletion(-) create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GZIPContentDecoder.java create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GZIPContentDecoder.java b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GZIPContentDecoder.java new file mode 100644 index 00000000000..66a27f39ab9 --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GZIPContentDecoder.java @@ -0,0 +1,455 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server.handler.gzip; + +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; +import java.util.zip.ZipException; + +import org.eclipse.jetty.io.ByteBufferPool; +import org.eclipse.jetty.util.BufferUtil; +import org.eclipse.jetty.util.component.Destroyable; + +/** + *

Decoder for the "gzip" content encoding.

+ *

This decoder inflates gzip compressed data, and has + * been optimized for async usage with minimal data copies.

+ */ +public class GZIPContentDecoder implements Destroyable +{ + private final List _inflateds = new ArrayList<>(); + private final Inflater _inflater = new Inflater(true); + private final ByteBufferPool _pool; + private final int _bufferSize; + private State _state; + private int _size; + private int _value; + private byte _flags; + private ByteBuffer _inflated; + + public GZIPContentDecoder() + { + this(null, 2048); + } + + public GZIPContentDecoder(int bufferSize) + { + this(null, bufferSize); + } + + public GZIPContentDecoder(ByteBufferPool pool, int bufferSize) + { + _bufferSize = bufferSize; + _pool = pool; + reset(); + } + + /** + *

Inflates compressed data from a buffer.

+ *

The buffers returned by this method should be released + * via {@link #release(ByteBuffer)}.

+ *

This method may fully consume the input buffer, but return + * only a chunk of the inflated bytes, to allow applications to + * consume the inflated chunk before performing further inflation, + * applying backpressure. In this case, this method should be + * invoked again with the same input buffer (even if + * it's already fully consumed) and that will produce another + * chunk of inflated bytes. Termination happens when the input + * buffer is fully consumed, and the returned buffer is empty.

+ *

See {@link #decodedChunk(ByteBuffer)} to perform inflating + * in a non-blocking way that allows to apply backpressure.

+ * + * @param compressed the buffer containing compressed data. + * @return a buffer containing inflated data. + */ + public ByteBuffer decode(ByteBuffer compressed) + { + decodeChunks(compressed); + + if (_inflateds.isEmpty()) + { + if (BufferUtil.isEmpty(_inflated) || _state == State.CRC || _state == State.ISIZE) + return BufferUtil.EMPTY_BUFFER; + ByteBuffer result = _inflated; + _inflated = null; + return result; + } + else + { + _inflateds.add(_inflated); + _inflated = null; + int length = _inflateds.stream().mapToInt(Buffer::remaining).sum(); + ByteBuffer result = acquire(length); + for (ByteBuffer buffer : _inflateds) + { + BufferUtil.append(result, buffer); + release(buffer); + } + _inflateds.clear(); + return result; + } + } + + /** + *

Called when a chunk of data is inflated.

+ *

The default implementation aggregates all the chunks + * into a single buffer returned from {@link #decode(ByteBuffer)}.

+ *

Derived implementations may choose to consume inflated chunks + * individually and return {@code true} from this method to prevent + * further inflation until a subsequent call to {@link #decode(ByteBuffer)} + * or {@link #decodeChunks(ByteBuffer)} is made. + * + * @param chunk the inflated chunk of data + * @return false if inflating should continue, or true if the call + * to {@link #decodeChunks(ByteBuffer)} or {@link #decode(ByteBuffer)} + * should return, allowing to consume the inflated chunk and apply + * backpressure + */ + protected boolean decodedChunk(ByteBuffer chunk) + { + if (_inflated == null) + { + _inflated = chunk; + } + else + { + if (BufferUtil.space(_inflated) >= chunk.remaining()) + { + BufferUtil.append(_inflated, chunk); + release(chunk); + } + else + { + _inflateds.add(_inflated); + _inflated = chunk; + } + } + return false; + } + + /** + *

Inflates compressed data.

+ *

Inflation continues until the compressed block end is reached, there is no + * more compressed data or a call to {@link #decodedChunk(ByteBuffer)} returns true.

+ * + * @param compressed the buffer of compressed data to inflate + */ + protected void decodeChunks(ByteBuffer compressed) + { + ByteBuffer buffer = null; + try + { + while (true) + { + switch (_state) + { + case INITIAL: + { + _state = State.ID; + break; + } + + case FLAGS: + { + if ((_flags & 0x04) == 0x04) + { + _state = State.EXTRA_LENGTH; + _size = 0; + _value = 0; + } + else if ((_flags & 0x08) == 0x08) + _state = State.NAME; + else if ((_flags & 0x10) == 0x10) + _state = State.COMMENT; + else if ((_flags & 0x2) == 0x2) + { + _state = State.HCRC; + _size = 0; + _value = 0; + } + else + { + _state = State.DATA; + continue; + } + break; + } + + case DATA: + { + while (true) + { + if (buffer == null) + buffer = acquire(_bufferSize); + + try + { + int length = _inflater.inflate(buffer.array(), buffer.arrayOffset(), buffer.capacity()); + buffer.limit(length); + } + catch (DataFormatException x) + { + throw new ZipException(x.getMessage()); + } + + if (buffer.hasRemaining()) + { + ByteBuffer chunk = buffer; + buffer = null; + if (decodedChunk(chunk)) + return; + } + else if (_inflater.needsInput()) + { + if (!compressed.hasRemaining()) + return; + if (compressed.hasArray()) + { + _inflater.setInput(compressed.array(), compressed.arrayOffset() + compressed.position(), compressed.remaining()); + compressed.position(compressed.limit()); + } + else + { + // TODO use the pool + byte[] input = new byte[compressed.remaining()]; + compressed.get(input); + _inflater.setInput(input); + } + } + else if (_inflater.finished()) + { + int remaining = _inflater.getRemaining(); + compressed.position(compressed.limit() - remaining); + _state = State.CRC; + _size = 0; + _value = 0; + break; + } + } + continue; + } + + default: + break; + } + + if (!compressed.hasRemaining()) + break; + + byte currByte = compressed.get(); + switch (_state) + { + case ID: + { + _value += (currByte & 0xFF) << 8 * _size; + ++_size; + if (_size == 2) + { + if (_value != 0x8B1F) + throw new ZipException("Invalid gzip bytes"); + _state = State.CM; + } + break; + } + case CM: + { + if ((currByte & 0xFF) != 0x08) + throw new ZipException("Invalid gzip compression method"); + _state = State.FLG; + break; + } + case FLG: + { + _flags = currByte; + _state = State.MTIME; + _size = 0; + _value = 0; + break; + } + case MTIME: + { + // Skip the 4 MTIME bytes + ++_size; + if (_size == 4) + _state = State.XFL; + break; + } + case XFL: + { + // Skip XFL + _state = State.OS; + break; + } + case OS: + { + // Skip OS + _state = State.FLAGS; + break; + } + case EXTRA_LENGTH: + { + _value += (currByte & 0xFF) << 8 * _size; + ++_size; + if (_size == 2) + _state = State.EXTRA; + break; + } + case EXTRA: + { + // Skip EXTRA bytes + --_value; + if (_value == 0) + { + // Clear the EXTRA flag and loop on the flags + _flags &= ~0x04; + _state = State.FLAGS; + } + break; + } + case NAME: + { + // Skip NAME bytes + if (currByte == 0) + { + // Clear the NAME flag and loop on the flags + _flags &= ~0x08; + _state = State.FLAGS; + } + break; + } + case COMMENT: + { + // Skip COMMENT bytes + if (currByte == 0) + { + // Clear the COMMENT flag and loop on the flags + _flags &= ~0x10; + _state = State.FLAGS; + } + break; + } + case HCRC: + { + // Skip HCRC + ++_size; + if (_size == 2) + { + // Clear the HCRC flag and loop on the flags + _flags &= ~0x02; + _state = State.FLAGS; + } + break; + } + case CRC: + { + _value += (currByte & 0xFF) << 8 * _size; + ++_size; + if (_size == 4) + { + // From RFC 1952, compliant decoders need not to verify the CRC + _state = State.ISIZE; + _size = 0; + _value = 0; + } + break; + } + case ISIZE: + { + _value += (currByte & 0xFF) << 8 * _size; + ++_size; + if (_size == 4) + { + if (_value != _inflater.getBytesWritten()) + throw new ZipException("Invalid input size"); + + // TODO ByteBuffer result = output == null ? BufferUtil.EMPTY_BUFFER : ByteBuffer.wrap(output); + reset(); + return; + } + break; + } + default: + throw new ZipException(); + } + } + } + catch (ZipException x) + { + throw new RuntimeException(x); + } + finally + { + if (buffer != null) + release(buffer); + } + } + + private void reset() + { + _inflater.reset(); + _state = State.INITIAL; + _size = 0; + _value = 0; + _flags = 0; + } + + @Override + public void destroy() + { + _inflater.end(); + } + + public boolean isFinished() + { + return _state == State.INITIAL; + } + + private enum State + { + INITIAL, ID, CM, FLG, MTIME, XFL, OS, FLAGS, EXTRA_LENGTH, EXTRA, NAME, COMMENT, HCRC, DATA, CRC, ISIZE + } + + /** + * @param capacity capacity of the ByteBuffer to acquire + * @return a heap buffer of the configured capacity either from the pool or freshly allocated. + */ + public ByteBuffer acquire(int capacity) + { + return _pool == null ? BufferUtil.allocate(capacity) : _pool.acquire(capacity, false); + } + + /** + *

Releases an allocated buffer.

+ *

This method calls {@link ByteBufferPool#release(ByteBuffer)} if a buffer pool has + * been configured.

+ *

This method should be called once for all buffers returned from {@link #decode(ByteBuffer)} + * or passed to {@link #decodedChunk(ByteBuffer)}.

+ * + * @param buffer the buffer to release. + */ + public void release(ByteBuffer buffer) + { + if (_pool != null && buffer!=BufferUtil.EMPTY_BUFFER) + _pool.release(buffer); + } +} diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java new file mode 100644 index 00000000000..2b6f2c3abe8 --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java @@ -0,0 +1,145 @@ +package org.eclipse.jetty.server.handler.gzip; + +import java.nio.ByteBuffer; +import java.util.Queue; +import java.util.regex.Pattern; + +import org.eclipse.jetty.http.BadMessageException; +import org.eclipse.jetty.http.HttpField; +import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.io.ArrayByteBufferPool; +import org.eclipse.jetty.io.ByteBufferPool; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpInput; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.util.ArrayQueue; +import org.eclipse.jetty.util.BufferUtil; + +public class GzipRequestCustomizer implements HttpConfiguration.Customizer +{ + public static final String GZIP = "gzip"; + private static final HttpField X_CE_GZIP = new HttpField("X-Content-Encoding","gzip"); + private static final Pattern COMMA_GZIP = Pattern.compile(".*, *gzip"); + + private final ByteBufferPool buffers = new ArrayByteBufferPool(); // TODO Configure + private int compressedBufferSize = 4*1024; // TODO configure + private int inflatedBufferSize = 16*1024; // TODO configure + @Override + public void customize(Connector connector, HttpConfiguration channelConfig, Request request) + { + try + { + HttpFields fields = request.getHttpFields(); + String content_encoding = fields.get(HttpHeader.CONTENT_ENCODING); + if (content_encoding == null) + return; + + if (content_encoding.equalsIgnoreCase("gzip")) + { + fields.remove(HttpHeader.CONTENT_ENCODING); + } + else if (COMMA_GZIP.matcher(content_encoding).matches()) + { + fields.remove(HttpHeader.CONTENT_ENCODING); + fields.add(HttpHeader.CONTENT_ENCODING, content_encoding.substring(0, content_encoding.lastIndexOf(','))); + } + else + { + return; + } + + fields.add(X_CE_GZIP); + + // Read all the compressed content into a queue of buffers + final HttpInput input = request.getHttpInput(); + Queue compressed = new ArrayQueue<>(); + ByteBuffer buffer = null; + while (true) + { + if (buffer==null || BufferUtil.isFull(buffer)) + { + buffer = buffers.acquire(compressedBufferSize,false); + compressed.add(buffer); + } + int l = input.read(buffer.array(), buffer.arrayOffset()+buffer.limit(), BufferUtil.space(buffer)); + if (l<0) + break; + buffer.limit(buffer.limit()+l); + } + input.recycle(); + + + // Handle no content + if (compressed.size()==1 && BufferUtil.isEmpty(buffer)) + { + input.eof(); + return; + } + + // TODO Perhaps pool docoders/inflators? + GZIPContentDecoder decoder = new GZIPContentDecoder(buffers, inflatedBufferSize) + { + @Override + protected boolean decodedChunk(ByteBuffer chunk) + { + super.decodedChunk(chunk); + return false; + } + }; + + input.addContent(new InflatingContent(input, decoder,compressed)); + + } + catch(Throwable t) + { + throw new BadMessageException(400,"Bad compressed request",t); + } + } + + private ByteBuffer inflate(GZIPContentDecoder decoder, Queue compressed) + { + while (!compressed.isEmpty() && BufferUtil.isEmpty(compressed.peek())) + buffers.release(compressed.poll()); + + if (compressed.isEmpty()) + return BufferUtil.EMPTY_BUFFER; + + ByteBuffer inflated = decoder.decode(compressed.peek()); + System.err.println(BufferUtil.toDetailString(inflated)); + return inflated; + } + + + private class InflatingContent extends HttpInput.Content + { + final HttpInput input; + final GZIPContentDecoder decoder; + final Queue compressed; + + public InflatingContent(HttpInput input, GZIPContentDecoder decoder, Queue compressed) + { + super(inflate(decoder,compressed)); + this.input = input; + this.decoder = decoder; + this.compressed = compressed; + } + + @Override + public void succeeded() + { + if (decoder.isFinished() && compressed.isEmpty()) + input.eof(); + else + input.addContent(new InflatingContent(input,decoder,compressed)); + } + + @Override + public void failed(Throwable x) + { + input.failed(x); + } + } + +} diff --git a/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java b/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java index 247956ccf28..2ad094c9210 100644 --- a/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java +++ b/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java @@ -33,8 +33,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Enumeration; import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; @@ -42,10 +45,13 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.eclipse.jetty.http.HttpTester; +import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.LocalConnector; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.gzip.GzipHandler; +import org.eclipse.jetty.server.handler.gzip.GzipRequestCustomizer; import org.eclipse.jetty.util.IO; +import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; @@ -82,6 +88,7 @@ public class GzipHandlerTest _server = new Server(); _connector = new LocalConnector(_server); _server.addConnector(_connector); + _connector.getConnectionFactory(HttpConnectionFactory.class).getHttpConfiguration().addCustomizer(new GzipRequestCustomizer()); GzipHandler gzipHandler = new GzipHandler(); gzipHandler.setExcludedAgentPatterns(); @@ -97,7 +104,10 @@ public class GzipHandlerTest servlets.addServletWithMapping(TestServlet.class,"/content"); servlets.addServletWithMapping(ForwardServlet.class,"/forward"); servlets.addServletWithMapping(IncludeServlet.class,"/include"); - + servlets.addServletWithMapping(EchoServlet.class,"/echo/*"); + servlets.addServletWithMapping(DumpServlet.class,"/dump/*"); + + _server.start(); } @@ -148,6 +158,34 @@ public class GzipHandlerTest } } + public static class EchoServlet extends HttpServlet + { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse response) throws ServletException, IOException + { + response.setContentType(req.getContentType()); + IO.copy(req.getInputStream(),response.getOutputStream()); + } + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse response) throws ServletException, IOException + { + doGet(req,response); + } + } + public static class DumpServlet extends HttpServlet + { + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse response) throws ServletException, IOException + { + response.setContentType("text/plain"); + for (Enumeration e = req.getParameterNames(); e.hasMoreElements(); ) + { + String n = e.nextElement(); + response.getWriter().printf("%s: %s\n",n,req.getParameter(n)); + } + } + } + public static class ForwardServlet extends HttpServlet { @Override @@ -392,4 +430,129 @@ public class GzipHandlerTest assertThat("Included Paths.size", includedPaths.length, is(2)); assertThat("Included Paths", Arrays.asList(includedPaths), contains("/foo","^/bar.*$")); } + + + + @Test + public void testGzipRequest() throws Exception + { + String data = "Hello Nice World! "; + for (int i = 0; i < 10; ++i) + data += data; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes(StandardCharsets.UTF_8)); + output.close(); + byte[] bytes = baos.toByteArray(); + + // generated and parsed test + HttpTester.Request request = HttpTester.newRequest(); + HttpTester.Response response; + + request.setMethod("POST"); + request.setURI("/ctx/echo"); + request.setVersion("HTTP/1.0"); + request.setHeader("Host","tester"); + request.setHeader("Content-Type","text/plain"); + request.setHeader("Content-Encoding","gzip"); + request.setContent(bytes); + + response = HttpTester.parseResponse(_connector.getResponse(request.generate())); + + MatcherAssert.assertThat(response.getStatus(),is(200)); + MatcherAssert.assertThat(response.getContent(),is(data)); + + } + + + @Test + public void testGzipRequestChunked() throws Exception + { + String data = "Hello Nice World! "; + for (int i = 0; i < 10; ++i) + data += data; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes(StandardCharsets.UTF_8)); + output.close(); + byte[] bytes = baos.toByteArray(); + + // generated and parsed test + HttpTester.Request request = HttpTester.newRequest(); + HttpTester.Response response; + + request.setMethod("POST"); + request.setURI("/ctx/echo"); + request.setVersion("HTTP/1.1"); + request.setHeader("Host","tester"); + request.setHeader("Content-Type","text/plain"); + request.setHeader("Content-Encoding","gzip"); + request.add("Transfer-Encoding", "chunked"); + request.setContent(bytes); + response = HttpTester.parseResponse(_connector.getResponse(request.generate())); + + MatcherAssert.assertThat(response.getStatus(),is(200)); + MatcherAssert.assertThat(response.getContent(),is(data)); + + } + + + @Test + public void testGzipFormRequest() throws Exception + { + String data = "name=value"; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes(StandardCharsets.UTF_8)); + output.close(); + byte[] bytes = baos.toByteArray(); + + // generated and parsed test + HttpTester.Request request = HttpTester.newRequest(); + HttpTester.Response response; + + request.setMethod("POST"); + request.setURI("/ctx/dump"); + request.setVersion("HTTP/1.0"); + request.setHeader("Host","tester"); + request.setHeader("Content-Type","application/x-www-form-urlencoded; charset=utf-8"); + request.setHeader("Content-Encoding","gzip"); + request.setContent(bytes); + + response = HttpTester.parseResponse(_connector.getResponse(request.generate())); + + MatcherAssert.assertThat(response.getStatus(),is(200)); + MatcherAssert.assertThat(response.getContent(),is("name: value\n")); + } + + @Test + public void testGzipBomb() throws Exception + { + byte[] data = new byte[512*1024]; + Arrays.fill(data,(byte)'X'); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data); + output.close(); + byte[] bytes = baos.toByteArray(); + + // generated and parsed test + HttpTester.Request request = HttpTester.newRequest(); + HttpTester.Response response; + + request.setMethod("POST"); + request.setURI("/ctx/echo"); + request.setVersion("HTTP/1.0"); + request.setHeader("Host","tester"); + request.setHeader("Content-Type","text/plain"); + request.setHeader("Content-Encoding","gzip"); + request.setContent(bytes); + + response = HttpTester.parseResponse(_connector.getResponse(request.generate())); + // TODO need to test back pressure works + + MatcherAssert.assertThat(response.getStatus(),is(200)); + MatcherAssert.assertThat(response.getContentBytes().length,is(512*1024)); + } }