From c1f6f6b6082dc9ca8687f1928edf02ac9c79829a Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Tue, 12 Mar 2019 18:05:41 +1100 Subject: [PATCH 1/2] Implementation of a request customizer to do gzip inflation Signed-off-by: Greg Wilkins --- .../handler/gzip/GZIPContentDecoder.java | 455 ++++++++++++++++++ .../handler/gzip/GzipRequestCustomizer.java | 145 ++++++ .../jetty/servlet/GzipHandlerTest.java | 165 ++++++- 3 files changed, 764 insertions(+), 1 deletion(-) create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GZIPContentDecoder.java create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GZIPContentDecoder.java b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GZIPContentDecoder.java new file mode 100644 index 00000000000..66a27f39ab9 --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GZIPContentDecoder.java @@ -0,0 +1,455 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server.handler.gzip; + +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; +import java.util.zip.ZipException; + +import org.eclipse.jetty.io.ByteBufferPool; +import org.eclipse.jetty.util.BufferUtil; +import org.eclipse.jetty.util.component.Destroyable; + +/** + *

Decoder for the "gzip" content encoding.

+ *

This decoder inflates gzip compressed data, and has + * been optimized for async usage with minimal data copies.

+ */ +public class GZIPContentDecoder implements Destroyable +{ + private final List _inflateds = new ArrayList<>(); + private final Inflater _inflater = new Inflater(true); + private final ByteBufferPool _pool; + private final int _bufferSize; + private State _state; + private int _size; + private int _value; + private byte _flags; + private ByteBuffer _inflated; + + public GZIPContentDecoder() + { + this(null, 2048); + } + + public GZIPContentDecoder(int bufferSize) + { + this(null, bufferSize); + } + + public GZIPContentDecoder(ByteBufferPool pool, int bufferSize) + { + _bufferSize = bufferSize; + _pool = pool; + reset(); + } + + /** + *

Inflates compressed data from a buffer.

+ *

The buffers returned by this method should be released + * via {@link #release(ByteBuffer)}.

+ *

This method may fully consume the input buffer, but return + * only a chunk of the inflated bytes, to allow applications to + * consume the inflated chunk before performing further inflation, + * applying backpressure. In this case, this method should be + * invoked again with the same input buffer (even if + * it's already fully consumed) and that will produce another + * chunk of inflated bytes. Termination happens when the input + * buffer is fully consumed, and the returned buffer is empty.

+ *

See {@link #decodedChunk(ByteBuffer)} to perform inflating + * in a non-blocking way that allows to apply backpressure.

+ * + * @param compressed the buffer containing compressed data. + * @return a buffer containing inflated data. + */ + public ByteBuffer decode(ByteBuffer compressed) + { + decodeChunks(compressed); + + if (_inflateds.isEmpty()) + { + if (BufferUtil.isEmpty(_inflated) || _state == State.CRC || _state == State.ISIZE) + return BufferUtil.EMPTY_BUFFER; + ByteBuffer result = _inflated; + _inflated = null; + return result; + } + else + { + _inflateds.add(_inflated); + _inflated = null; + int length = _inflateds.stream().mapToInt(Buffer::remaining).sum(); + ByteBuffer result = acquire(length); + for (ByteBuffer buffer : _inflateds) + { + BufferUtil.append(result, buffer); + release(buffer); + } + _inflateds.clear(); + return result; + } + } + + /** + *

Called when a chunk of data is inflated.

+ *

The default implementation aggregates all the chunks + * into a single buffer returned from {@link #decode(ByteBuffer)}.

+ *

Derived implementations may choose to consume inflated chunks + * individually and return {@code true} from this method to prevent + * further inflation until a subsequent call to {@link #decode(ByteBuffer)} + * or {@link #decodeChunks(ByteBuffer)} is made. + * + * @param chunk the inflated chunk of data + * @return false if inflating should continue, or true if the call + * to {@link #decodeChunks(ByteBuffer)} or {@link #decode(ByteBuffer)} + * should return, allowing to consume the inflated chunk and apply + * backpressure + */ + protected boolean decodedChunk(ByteBuffer chunk) + { + if (_inflated == null) + { + _inflated = chunk; + } + else + { + if (BufferUtil.space(_inflated) >= chunk.remaining()) + { + BufferUtil.append(_inflated, chunk); + release(chunk); + } + else + { + _inflateds.add(_inflated); + _inflated = chunk; + } + } + return false; + } + + /** + *

Inflates compressed data.

+ *

Inflation continues until the compressed block end is reached, there is no + * more compressed data or a call to {@link #decodedChunk(ByteBuffer)} returns true.

+ * + * @param compressed the buffer of compressed data to inflate + */ + protected void decodeChunks(ByteBuffer compressed) + { + ByteBuffer buffer = null; + try + { + while (true) + { + switch (_state) + { + case INITIAL: + { + _state = State.ID; + break; + } + + case FLAGS: + { + if ((_flags & 0x04) == 0x04) + { + _state = State.EXTRA_LENGTH; + _size = 0; + _value = 0; + } + else if ((_flags & 0x08) == 0x08) + _state = State.NAME; + else if ((_flags & 0x10) == 0x10) + _state = State.COMMENT; + else if ((_flags & 0x2) == 0x2) + { + _state = State.HCRC; + _size = 0; + _value = 0; + } + else + { + _state = State.DATA; + continue; + } + break; + } + + case DATA: + { + while (true) + { + if (buffer == null) + buffer = acquire(_bufferSize); + + try + { + int length = _inflater.inflate(buffer.array(), buffer.arrayOffset(), buffer.capacity()); + buffer.limit(length); + } + catch (DataFormatException x) + { + throw new ZipException(x.getMessage()); + } + + if (buffer.hasRemaining()) + { + ByteBuffer chunk = buffer; + buffer = null; + if (decodedChunk(chunk)) + return; + } + else if (_inflater.needsInput()) + { + if (!compressed.hasRemaining()) + return; + if (compressed.hasArray()) + { + _inflater.setInput(compressed.array(), compressed.arrayOffset() + compressed.position(), compressed.remaining()); + compressed.position(compressed.limit()); + } + else + { + // TODO use the pool + byte[] input = new byte[compressed.remaining()]; + compressed.get(input); + _inflater.setInput(input); + } + } + else if (_inflater.finished()) + { + int remaining = _inflater.getRemaining(); + compressed.position(compressed.limit() - remaining); + _state = State.CRC; + _size = 0; + _value = 0; + break; + } + } + continue; + } + + default: + break; + } + + if (!compressed.hasRemaining()) + break; + + byte currByte = compressed.get(); + switch (_state) + { + case ID: + { + _value += (currByte & 0xFF) << 8 * _size; + ++_size; + if (_size == 2) + { + if (_value != 0x8B1F) + throw new ZipException("Invalid gzip bytes"); + _state = State.CM; + } + break; + } + case CM: + { + if ((currByte & 0xFF) != 0x08) + throw new ZipException("Invalid gzip compression method"); + _state = State.FLG; + break; + } + case FLG: + { + _flags = currByte; + _state = State.MTIME; + _size = 0; + _value = 0; + break; + } + case MTIME: + { + // Skip the 4 MTIME bytes + ++_size; + if (_size == 4) + _state = State.XFL; + break; + } + case XFL: + { + // Skip XFL + _state = State.OS; + break; + } + case OS: + { + // Skip OS + _state = State.FLAGS; + break; + } + case EXTRA_LENGTH: + { + _value += (currByte & 0xFF) << 8 * _size; + ++_size; + if (_size == 2) + _state = State.EXTRA; + break; + } + case EXTRA: + { + // Skip EXTRA bytes + --_value; + if (_value == 0) + { + // Clear the EXTRA flag and loop on the flags + _flags &= ~0x04; + _state = State.FLAGS; + } + break; + } + case NAME: + { + // Skip NAME bytes + if (currByte == 0) + { + // Clear the NAME flag and loop on the flags + _flags &= ~0x08; + _state = State.FLAGS; + } + break; + } + case COMMENT: + { + // Skip COMMENT bytes + if (currByte == 0) + { + // Clear the COMMENT flag and loop on the flags + _flags &= ~0x10; + _state = State.FLAGS; + } + break; + } + case HCRC: + { + // Skip HCRC + ++_size; + if (_size == 2) + { + // Clear the HCRC flag and loop on the flags + _flags &= ~0x02; + _state = State.FLAGS; + } + break; + } + case CRC: + { + _value += (currByte & 0xFF) << 8 * _size; + ++_size; + if (_size == 4) + { + // From RFC 1952, compliant decoders need not to verify the CRC + _state = State.ISIZE; + _size = 0; + _value = 0; + } + break; + } + case ISIZE: + { + _value += (currByte & 0xFF) << 8 * _size; + ++_size; + if (_size == 4) + { + if (_value != _inflater.getBytesWritten()) + throw new ZipException("Invalid input size"); + + // TODO ByteBuffer result = output == null ? BufferUtil.EMPTY_BUFFER : ByteBuffer.wrap(output); + reset(); + return; + } + break; + } + default: + throw new ZipException(); + } + } + } + catch (ZipException x) + { + throw new RuntimeException(x); + } + finally + { + if (buffer != null) + release(buffer); + } + } + + private void reset() + { + _inflater.reset(); + _state = State.INITIAL; + _size = 0; + _value = 0; + _flags = 0; + } + + @Override + public void destroy() + { + _inflater.end(); + } + + public boolean isFinished() + { + return _state == State.INITIAL; + } + + private enum State + { + INITIAL, ID, CM, FLG, MTIME, XFL, OS, FLAGS, EXTRA_LENGTH, EXTRA, NAME, COMMENT, HCRC, DATA, CRC, ISIZE + } + + /** + * @param capacity capacity of the ByteBuffer to acquire + * @return a heap buffer of the configured capacity either from the pool or freshly allocated. + */ + public ByteBuffer acquire(int capacity) + { + return _pool == null ? BufferUtil.allocate(capacity) : _pool.acquire(capacity, false); + } + + /** + *

Releases an allocated buffer.

+ *

This method calls {@link ByteBufferPool#release(ByteBuffer)} if a buffer pool has + * been configured.

+ *

This method should be called once for all buffers returned from {@link #decode(ByteBuffer)} + * or passed to {@link #decodedChunk(ByteBuffer)}.

+ * + * @param buffer the buffer to release. + */ + public void release(ByteBuffer buffer) + { + if (_pool != null && buffer!=BufferUtil.EMPTY_BUFFER) + _pool.release(buffer); + } +} diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java new file mode 100644 index 00000000000..2b6f2c3abe8 --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java @@ -0,0 +1,145 @@ +package org.eclipse.jetty.server.handler.gzip; + +import java.nio.ByteBuffer; +import java.util.Queue; +import java.util.regex.Pattern; + +import org.eclipse.jetty.http.BadMessageException; +import org.eclipse.jetty.http.HttpField; +import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.io.ArrayByteBufferPool; +import org.eclipse.jetty.io.ByteBufferPool; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpInput; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.util.ArrayQueue; +import org.eclipse.jetty.util.BufferUtil; + +public class GzipRequestCustomizer implements HttpConfiguration.Customizer +{ + public static final String GZIP = "gzip"; + private static final HttpField X_CE_GZIP = new HttpField("X-Content-Encoding","gzip"); + private static final Pattern COMMA_GZIP = Pattern.compile(".*, *gzip"); + + private final ByteBufferPool buffers = new ArrayByteBufferPool(); // TODO Configure + private int compressedBufferSize = 4*1024; // TODO configure + private int inflatedBufferSize = 16*1024; // TODO configure + @Override + public void customize(Connector connector, HttpConfiguration channelConfig, Request request) + { + try + { + HttpFields fields = request.getHttpFields(); + String content_encoding = fields.get(HttpHeader.CONTENT_ENCODING); + if (content_encoding == null) + return; + + if (content_encoding.equalsIgnoreCase("gzip")) + { + fields.remove(HttpHeader.CONTENT_ENCODING); + } + else if (COMMA_GZIP.matcher(content_encoding).matches()) + { + fields.remove(HttpHeader.CONTENT_ENCODING); + fields.add(HttpHeader.CONTENT_ENCODING, content_encoding.substring(0, content_encoding.lastIndexOf(','))); + } + else + { + return; + } + + fields.add(X_CE_GZIP); + + // Read all the compressed content into a queue of buffers + final HttpInput input = request.getHttpInput(); + Queue compressed = new ArrayQueue<>(); + ByteBuffer buffer = null; + while (true) + { + if (buffer==null || BufferUtil.isFull(buffer)) + { + buffer = buffers.acquire(compressedBufferSize,false); + compressed.add(buffer); + } + int l = input.read(buffer.array(), buffer.arrayOffset()+buffer.limit(), BufferUtil.space(buffer)); + if (l<0) + break; + buffer.limit(buffer.limit()+l); + } + input.recycle(); + + + // Handle no content + if (compressed.size()==1 && BufferUtil.isEmpty(buffer)) + { + input.eof(); + return; + } + + // TODO Perhaps pool docoders/inflators? + GZIPContentDecoder decoder = new GZIPContentDecoder(buffers, inflatedBufferSize) + { + @Override + protected boolean decodedChunk(ByteBuffer chunk) + { + super.decodedChunk(chunk); + return false; + } + }; + + input.addContent(new InflatingContent(input, decoder,compressed)); + + } + catch(Throwable t) + { + throw new BadMessageException(400,"Bad compressed request",t); + } + } + + private ByteBuffer inflate(GZIPContentDecoder decoder, Queue compressed) + { + while (!compressed.isEmpty() && BufferUtil.isEmpty(compressed.peek())) + buffers.release(compressed.poll()); + + if (compressed.isEmpty()) + return BufferUtil.EMPTY_BUFFER; + + ByteBuffer inflated = decoder.decode(compressed.peek()); + System.err.println(BufferUtil.toDetailString(inflated)); + return inflated; + } + + + private class InflatingContent extends HttpInput.Content + { + final HttpInput input; + final GZIPContentDecoder decoder; + final Queue compressed; + + public InflatingContent(HttpInput input, GZIPContentDecoder decoder, Queue compressed) + { + super(inflate(decoder,compressed)); + this.input = input; + this.decoder = decoder; + this.compressed = compressed; + } + + @Override + public void succeeded() + { + if (decoder.isFinished() && compressed.isEmpty()) + input.eof(); + else + input.addContent(new InflatingContent(input,decoder,compressed)); + } + + @Override + public void failed(Throwable x) + { + input.failed(x); + } + } + +} diff --git a/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java b/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java index 247956ccf28..2ad094c9210 100644 --- a/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java +++ b/jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java @@ -33,8 +33,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Enumeration; import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; @@ -42,10 +45,13 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.eclipse.jetty.http.HttpTester; +import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.LocalConnector; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.gzip.GzipHandler; +import org.eclipse.jetty.server.handler.gzip.GzipRequestCustomizer; import org.eclipse.jetty.util.IO; +import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; @@ -82,6 +88,7 @@ public class GzipHandlerTest _server = new Server(); _connector = new LocalConnector(_server); _server.addConnector(_connector); + _connector.getConnectionFactory(HttpConnectionFactory.class).getHttpConfiguration().addCustomizer(new GzipRequestCustomizer()); GzipHandler gzipHandler = new GzipHandler(); gzipHandler.setExcludedAgentPatterns(); @@ -97,7 +104,10 @@ public class GzipHandlerTest servlets.addServletWithMapping(TestServlet.class,"/content"); servlets.addServletWithMapping(ForwardServlet.class,"/forward"); servlets.addServletWithMapping(IncludeServlet.class,"/include"); - + servlets.addServletWithMapping(EchoServlet.class,"/echo/*"); + servlets.addServletWithMapping(DumpServlet.class,"/dump/*"); + + _server.start(); } @@ -148,6 +158,34 @@ public class GzipHandlerTest } } + public static class EchoServlet extends HttpServlet + { + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse response) throws ServletException, IOException + { + response.setContentType(req.getContentType()); + IO.copy(req.getInputStream(),response.getOutputStream()); + } + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse response) throws ServletException, IOException + { + doGet(req,response); + } + } + public static class DumpServlet extends HttpServlet + { + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse response) throws ServletException, IOException + { + response.setContentType("text/plain"); + for (Enumeration e = req.getParameterNames(); e.hasMoreElements(); ) + { + String n = e.nextElement(); + response.getWriter().printf("%s: %s\n",n,req.getParameter(n)); + } + } + } + public static class ForwardServlet extends HttpServlet { @Override @@ -392,4 +430,129 @@ public class GzipHandlerTest assertThat("Included Paths.size", includedPaths.length, is(2)); assertThat("Included Paths", Arrays.asList(includedPaths), contains("/foo","^/bar.*$")); } + + + + @Test + public void testGzipRequest() throws Exception + { + String data = "Hello Nice World! "; + for (int i = 0; i < 10; ++i) + data += data; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes(StandardCharsets.UTF_8)); + output.close(); + byte[] bytes = baos.toByteArray(); + + // generated and parsed test + HttpTester.Request request = HttpTester.newRequest(); + HttpTester.Response response; + + request.setMethod("POST"); + request.setURI("/ctx/echo"); + request.setVersion("HTTP/1.0"); + request.setHeader("Host","tester"); + request.setHeader("Content-Type","text/plain"); + request.setHeader("Content-Encoding","gzip"); + request.setContent(bytes); + + response = HttpTester.parseResponse(_connector.getResponse(request.generate())); + + MatcherAssert.assertThat(response.getStatus(),is(200)); + MatcherAssert.assertThat(response.getContent(),is(data)); + + } + + + @Test + public void testGzipRequestChunked() throws Exception + { + String data = "Hello Nice World! "; + for (int i = 0; i < 10; ++i) + data += data; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes(StandardCharsets.UTF_8)); + output.close(); + byte[] bytes = baos.toByteArray(); + + // generated and parsed test + HttpTester.Request request = HttpTester.newRequest(); + HttpTester.Response response; + + request.setMethod("POST"); + request.setURI("/ctx/echo"); + request.setVersion("HTTP/1.1"); + request.setHeader("Host","tester"); + request.setHeader("Content-Type","text/plain"); + request.setHeader("Content-Encoding","gzip"); + request.add("Transfer-Encoding", "chunked"); + request.setContent(bytes); + response = HttpTester.parseResponse(_connector.getResponse(request.generate())); + + MatcherAssert.assertThat(response.getStatus(),is(200)); + MatcherAssert.assertThat(response.getContent(),is(data)); + + } + + + @Test + public void testGzipFormRequest() throws Exception + { + String data = "name=value"; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data.getBytes(StandardCharsets.UTF_8)); + output.close(); + byte[] bytes = baos.toByteArray(); + + // generated and parsed test + HttpTester.Request request = HttpTester.newRequest(); + HttpTester.Response response; + + request.setMethod("POST"); + request.setURI("/ctx/dump"); + request.setVersion("HTTP/1.0"); + request.setHeader("Host","tester"); + request.setHeader("Content-Type","application/x-www-form-urlencoded; charset=utf-8"); + request.setHeader("Content-Encoding","gzip"); + request.setContent(bytes); + + response = HttpTester.parseResponse(_connector.getResponse(request.generate())); + + MatcherAssert.assertThat(response.getStatus(),is(200)); + MatcherAssert.assertThat(response.getContent(),is("name: value\n")); + } + + @Test + public void testGzipBomb() throws Exception + { + byte[] data = new byte[512*1024]; + Arrays.fill(data,(byte)'X'); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream output = new GZIPOutputStream(baos); + output.write(data); + output.close(); + byte[] bytes = baos.toByteArray(); + + // generated and parsed test + HttpTester.Request request = HttpTester.newRequest(); + HttpTester.Response response; + + request.setMethod("POST"); + request.setURI("/ctx/echo"); + request.setVersion("HTTP/1.0"); + request.setHeader("Host","tester"); + request.setHeader("Content-Type","text/plain"); + request.setHeader("Content-Encoding","gzip"); + request.setContent(bytes); + + response = HttpTester.parseResponse(_connector.getResponse(request.generate())); + // TODO need to test back pressure works + + MatcherAssert.assertThat(response.getStatus(),is(200)); + MatcherAssert.assertThat(response.getContentBytes().length,is(512*1024)); + } } From 6db357d2fcdb1a8fe1d084afd03bf1958c06b211 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Wed, 13 Mar 2019 13:30:54 +1100 Subject: [PATCH 2/2] Completed implementation of GzipRequestCustomizer --- .../main/config/etc/jetty-gzip-inflate.xml | 12 + .../src/main/config/modules/gzip-inflate.mod | 17 + .../handler/gzip/GZIPContentDecoder.java | 455 ------------------ .../handler/gzip/GzipRequestCustomizer.java | 351 ++++++++++++-- 4 files changed, 337 insertions(+), 498 deletions(-) create mode 100644 jetty-server/src/main/config/etc/jetty-gzip-inflate.xml create mode 100644 jetty-server/src/main/config/modules/gzip-inflate.mod delete mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GZIPContentDecoder.java diff --git a/jetty-server/src/main/config/etc/jetty-gzip-inflate.xml b/jetty-server/src/main/config/etc/jetty-gzip-inflate.xml new file mode 100644 index 00000000000..8b1eec30df5 --- /dev/null +++ b/jetty-server/src/main/config/etc/jetty-gzip-inflate.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/jetty-server/src/main/config/modules/gzip-inflate.mod b/jetty-server/src/main/config/modules/gzip-inflate.mod new file mode 100644 index 00000000000..e9873a655d2 --- /dev/null +++ b/jetty-server/src/main/config/modules/gzip-inflate.mod @@ -0,0 +1,17 @@ +# +# GZIP inflate module +# Applies GzipRequestCustomizer to entire server to inflate gzipped requests +# + +[depend] +server + +[xml] +etc/jetty-gzip-inflate.xml + +[ini-template] +## Buffer size for compressed data +# jetty.gzip.inflate.compressedBufferSize=4096 + +## Buffer size for compressed data +# jetty.gzip.inflate.inflatedBufferSize=16384 diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GZIPContentDecoder.java b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GZIPContentDecoder.java deleted file mode 100644 index 66a27f39ab9..00000000000 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GZIPContentDecoder.java +++ /dev/null @@ -1,455 +0,0 @@ -// -// ======================================================================== -// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. -// ------------------------------------------------------------------------ -// All rights reserved. This program and the accompanying materials -// are made available under the terms of the Eclipse Public License v1.0 -// and Apache License v2.0 which accompanies this distribution. -// -// The Eclipse Public License is available at -// http://www.eclipse.org/legal/epl-v10.html -// -// The Apache License v2.0 is available at -// http://www.opensource.org/licenses/apache2.0.php -// -// You may elect to redistribute this code under either of these licenses. -// ======================================================================== -// - -package org.eclipse.jetty.server.handler.gzip; - -import java.nio.Buffer; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.zip.DataFormatException; -import java.util.zip.Inflater; -import java.util.zip.ZipException; - -import org.eclipse.jetty.io.ByteBufferPool; -import org.eclipse.jetty.util.BufferUtil; -import org.eclipse.jetty.util.component.Destroyable; - -/** - *

Decoder for the "gzip" content encoding.

- *

This decoder inflates gzip compressed data, and has - * been optimized for async usage with minimal data copies.

- */ -public class GZIPContentDecoder implements Destroyable -{ - private final List _inflateds = new ArrayList<>(); - private final Inflater _inflater = new Inflater(true); - private final ByteBufferPool _pool; - private final int _bufferSize; - private State _state; - private int _size; - private int _value; - private byte _flags; - private ByteBuffer _inflated; - - public GZIPContentDecoder() - { - this(null, 2048); - } - - public GZIPContentDecoder(int bufferSize) - { - this(null, bufferSize); - } - - public GZIPContentDecoder(ByteBufferPool pool, int bufferSize) - { - _bufferSize = bufferSize; - _pool = pool; - reset(); - } - - /** - *

Inflates compressed data from a buffer.

- *

The buffers returned by this method should be released - * via {@link #release(ByteBuffer)}.

- *

This method may fully consume the input buffer, but return - * only a chunk of the inflated bytes, to allow applications to - * consume the inflated chunk before performing further inflation, - * applying backpressure. In this case, this method should be - * invoked again with the same input buffer (even if - * it's already fully consumed) and that will produce another - * chunk of inflated bytes. Termination happens when the input - * buffer is fully consumed, and the returned buffer is empty.

- *

See {@link #decodedChunk(ByteBuffer)} to perform inflating - * in a non-blocking way that allows to apply backpressure.

- * - * @param compressed the buffer containing compressed data. - * @return a buffer containing inflated data. - */ - public ByteBuffer decode(ByteBuffer compressed) - { - decodeChunks(compressed); - - if (_inflateds.isEmpty()) - { - if (BufferUtil.isEmpty(_inflated) || _state == State.CRC || _state == State.ISIZE) - return BufferUtil.EMPTY_BUFFER; - ByteBuffer result = _inflated; - _inflated = null; - return result; - } - else - { - _inflateds.add(_inflated); - _inflated = null; - int length = _inflateds.stream().mapToInt(Buffer::remaining).sum(); - ByteBuffer result = acquire(length); - for (ByteBuffer buffer : _inflateds) - { - BufferUtil.append(result, buffer); - release(buffer); - } - _inflateds.clear(); - return result; - } - } - - /** - *

Called when a chunk of data is inflated.

- *

The default implementation aggregates all the chunks - * into a single buffer returned from {@link #decode(ByteBuffer)}.

- *

Derived implementations may choose to consume inflated chunks - * individually and return {@code true} from this method to prevent - * further inflation until a subsequent call to {@link #decode(ByteBuffer)} - * or {@link #decodeChunks(ByteBuffer)} is made. - * - * @param chunk the inflated chunk of data - * @return false if inflating should continue, or true if the call - * to {@link #decodeChunks(ByteBuffer)} or {@link #decode(ByteBuffer)} - * should return, allowing to consume the inflated chunk and apply - * backpressure - */ - protected boolean decodedChunk(ByteBuffer chunk) - { - if (_inflated == null) - { - _inflated = chunk; - } - else - { - if (BufferUtil.space(_inflated) >= chunk.remaining()) - { - BufferUtil.append(_inflated, chunk); - release(chunk); - } - else - { - _inflateds.add(_inflated); - _inflated = chunk; - } - } - return false; - } - - /** - *

Inflates compressed data.

- *

Inflation continues until the compressed block end is reached, there is no - * more compressed data or a call to {@link #decodedChunk(ByteBuffer)} returns true.

- * - * @param compressed the buffer of compressed data to inflate - */ - protected void decodeChunks(ByteBuffer compressed) - { - ByteBuffer buffer = null; - try - { - while (true) - { - switch (_state) - { - case INITIAL: - { - _state = State.ID; - break; - } - - case FLAGS: - { - if ((_flags & 0x04) == 0x04) - { - _state = State.EXTRA_LENGTH; - _size = 0; - _value = 0; - } - else if ((_flags & 0x08) == 0x08) - _state = State.NAME; - else if ((_flags & 0x10) == 0x10) - _state = State.COMMENT; - else if ((_flags & 0x2) == 0x2) - { - _state = State.HCRC; - _size = 0; - _value = 0; - } - else - { - _state = State.DATA; - continue; - } - break; - } - - case DATA: - { - while (true) - { - if (buffer == null) - buffer = acquire(_bufferSize); - - try - { - int length = _inflater.inflate(buffer.array(), buffer.arrayOffset(), buffer.capacity()); - buffer.limit(length); - } - catch (DataFormatException x) - { - throw new ZipException(x.getMessage()); - } - - if (buffer.hasRemaining()) - { - ByteBuffer chunk = buffer; - buffer = null; - if (decodedChunk(chunk)) - return; - } - else if (_inflater.needsInput()) - { - if (!compressed.hasRemaining()) - return; - if (compressed.hasArray()) - { - _inflater.setInput(compressed.array(), compressed.arrayOffset() + compressed.position(), compressed.remaining()); - compressed.position(compressed.limit()); - } - else - { - // TODO use the pool - byte[] input = new byte[compressed.remaining()]; - compressed.get(input); - _inflater.setInput(input); - } - } - else if (_inflater.finished()) - { - int remaining = _inflater.getRemaining(); - compressed.position(compressed.limit() - remaining); - _state = State.CRC; - _size = 0; - _value = 0; - break; - } - } - continue; - } - - default: - break; - } - - if (!compressed.hasRemaining()) - break; - - byte currByte = compressed.get(); - switch (_state) - { - case ID: - { - _value += (currByte & 0xFF) << 8 * _size; - ++_size; - if (_size == 2) - { - if (_value != 0x8B1F) - throw new ZipException("Invalid gzip bytes"); - _state = State.CM; - } - break; - } - case CM: - { - if ((currByte & 0xFF) != 0x08) - throw new ZipException("Invalid gzip compression method"); - _state = State.FLG; - break; - } - case FLG: - { - _flags = currByte; - _state = State.MTIME; - _size = 0; - _value = 0; - break; - } - case MTIME: - { - // Skip the 4 MTIME bytes - ++_size; - if (_size == 4) - _state = State.XFL; - break; - } - case XFL: - { - // Skip XFL - _state = State.OS; - break; - } - case OS: - { - // Skip OS - _state = State.FLAGS; - break; - } - case EXTRA_LENGTH: - { - _value += (currByte & 0xFF) << 8 * _size; - ++_size; - if (_size == 2) - _state = State.EXTRA; - break; - } - case EXTRA: - { - // Skip EXTRA bytes - --_value; - if (_value == 0) - { - // Clear the EXTRA flag and loop on the flags - _flags &= ~0x04; - _state = State.FLAGS; - } - break; - } - case NAME: - { - // Skip NAME bytes - if (currByte == 0) - { - // Clear the NAME flag and loop on the flags - _flags &= ~0x08; - _state = State.FLAGS; - } - break; - } - case COMMENT: - { - // Skip COMMENT bytes - if (currByte == 0) - { - // Clear the COMMENT flag and loop on the flags - _flags &= ~0x10; - _state = State.FLAGS; - } - break; - } - case HCRC: - { - // Skip HCRC - ++_size; - if (_size == 2) - { - // Clear the HCRC flag and loop on the flags - _flags &= ~0x02; - _state = State.FLAGS; - } - break; - } - case CRC: - { - _value += (currByte & 0xFF) << 8 * _size; - ++_size; - if (_size == 4) - { - // From RFC 1952, compliant decoders need not to verify the CRC - _state = State.ISIZE; - _size = 0; - _value = 0; - } - break; - } - case ISIZE: - { - _value += (currByte & 0xFF) << 8 * _size; - ++_size; - if (_size == 4) - { - if (_value != _inflater.getBytesWritten()) - throw new ZipException("Invalid input size"); - - // TODO ByteBuffer result = output == null ? BufferUtil.EMPTY_BUFFER : ByteBuffer.wrap(output); - reset(); - return; - } - break; - } - default: - throw new ZipException(); - } - } - } - catch (ZipException x) - { - throw new RuntimeException(x); - } - finally - { - if (buffer != null) - release(buffer); - } - } - - private void reset() - { - _inflater.reset(); - _state = State.INITIAL; - _size = 0; - _value = 0; - _flags = 0; - } - - @Override - public void destroy() - { - _inflater.end(); - } - - public boolean isFinished() - { - return _state == State.INITIAL; - } - - private enum State - { - INITIAL, ID, CM, FLG, MTIME, XFL, OS, FLAGS, EXTRA_LENGTH, EXTRA, NAME, COMMENT, HCRC, DATA, CRC, ISIZE - } - - /** - * @param capacity capacity of the ByteBuffer to acquire - * @return a heap buffer of the configured capacity either from the pool or freshly allocated. - */ - public ByteBuffer acquire(int capacity) - { - return _pool == null ? BufferUtil.allocate(capacity) : _pool.acquire(capacity, false); - } - - /** - *

Releases an allocated buffer.

- *

This method calls {@link ByteBufferPool#release(ByteBuffer)} if a buffer pool has - * been configured.

- *

This method should be called once for all buffers returned from {@link #decode(ByteBuffer)} - * or passed to {@link #decodedChunk(ByteBuffer)}.

- * - * @param buffer the buffer to release. - */ - public void release(ByteBuffer buffer) - { - if (_pool != null && buffer!=BufferUtil.EMPTY_BUFFER) - _pool.release(buffer); - } -} diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java index 2b6f2c3abe8..ee949038109 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java @@ -1,14 +1,34 @@ +// +// ======================================================================== +// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + package org.eclipse.jetty.server.handler.gzip; import java.nio.ByteBuffer; import java.util.Queue; import java.util.regex.Pattern; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; +import java.util.zip.ZipException; import org.eclipse.jetty.http.BadMessageException; import org.eclipse.jetty.http.HttpField; import org.eclipse.jetty.http.HttpFields; import org.eclipse.jetty.http.HttpHeader; -import org.eclipse.jetty.io.ArrayByteBufferPool; import org.eclipse.jetty.io.ByteBufferPool; import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.HttpConfiguration; @@ -16,6 +36,7 @@ import org.eclipse.jetty.server.HttpInput; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.util.ArrayQueue; import org.eclipse.jetty.util.BufferUtil; +import org.eclipse.jetty.util.annotation.Name; public class GzipRequestCustomizer implements HttpConfiguration.Customizer { @@ -23,12 +44,25 @@ public class GzipRequestCustomizer implements HttpConfiguration.Customizer private static final HttpField X_CE_GZIP = new HttpField("X-Content-Encoding","gzip"); private static final Pattern COMMA_GZIP = Pattern.compile(".*, *gzip"); - private final ByteBufferPool buffers = new ArrayByteBufferPool(); // TODO Configure - private int compressedBufferSize = 4*1024; // TODO configure - private int inflatedBufferSize = 16*1024; // TODO configure + private final int _compressedBufferSize; + private final int _inflatedBufferSize; + + public GzipRequestCustomizer() + { + this(-1, -1); + } + + public GzipRequestCustomizer(@Name("compressedBufferSize") int compressedBufferSize, @Name("inflatedBufferSize") int inflatedBufferSize) + { + _compressedBufferSize = compressedBufferSize<=0?4*1024:compressedBufferSize; + _inflatedBufferSize = inflatedBufferSize<=0?16*1024:inflatedBufferSize; + } + @Override public void customize(Connector connector, HttpConfiguration channelConfig, Request request) { + ByteBufferPool bufferPool = request.getHttpChannel().getByteBufferPool(); + try { HttpFields fields = request.getHttpFields(); @@ -60,7 +94,7 @@ public class GzipRequestCustomizer implements HttpConfiguration.Customizer { if (buffer==null || BufferUtil.isFull(buffer)) { - buffer = buffers.acquire(compressedBufferSize,false); + buffer = bufferPool.acquire(_compressedBufferSize,false); compressed.add(buffer); } int l = input.read(buffer.array(), buffer.arrayOffset()+buffer.limit(), BufferUtil.space(buffer)); @@ -78,18 +112,7 @@ public class GzipRequestCustomizer implements HttpConfiguration.Customizer return; } - // TODO Perhaps pool docoders/inflators? - GZIPContentDecoder decoder = new GZIPContentDecoder(buffers, inflatedBufferSize) - { - @Override - protected boolean decodedChunk(ByteBuffer chunk) - { - super.decodedChunk(chunk); - return false; - } - }; - - input.addContent(new InflatingContent(input, decoder,compressed)); + input.addContent(new InflatingContent(bufferPool, input, compressed)); } catch(Throwable t) @@ -97,49 +120,291 @@ public class GzipRequestCustomizer implements HttpConfiguration.Customizer throw new BadMessageException(400,"Bad compressed request",t); } } - - private ByteBuffer inflate(GZIPContentDecoder decoder, Queue compressed) + + private enum State { - while (!compressed.isEmpty() && BufferUtil.isEmpty(compressed.peek())) - buffers.release(compressed.poll()); - - if (compressed.isEmpty()) - return BufferUtil.EMPTY_BUFFER; - - ByteBuffer inflated = decoder.decode(compressed.peek()); - System.err.println(BufferUtil.toDetailString(inflated)); - return inflated; + INITIAL, ID, CM, FLG, MTIME, XFL, OS, FLAGS, EXTRA_LENGTH, EXTRA, NAME, COMMENT, HCRC, DATA, CRC, ISIZE, END } - - + private class InflatingContent extends HttpInput.Content { - final HttpInput input; - final GZIPContentDecoder decoder; - final Queue compressed; + final ByteBufferPool _bufferPool; + final HttpInput _input; + final Queue _compressed; + private final Inflater _inflater = new Inflater(true); + private State _state = State.INITIAL; + private int _size; + private int _value; + private byte _flags; - public InflatingContent(HttpInput input, GZIPContentDecoder decoder, Queue compressed) + public InflatingContent(ByteBufferPool bufferPool, HttpInput input, Queue compressed) { - super(inflate(decoder,compressed)); - this.input = input; - this.decoder = decoder; - this.compressed = compressed; + super(bufferPool.acquire(_inflatedBufferSize,false)); + _bufferPool = bufferPool; + _input = input; + _compressed = compressed; + + inflate(); } @Override public void succeeded() { - if (decoder.isFinished() && compressed.isEmpty()) - input.eof(); + BufferUtil.clear(getContent()); + inflate(); + if (BufferUtil.isEmpty(getContent()) && _state==State.END) + { + _bufferPool.release(getContent()); + _input.eof(); + } else - input.addContent(new InflatingContent(input,decoder,compressed)); + { + _input.addContent(this); + } } @Override public void failed(Throwable x) { - input.failed(x); + _input.failed(x); + } + + protected void inflate() + { + try + { + while (true) + { + switch (_state) + { + case INITIAL: + { + _state = State.ID; + break; + } + + case FLAGS: + { + if ((_flags & 0x04) == 0x04) + { + _state = State.EXTRA_LENGTH; + _size = 0; + _value = 0; + } + else if ((_flags & 0x08) == 0x08) + _state = State.NAME; + else if ((_flags & 0x10) == 0x10) + _state = State.COMMENT; + else if ((_flags & 0x2) == 0x2) + { + _state = State.HCRC; + _size = 0; + _value = 0; + } + else + { + _state = State.DATA; + continue; + } + break; + } + + case DATA: + { + while (true) + { + ByteBuffer buffer = getContent(); + + if (BufferUtil.isFull(buffer)) + return; + + try + { + int length = _inflater.inflate(buffer.array(), buffer.arrayOffset() + buffer.position(), BufferUtil.space(buffer)); + buffer.limit(buffer.limit()+length); + } + catch (DataFormatException x) + { + throw new ZipException(x.getMessage()); + } + + if (_inflater.needsInput()) + { + ByteBuffer data = _compressed.peek(); + while(data!=null && BufferUtil.isEmpty(data)) + { + _bufferPool.release(_compressed.poll()); + data = _compressed.peek(); + } + if (data==null) + return; + + _inflater.setInput(data.array(), data.arrayOffset() + data.position(), data.remaining()); + data.position(data.limit()); + } + else if (_inflater.finished()) + { + ByteBuffer data = _compressed.peek(); + int remaining = _inflater.getRemaining(); + data.position(data.limit() - remaining); + _state = State.CRC; + _size = 0; + _value = 0; + break; + } + } + continue; + } + + default: + break; + } + + ByteBuffer data = _compressed.peek(); + if (BufferUtil.isEmpty(data)) + break; + + byte currByte = data.get(); + switch (_state) + { + case ID: + { + _value += (currByte & 0xFF) << 8 * _size; + ++_size; + if (_size == 2) + { + if (_value != 0x8B1F) + throw new ZipException("Invalid gzip bytes"); + _state = State.CM; + } + break; + } + case CM: + { + if ((currByte & 0xFF) != 0x08) + throw new ZipException("Invalid gzip compression method"); + _state = State.FLG; + break; + } + case FLG: + { + _flags = currByte; + _state = State.MTIME; + _size = 0; + _value = 0; + break; + } + case MTIME: + { + // Skip the 4 MTIME bytes + ++_size; + if (_size == 4) + _state = State.XFL; + break; + } + case XFL: + { + // Skip XFL + _state = State.OS; + break; + } + case OS: + { + // Skip OS + _state = State.FLAGS; + break; + } + case EXTRA_LENGTH: + { + _value += (currByte & 0xFF) << 8 * _size; + ++_size; + if (_size == 2) + _state = State.EXTRA; + break; + } + case EXTRA: + { + // Skip EXTRA bytes + --_value; + if (_value == 0) + { + // Clear the EXTRA flag and loop on the flags + _flags &= ~0x04; + _state = State.FLAGS; + } + break; + } + case NAME: + { + // Skip NAME bytes + if (currByte == 0) + { + // Clear the NAME flag and loop on the flags + _flags &= ~0x08; + _state = State.FLAGS; + } + break; + } + case COMMENT: + { + // Skip COMMENT bytes + if (currByte == 0) + { + // Clear the COMMENT flag and loop on the flags + _flags &= ~0x10; + _state = State.FLAGS; + } + break; + } + case HCRC: + { + // Skip HCRC + ++_size; + if (_size == 2) + { + // Clear the HCRC flag and loop on the flags + _flags &= ~0x02; + _state = State.FLAGS; + } + break; + } + case CRC: + { + _value += (currByte & 0xFF) << 8 * _size; + ++_size; + if (_size == 4) + { + // From RFC 1952, compliant decoders need not to verify the CRC + _state = State.ISIZE; + _size = 0; + _value = 0; + } + break; + } + case ISIZE: + { + _value += (currByte & 0xFF) << 8 * _size; + ++_size; + if (_size == 4) + { + if (_value != _inflater.getBytesWritten()) + throw new ZipException("Invalid input size"); + + _inflater.reset(); + _state = State.END; + return; + } + break; + } + default: + throw new ZipException(); + } + } + } + catch (ZipException x) + { + throw new RuntimeException(x); + } } } - }