Eclipse
/
jetty.project
mirror of https://github.com/jetty/jetty.project.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3461 from eclipse/jetty-9.3.x-gzipRequestCustomizer 

Jetty 9.3.x gzip request customizer
This commit is contained in:
Greg Wilkins 2019-03-19 13:50:50 +11:00 committed by GitHub
parent 25afa868da 6db357d2fc
commit 8427c52c8f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 603 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
jetty-server/src/main/config/etc/jetty-gzip-inflate.xml Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0"?>
<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure_9_3.dtd">
<Configure id="httpConfig" class="org.eclipse.jetty.server.HttpConfiguration">
<Call name="addCustomizer">
<Arg>
<New class="org.eclipse.jetty.server.handler.gzip.GzipRequestCustomizer">
<Arg name="compressedBufferSize" type="int"><Property name="jetty.gzip.inflate.compressedBufferSize" default="4096"/></Arg>
<Arg name="inflatedBufferSize" type="int"><Property name="jetty.gzip.inflate.inflatedBufferSize" default="16384"/></Arg>
</New>
</Arg>
</Call>
</Configure>

17
jetty-server/src/main/config/modules/gzip-inflate.mod Normal file
View File

@ -0,0 +1,17 @@
#
# GZIP inflate module
# Applies GzipRequestCustomizer to entire server to inflate gzipped requests
#
[depend]
server
[xml]
etc/jetty-gzip-inflate.xml
[ini-template]
## Buffer size for compressed data
# jetty.gzip.inflate.compressedBufferSize=4096
## Buffer size for compressed data
# jetty.gzip.inflate.inflatedBufferSize=16384

410
jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipRequestCustomizer.java Normal file
View File

@ -0,0 +1,410 @@
//
// ========================================================================
// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.server.handler.gzip;
import java.nio.ByteBuffer;
import java.util.Queue;
import java.util.regex.Pattern;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;
import java.util.zip.ZipException;
import org.eclipse.jetty.http.BadMessageException;
import org.eclipse.jetty.http.HttpField;
import org.eclipse.jetty.http.HttpFields;
import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.io.ByteBufferPool;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpInput;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.util.ArrayQueue;
import org.eclipse.jetty.util.BufferUtil;
import org.eclipse.jetty.util.annotation.Name;
public class GzipRequestCustomizer implements HttpConfiguration.Customizer
{
public static final String GZIP = "gzip";
private static final HttpField X_CE_GZIP = new HttpField("X-Content-Encoding","gzip");
private static final Pattern COMMA_GZIP = Pattern.compile(".*, *gzip");
private final int _compressedBufferSize;
private final int _inflatedBufferSize;
public GzipRequestCustomizer()
{
this(-1, -1);
}
public GzipRequestCustomizer(@Name("compressedBufferSize") int compressedBufferSize, @Name("inflatedBufferSize") int inflatedBufferSize)
{
_compressedBufferSize = compressedBufferSize<=0?4*1024:compressedBufferSize;
_inflatedBufferSize = inflatedBufferSize<=0?16*1024:inflatedBufferSize;
}
@Override
public void customize(Connector connector, HttpConfiguration channelConfig, Request request)
{
ByteBufferPool bufferPool = request.getHttpChannel().getByteBufferPool();
try
{
HttpFields fields = request.getHttpFields();
String content_encoding = fields.get(HttpHeader.CONTENT_ENCODING);
if (content_encoding == null)
return;
if (content_encoding.equalsIgnoreCase("gzip"))
{
fields.remove(HttpHeader.CONTENT_ENCODING);
}
else if (COMMA_GZIP.matcher(content_encoding).matches())
{
fields.remove(HttpHeader.CONTENT_ENCODING);
fields.add(HttpHeader.CONTENT_ENCODING, content_encoding.substring(0, content_encoding.lastIndexOf(',')));
}
else
{
return;
}
fields.add(X_CE_GZIP);
// Read all the compressed content into a queue of buffers
final HttpInput input = request.getHttpInput();
Queue<ByteBuffer> compressed = new ArrayQueue<>();
ByteBuffer buffer = null;
while (true)
{
if (buffer==null || BufferUtil.isFull(buffer))
{
buffer = bufferPool.acquire(_compressedBufferSize,false);
compressed.add(buffer);
}
int l = input.read(buffer.array(), buffer.arrayOffset()+buffer.limit(), BufferUtil.space(buffer));
if (l<0)
break;
buffer.limit(buffer.limit()+l);
}
input.recycle();
// Handle no content
if (compressed.size()==1 && BufferUtil.isEmpty(buffer))
{
input.eof();
return;
}
input.addContent(new InflatingContent(bufferPool, input, compressed));
}
catch(Throwable t)
{
throw new BadMessageException(400,"Bad compressed request",t);
}
}
private enum State
{
INITIAL, ID, CM, FLG, MTIME, XFL, OS, FLAGS, EXTRA_LENGTH, EXTRA, NAME, COMMENT, HCRC, DATA, CRC, ISIZE, END
}
private class InflatingContent extends HttpInput.Content
{
final ByteBufferPool _bufferPool;
final HttpInput _input;
final Queue<ByteBuffer> _compressed;
private final Inflater _inflater = new Inflater(true);
private State _state = State.INITIAL;
private int _size;
private int _value;
private byte _flags;
public InflatingContent(ByteBufferPool bufferPool, HttpInput input, Queue<ByteBuffer> compressed)
{
super(bufferPool.acquire(_inflatedBufferSize,false));
_bufferPool = bufferPool;
_input = input;
_compressed = compressed;
inflate();
}
@Override
public void succeeded()
{
BufferUtil.clear(getContent());
inflate();
if (BufferUtil.isEmpty(getContent()) && _state==State.END)
{
_bufferPool.release(getContent());
_input.eof();
}
else
{
_input.addContent(this);
}
}
@Override
public void failed(Throwable x)
{
_input.failed(x);
}
protected void inflate()
{
try
{
while (true)
{
switch (_state)
{
case INITIAL:
{
_state = State.ID;
break;
}
case FLAGS:
{
if ((_flags & 0x04) == 0x04)
{
_state = State.EXTRA_LENGTH;
_size = 0;
_value = 0;
}
else if ((_flags & 0x08) == 0x08)
_state = State.NAME;
else if ((_flags & 0x10) == 0x10)
_state = State.COMMENT;
else if ((_flags & 0x2) == 0x2)
{
_state = State.HCRC;
_size = 0;
_value = 0;
}
else
{
_state = State.DATA;
continue;
}
break;
}
case DATA:
{
while (true)
{
ByteBuffer buffer = getContent();
if (BufferUtil.isFull(buffer))
return;
try
{
int length = _inflater.inflate(buffer.array(), buffer.arrayOffset() + buffer.position(), BufferUtil.space(buffer));
buffer.limit(buffer.limit()+length);
}
catch (DataFormatException x)
{
throw new ZipException(x.getMessage());
}
if (_inflater.needsInput())
{
ByteBuffer data = _compressed.peek();
while(data!=null && BufferUtil.isEmpty(data))
{
_bufferPool.release(_compressed.poll());
data = _compressed.peek();
}
if (data==null)
return;
_inflater.setInput(data.array(), data.arrayOffset() + data.position(), data.remaining());
data.position(data.limit());
}
else if (_inflater.finished())
{
ByteBuffer data = _compressed.peek();
int remaining = _inflater.getRemaining();
data.position(data.limit() - remaining);
_state = State.CRC;
_size = 0;
_value = 0;
break;
}
}
continue;
}
default:
break;
}
ByteBuffer data = _compressed.peek();
if (BufferUtil.isEmpty(data))
break;
byte currByte = data.get();
switch (_state)
{
case ID:
{
_value += (currByte & 0xFF) << 8 * _size;
++_size;
if (_size == 2)
{
if (_value != 0x8B1F)
throw new ZipException("Invalid gzip bytes");
_state = State.CM;
}
break;
}
case CM:
{
if ((currByte & 0xFF) != 0x08)
throw new ZipException("Invalid gzip compression method");
_state = State.FLG;
break;
}
case FLG:
{
_flags = currByte;
_state = State.MTIME;
_size = 0;
_value = 0;
break;
}
case MTIME:
{
// Skip the 4 MTIME bytes
++_size;
if (_size == 4)
_state = State.XFL;
break;
}
case XFL:
{
// Skip XFL
_state = State.OS;
break;
}
case OS:
{
// Skip OS
_state = State.FLAGS;
break;
}
case EXTRA_LENGTH:
{
_value += (currByte & 0xFF) << 8 * _size;
++_size;
if (_size == 2)
_state = State.EXTRA;
break;
}
case EXTRA:
{
// Skip EXTRA bytes
--_value;
if (_value == 0)
{
// Clear the EXTRA flag and loop on the flags
_flags &= ~0x04;
_state = State.FLAGS;
}
break;
}
case NAME:
{
// Skip NAME bytes
if (currByte == 0)
{
// Clear the NAME flag and loop on the flags
_flags &= ~0x08;
_state = State.FLAGS;
}
break;
}
case COMMENT:
{
// Skip COMMENT bytes
if (currByte == 0)
{
// Clear the COMMENT flag and loop on the flags
_flags &= ~0x10;
_state = State.FLAGS;
}
break;
}
case HCRC:
{
// Skip HCRC
++_size;
if (_size == 2)
{
// Clear the HCRC flag and loop on the flags
_flags &= ~0x02;
_state = State.FLAGS;
}
break;
}
case CRC:
{
_value += (currByte & 0xFF) << 8 * _size;
++_size;
if (_size == 4)
{
// From RFC 1952, compliant decoders need not to verify the CRC
_state = State.ISIZE;
_size = 0;
_value = 0;
}
break;
}
case ISIZE:
{
_value += (currByte & 0xFF) << 8 * _size;
++_size;
if (_size == 4)
{
if (_value != _inflater.getBytesWritten())
throw new ZipException("Invalid input size");
_inflater.reset();
_state = State.END;
return;
}
break;
}
default:
throw new ZipException();
}
}
}
catch (ZipException x)
{
throw new RuntimeException(x);
}
}
}
}

163
jetty-servlet/src/test/java/org/eclipse/jetty/servlet/GzipHandlerTest.java
View File

@ -33,8 +33,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
@ -42,10 +45,13 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.eclipse.jetty.http.HttpTester;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.LocalConnector;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.handler.gzip.GzipHandler;
import org.eclipse.jetty.server.handler.gzip.GzipRequestCustomizer;
import org.eclipse.jetty.util.IO;
import org.hamcrest.MatcherAssert;
import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.Before;
@ -82,6 +88,7 @@ public class GzipHandlerTest
_server = new Server();
_connector = new LocalConnector(_server);
_server.addConnector(_connector);
_connector.getConnectionFactory(HttpConnectionFactory.class).getHttpConfiguration().addCustomizer(new GzipRequestCustomizer());
GzipHandler gzipHandler = new GzipHandler();
gzipHandler.setExcludedAgentPatterns();
@ -97,6 +104,9 @@ public class GzipHandlerTest
servlets.addServletWithMapping(TestServlet.class,"/content");
servlets.addServletWithMapping(ForwardServlet.class,"/forward");
servlets.addServletWithMapping(IncludeServlet.class,"/include");
servlets.addServletWithMapping(EchoServlet.class,"/echo/*");
servlets.addServletWithMapping(DumpServlet.class,"/dump/*");
_server.start();
}
@ -148,6 +158,34 @@ public class GzipHandlerTest
}
}
public static class EchoServlet extends HttpServlet
{
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse response) throws ServletException, IOException
{
response.setContentType(req.getContentType());
IO.copy(req.getInputStream(),response.getOutputStream());
}
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse response) throws ServletException, IOException
{
doGet(req,response);
}
}
public static class DumpServlet extends HttpServlet
{
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse response) throws ServletException, IOException
{
response.setContentType("text/plain");
for (Enumeration<String> e = req.getParameterNames(); e.hasMoreElements(); )
{
String n = e.nextElement();
response.getWriter().printf("%s: %s\n",n,req.getParameter(n));
}
}
}
public static class ForwardServlet extends HttpServlet
{
@Override
@ -392,4 +430,129 @@ public class GzipHandlerTest
assertThat("Included Paths.size", includedPaths.length, is(2));
assertThat("Included Paths", Arrays.asList(includedPaths), contains("/foo","^/bar.*$"));
}
@Test
public void testGzipRequest() throws Exception
{
String data = "Hello Nice World! ";
for (int i = 0; i < 10; ++i)
data += data;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data.getBytes(StandardCharsets.UTF_8));
output.close();
byte[] bytes = baos.toByteArray();
// generated and parsed test
HttpTester.Request request = HttpTester.newRequest();
HttpTester.Response response;
request.setMethod("POST");
request.setURI("/ctx/echo");
request.setVersion("HTTP/1.0");
request.setHeader("Host","tester");
request.setHeader("Content-Type","text/plain");
request.setHeader("Content-Encoding","gzip");
request.setContent(bytes);
response = HttpTester.parseResponse(_connector.getResponse(request.generate()));
MatcherAssert.assertThat(response.getStatus(),is(200));
MatcherAssert.assertThat(response.getContent(),is(data));
}
@Test
public void testGzipRequestChunked() throws Exception
{
String data = "Hello Nice World! ";
for (int i = 0; i < 10; ++i)
data += data;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data.getBytes(StandardCharsets.UTF_8));
output.close();
byte[] bytes = baos.toByteArray();
// generated and parsed test
HttpTester.Request request = HttpTester.newRequest();
HttpTester.Response response;
request.setMethod("POST");
request.setURI("/ctx/echo");
request.setVersion("HTTP/1.1");
request.setHeader("Host","tester");
request.setHeader("Content-Type","text/plain");
request.setHeader("Content-Encoding","gzip");
request.add("Transfer-Encoding", "chunked");
request.setContent(bytes);
response = HttpTester.parseResponse(_connector.getResponse(request.generate()));
MatcherAssert.assertThat(response.getStatus(),is(200));
MatcherAssert.assertThat(response.getContent(),is(data));
}
@Test
public void testGzipFormRequest() throws Exception
{
String data = "name=value";
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data.getBytes(StandardCharsets.UTF_8));
output.close();
byte[] bytes = baos.toByteArray();
// generated and parsed test
HttpTester.Request request = HttpTester.newRequest();
HttpTester.Response response;
request.setMethod("POST");
request.setURI("/ctx/dump");
request.setVersion("HTTP/1.0");
request.setHeader("Host","tester");
request.setHeader("Content-Type","application/x-www-form-urlencoded; charset=utf-8");
request.setHeader("Content-Encoding","gzip");
request.setContent(bytes);
response = HttpTester.parseResponse(_connector.getResponse(request.generate()));
MatcherAssert.assertThat(response.getStatus(),is(200));
MatcherAssert.assertThat(response.getContent(),is("name: value\n"));
}
@Test
public void testGzipBomb() throws Exception
{
byte[] data = new byte[512*1024];
Arrays.fill(data,(byte)'X');
ByteArrayOutputStream baos = new ByteArrayOutputStream();
GZIPOutputStream output = new GZIPOutputStream(baos);
output.write(data);
output.close();
byte[] bytes = baos.toByteArray();
// generated and parsed test
HttpTester.Request request = HttpTester.newRequest();
HttpTester.Response response;
request.setMethod("POST");
request.setURI("/ctx/echo");
request.setVersion("HTTP/1.0");
request.setHeader("Host","tester");
request.setHeader("Content-Type","text/plain");
request.setHeader("Content-Encoding","gzip");
request.setContent(bytes);
response = HttpTester.parseResponse(_connector.getResponse(request.generate()));
// TODO need to test back pressure works
MatcherAssert.assertThat(response.getStatus(),is(200));
MatcherAssert.assertThat(response.getContentBytes().length,is(512*1024));
}
}