Eclipse
/
jetty.project
mirror of https://github.com/jetty/jetty.project.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5248 from eclipse/jetty-10.0.x-5198-UpdateGzipHandler 

Issue #5198 - update gzip handler
This commit is contained in:
Lachlan 2020-09-21 10:41:16 +10:00 committed by GitHub
parent 7318443a8f f2abf21df6
commit df085a610f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 234 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
jetty-http/src/main/java/org/eclipse/jetty/http/GZIPContentDecoder.java
View File

@ -29,6 +29,7 @@ import java.util.zip.ZipException;
import org.eclipse.jetty.io.ByteBufferPool;
import org.eclipse.jetty.util.BufferUtil;
import org.eclipse.jetty.util.component.Destroyable;
import org.eclipse.jetty.util.compression.InflaterPool;
/**
* <p>Decoder for the "gzip" content encoding.</p>
@ -41,9 +42,10 @@ public class GZIPContentDecoder implements Destroyable
private static final long UINT_MAX = 0xFFFFFFFFL;
private final List<ByteBuffer> _inflateds = new ArrayList<>();
private final Inflater _inflater = new Inflater(true);
private final InflaterPool _inflaterPool;
private final ByteBufferPool _pool;
private final int _bufferSize;
private Inflater _inflater;
private State _state;
private int _size;
private long _value;
@ -62,6 +64,13 @@ public class GZIPContentDecoder implements Destroyable
public GZIPContentDecoder(ByteBufferPool pool, int bufferSize)
{
this(null, pool, bufferSize);
}
public GZIPContentDecoder(InflaterPool inflaterPool, ByteBufferPool pool, int bufferSize)
{
_inflaterPool = inflaterPool;
_inflater = (inflaterPool == null) ? new Inflater(true) : inflaterPool.acquire();
_bufferSize = bufferSize;
_pool = pool;
reset();
@ -207,8 +216,9 @@ public class GZIPContentDecoder implements Destroyable
try
{
int length = _inflater.inflate(buffer.array(), buffer.arrayOffset(), buffer.capacity());
buffer.limit(length);
int pos = BufferUtil.flipToFill(buffer);
_inflater.inflate(buffer);
BufferUtil.flipToFlush(buffer, pos);
}
catch (DataFormatException x)
{
@ -226,23 +236,10 @@ public class GZIPContentDecoder implements Destroyable
{
if (!compressed.hasRemaining())
return;
if (compressed.hasArray())
{
_inflater.setInput(compressed.array(), compressed.arrayOffset() + compressed.position(), compressed.remaining());
compressed.position(compressed.limit());
}
else
{
// TODO use the pool
byte[] input = new byte[compressed.remaining()];
compressed.get(input);
_inflater.setInput(input);
}
_inflater.setInput(compressed);
}
else if (_inflater.finished())
{
int remaining = _inflater.getRemaining();
compressed.position(compressed.limit() - remaining);
_state = State.CRC;
_size = 0;
_value = 0;
@ -386,7 +383,6 @@ public class GZIPContentDecoder implements Destroyable
if (_value != (_inflater.getBytesWritten() & UINT_MAX))
throw new ZipException("Invalid input size");
// TODO ByteBuffer result = output == null ? BufferUtil.EMPTY_BUFFER : ByteBuffer.wrap(output);
reset();
return;
}
@ -420,7 +416,12 @@ public class GZIPContentDecoder implements Destroyable
@Override
public void destroy()
{
_inflater.end();
if (_inflaterPool == null)
_inflater.end();
else
_inflaterPool.release(_inflater);
_inflater = null;
}
public boolean isFinished()

6
jetty-server/src/main/config/etc/jetty-gzip.xml
View File

@ -16,10 +16,16 @@
<Set name="checkGzExists" property="jetty.gzip.checkGzExists"/>
<Set name="compressionLevel" property="jetty.gzip.compressionLevel"/>
<Set name="inflateBufferSize" property="jetty.gzip.inflateBufferSize"/>
<Set name="inflaterPoolCapacity" property="jetty.gzip.inflaterPoolCapacity"/>
<Set name="deflaterPoolCapacity" property="jetty.gzip.deflaterPoolCapacity"/>
<Set name="syncFlush" property="jetty.gzip.syncFlush"/>
<Set name="dispatcherTypes" property="jetty.gzip.dispatcherTypes"/>
<Set name="includedMethodList" property="jetty.gzip.includedMethodList"/>
<Set name="excludedMethodList" property="jetty.gzip.excludedMethodList"/>
<Set name="includedMimeTypes" property="jetty.gzip.includedMimeTypeList"/>
<Set name="excludedMimeTypes" property="jetty.gzip.excludedMimeTypeList"/>
<Set name="includedPaths" property="jetty.gzip.includedPathList"/>
<Set name="excludedPaths" property="jetty.gzip.excludedPathList"/>
<!--
<Set name="includedMethods">

27
jetty-server/src/main/config/modules/gzip.mod
View File

@ -27,11 +27,32 @@ etc/jetty-gzip.xml
## Inflate request buffer size, or 0 for no request inflation
# jetty.gzip.inflateBufferSize=0
## Deflater pool max size (-1 for unlimited, 0 for no pool)
## Deflater pool max size (-1 for unlimited, 0 for no pooling)
# jetty.gzip.deflaterPoolCapacity=-1
## Comma separated list of included methods
## Inflater pool max size (-1 for unlimited, 0 for no pooling)
# jetty.gzip.inflaterPoolCapacity=-1
## Set the {@link Deflater} flush mode to use.
# jetty.gzip.syncFlush=false
## The set of DispatcherType that this filter will operate on
# jetty.gzip.dispatcherTypes=REQUEST
## Comma separated list of included HTTP methods
# jetty.gzip.includedMethodList=GET,POST
## Comma separated list of excluded methods
## Comma separated list of excluded HTTP methods
# jetty.gzip.excludedMethodList=
## Comma separated list of included MIME types
# jetty.gzip.includedMimeTypeList=
## Comma separated list of excluded MIME types
# jetty.gzip.excludedMimeTypeList=
## Comma separated list of included Path specs
# jetty.gzip.includedPathList=
## Comma separated list of excluded Path specs
# jetty.gzip.excludedPathList=

72
jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipHandler.java
View File

@ -23,6 +23,8 @@ import java.util.Arrays;
import java.util.EnumSet;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.Deflater;
import javax.servlet.DispatcherType;
import javax.servlet.ServletContext;
@ -42,9 +44,12 @@ import org.eclipse.jetty.http.pathmap.PathSpecSet;
import org.eclipse.jetty.server.HttpOutput;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.HandlerWrapper;
import org.eclipse.jetty.util.AsciiLowerCaseSet;
import org.eclipse.jetty.util.IncludeExclude;
import org.eclipse.jetty.util.StringUtil;
import org.eclipse.jetty.util.compression.CompressionPool;
import org.eclipse.jetty.util.compression.DeflaterPool;
import org.eclipse.jetty.util.compression.InflaterPool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -158,8 +163,8 @@ public class GzipHandler extends HandlerWrapper implements GzipFactory
private static final HttpField TE_CHUNKED = new PreEncodedHttpField(HttpHeader.TRANSFER_ENCODING, HttpHeaderValue.CHUNKED.asString());
private static final Pattern COMMA_GZIP = Pattern.compile(".*, *gzip");
private int poolCapacity = -1;
private DeflaterPool _deflaterPool = null;
private final InflaterPool _inflaterPool;
private final DeflaterPool _deflaterPool;
private int _minGzipSize = DEFAULT_MIN_GZIP_SIZE;
private boolean _syncFlush = false;
@ -168,7 +173,7 @@ public class GzipHandler extends HandlerWrapper implements GzipFactory
// non-static, as other GzipHandler instances may have different configurations
private final IncludeExclude<String> _methods = new IncludeExclude<>();
private final IncludeExclude<String> _paths = new IncludeExclude<>(PathSpecSet.class);
private final IncludeExclude<String> _mimeTypes = new IncludeExclude<>();
private final IncludeExclude<String> _mimeTypes = new IncludeExclude<>(AsciiLowerCaseSet.class);
private HttpField _vary = GzipHttpOutputInterceptor.VARY_ACCEPT_ENCODING;
/**
@ -197,6 +202,11 @@ public class GzipHandler extends HandlerWrapper implements GzipFactory
if (LOG.isDebugEnabled())
LOG.debug("{} mime types {}", this, _mimeTypes);
_deflaterPool = newDeflaterPool();
_inflaterPool = newInflaterPool();
addBean(_deflaterPool);
addBean(_inflaterPool);
}
/**
@ -407,13 +417,6 @@ public class GzipHandler extends HandlerWrapper implements GzipFactory
}
}
@Override
protected void doStart() throws Exception
{
_deflaterPool = newDeflaterPool(poolCapacity);
super.doStart();
}
@Override
public Deflater getDeflater(Request request, long contentLength)
{
@ -564,7 +567,7 @@ public class GzipHandler extends HandlerWrapper implements GzipFactory
{
if (LOG.isDebugEnabled())
LOG.debug("{} inflate {}", this, request);
baseRequest.getHttpInput().addInterceptor(new GzipHttpInputInterceptor(baseRequest.getHttpChannel().getByteBufferPool(), _inflateBufferSize));
baseRequest.getHttpInput().addInterceptor(new GzipHttpInputInterceptor(_inflaterPool, baseRequest.getHttpChannel().getByteBufferPool(), _inflateBufferSize));
}
// Are we already being gzipped?
@ -771,6 +774,19 @@ public class GzipHandler extends HandlerWrapper implements GzipFactory
_paths.exclude(pathspecs);
}
/**
* Set of supported {@link DispatcherType} that this filter will operate on.
*
* @param dispatchers the set of {@link DispatcherType} that this filter will operate on
*/
public void setDispatcherTypes(String... dispatchers)
{
_dispatchers = EnumSet.copyOf(Stream.of(dispatchers)
.flatMap(s -> Stream.of(StringUtil.csvSplit(s)))
.map(DispatcherType::valueOf)
.collect(Collectors.toSet()));
}
/**
* Set the included filter list of HTTP methods (replacing any previously set)
*
@ -876,7 +892,7 @@ public class GzipHandler extends HandlerWrapper implements GzipFactory
*/
public int getDeflaterPoolCapacity()
{
return poolCapacity;
return _deflaterPool.getCapacity();
}
/**
@ -887,12 +903,38 @@ public class GzipHandler extends HandlerWrapper implements GzipFactory
if (isStarted())
throw new IllegalStateException(getState());
poolCapacity = capacity;
_deflaterPool.setCapacity(capacity);
}
protected DeflaterPool newDeflaterPool(int capacity)
/**
* Gets the maximum number of Inflators that the DeflaterPool can hold.
*
* @return the Deflater pool capacity
*/
public int getInflaterPoolCapacity()
{
return new DeflaterPool(capacity, Deflater.DEFAULT_COMPRESSION, true);
return _inflaterPool.getCapacity();
}
/**
* Sets the maximum number of Inflators that the DeflaterPool can hold.
*/
public void setInflaterPoolCapacity(int capacity)
{
if (isStarted())
throw new IllegalStateException(getState());
_inflaterPool.setCapacity(capacity);
}
protected InflaterPool newInflaterPool()
{
return new InflaterPool(CompressionPool.INFINITE_CAPACITY, true);
}
protected DeflaterPool newDeflaterPool()
{
return new DeflaterPool(CompressionPool.INFINITE_CAPACITY, Deflater.DEFAULT_COMPRESSION, true);
}
@Override

9
jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipHttpInputInterceptor.java
View File

@ -25,6 +25,7 @@ import org.eclipse.jetty.io.ByteBufferPool;
import org.eclipse.jetty.server.HttpInput;
import org.eclipse.jetty.server.HttpInput.Content;
import org.eclipse.jetty.util.component.Destroyable;
import org.eclipse.jetty.util.compression.InflaterPool;
/**
* An HttpInput Interceptor that inflates GZIP encoded request content.
@ -34,9 +35,9 @@ public class GzipHttpInputInterceptor implements HttpInput.Interceptor, Destroya
private final Decoder _decoder;
private ByteBuffer _chunk;
public GzipHttpInputInterceptor(ByteBufferPool pool, int bufferSize)
public GzipHttpInputInterceptor(InflaterPool inflaterPool, ByteBufferPool pool, int bufferSize)
{
_decoder = new Decoder(pool, bufferSize);
_decoder = new Decoder(inflaterPool, pool, bufferSize);
}
@Override
@ -66,9 +67,9 @@ public class GzipHttpInputInterceptor implements HttpInput.Interceptor, Destroya
private class Decoder extends GZIPContentDecoder
{
private Decoder(ByteBufferPool pool, int bufferSize)
private Decoder(InflaterPool inflaterPool, ByteBufferPool bufferPool, int bufferSize)
{
super(pool, bufferSize);
super(inflaterPool, bufferPool, bufferSize);
}
@Override

5
jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipHttpOutputInterceptor.java
View File

@ -196,7 +196,6 @@ public class GzipHttpOutputInterceptor implements HttpOutput.Interceptor
contentLength = content.remaining();
_deflater = _factory.getDeflater(_channel.getRequest(), contentLength);
if (_deflater == null)
{
LOG.debug("{} exclude no deflater", this);
@ -355,7 +354,9 @@ public class GzipHttpOutputInterceptor implements HttpOutput.Interceptor
int off = slice.arrayOffset() + slice.position();
int len = slice.remaining();
_crc.update(array, off, len);
_deflater.setInput(array, off, len); // TODO use ByteBuffer API in Jetty-10
// Ideally we would want to use the ByteBuffer API for Deflaters. However due the the ByteBuffer implementation
// of the CRC32.update() it is less efficient for us to use this rather than to convert to array ourselves.
_deflater.setInput(array, off, len);
slice.position(slice.position() + len);
if (_last && BufferUtil.isEmpty(_content))
_deflater.finish();

58
jetty-servlets/src/test/java/org/eclipse/jetty/servlets/GzipDefaultServletTest.java
View File

@ -943,4 +943,62 @@ public class GzipDefaultServletTest extends AbstractGzipTest
UncompressedMetadata metadata = parseResponseContent(response);
assertThat("Response Content Length", metadata.contentLength, is(fileSize));
}
@Test
public void testUpperCaseMimeType() throws Exception
{
GzipHandler gzipHandler = new GzipHandler();
gzipHandler.addExcludedMimeTypes("text/PLAIN");
server = new Server();
LocalConnector localConnector = new LocalConnector(server);
server.addConnector(localConnector);
Path contextDir = workDir.resolve("context");
FS.ensureDirExists(contextDir);
ServletContextHandler servletContextHandler = new ServletContextHandler();
servletContextHandler.setContextPath("/context");
servletContextHandler.setBaseResource(new PathResource(contextDir));
ServletHolder holder = new ServletHolder("default", DefaultServlet.class);
holder.setInitParameter("etags", "true");
servletContextHandler.addServlet(holder, "/");
servletContextHandler.insertHandler(gzipHandler);
server.setHandler(servletContextHandler);
// Prepare Server File
int fileSize = DEFAULT_OUTPUT_BUFFER_SIZE * 4;
Path file = createFile(contextDir, "file.txt", fileSize);
String expectedSha1Sum = Sha1Sum.calculate(file);
server.start();
// Setup request
HttpTester.Request request = HttpTester.newRequest();
request.setMethod("GET");
request.setVersion(HttpVersion.HTTP_1_1);
request.setHeader("Host", "tester");
request.setHeader("Connection", "close");
request.setHeader("Accept-Encoding", "gzip");
request.setURI("/context/file.txt");
// Issue request
ByteBuffer rawResponse = localConnector.getResponse(request.generate(), 5, TimeUnit.SECONDS);
// Parse response
HttpTester.Response response = HttpTester.parseResponse(rawResponse);
assertThat("Response status", response.getStatus(), is(HttpStatus.OK_200));
// Response Content-Encoding check
assertThat("Response[Content-Encoding]", response.get("Content-Encoding"), not(containsString("gzip")));
assertThat("Response[Vary]", response.get("Vary"), is(nullValue()));
// Response Content checks
UncompressedMetadata metadata = parseResponseContent(response);
assertThat("Response Content Length", metadata.contentLength, is(fileSize));
assertThat("(Uncompressed) Content Length", metadata.uncompressedSize, is(fileSize));
assertThat("(Uncompressed) Content Hash", metadata.uncompressedSha1Sum, is(expectedSha1Sum));
}
}

38
jetty-util/src/main/java/org/eclipse/jetty/util/AsciiLowerCaseSet.java Normal file
View File

@ -0,0 +1,38 @@
//
// ========================================================================
// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
//
// This program and the accompanying materials are made available under
// the terms of the Eclipse Public License 2.0 which is available at
// https://www.eclipse.org/legal/epl-2.0
//
// This Source Code may also be made available under the following
// Secondary Licenses when the conditions for such availability set
// forth in the Eclipse Public License, v. 2.0 are satisfied:
// the Apache License v2.0 which is available at
// https://www.apache.org/licenses/LICENSE-2.0
//
// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
// ========================================================================
//
package org.eclipse.jetty.util;
import java.util.HashSet;
public class AsciiLowerCaseSet extends HashSet<String>
{
@Override
public boolean add(String s)
{
return super.add(s == null ? null : StringUtil.asciiToLowerCase(s));
}
@Override
public boolean contains(Object o)
{
if (o instanceof String)
return super.contains(StringUtil.asciiToLowerCase((String)o));
return super.contains(o);
}
}

26
jetty-util/src/main/java/org/eclipse/jetty/util/compression/CompressionPool.java
View File

@ -30,7 +30,7 @@ public abstract class CompressionPool<T> extends AbstractLifeCycle
private final Queue<T> _pool;
private final AtomicInteger _numObjects = new AtomicInteger(0);
private final int _capacity;
private int _capacity;
/**
* Create a Pool of {@link T} instances.
@ -44,7 +44,17 @@ public abstract class CompressionPool<T> extends AbstractLifeCycle
public CompressionPool(int capacity)
{
_capacity = capacity;
_pool = (_capacity == 0) ? null : new ConcurrentLinkedQueue<>();
_pool = new ConcurrentLinkedQueue<>();
}
public int getCapacity()
{
return _capacity;
}
public void setCapacity(int capacity)
{
_capacity = capacity;
}
protected abstract T newObject();
@ -85,7 +95,6 @@ public abstract class CompressionPool<T> extends AbstractLifeCycle
if (_capacity == 0 || !isRunning())
{
end(object);
return;
}
else if (_capacity < 0)
{
@ -125,4 +134,15 @@ public abstract class CompressionPool<T> extends AbstractLifeCycle
}
_numObjects.set(0);
}
@Override
public String toString()
{
return String.format("%s@%x{%s,size=%d,capacity=%s}",
getClass().getSimpleName(),
hashCode(),
getState(),
_pool.size(),
_capacity < 0 ? "UNLIMITED" : _capacity);
}
}