Merge pull request #4417 from eclipse/jetty-9.4.x-4415-gziphandler-large-stream-support

Issue #4415 - Addressing Gzip Decoding of large files

jetty-http/src/main/java/org/eclipse/jetty/http/GZIPContentDecoder.java

@@ -37,13 +37,16 @@ import org.eclipse.jetty.util.component.Destroyable;
  */
 public class GZIPContentDecoder implements Destroyable
 {
+    // Unsigned Integer Max == 2^32
+    private static final long UINT_MAX = 0xFFFFFFFFL;
+
     private final List<ByteBuffer> _inflateds = new ArrayList<>();
     private final Inflater _inflater = new Inflater(true);
     private final ByteBufferPool _pool;
     private final int _bufferSize;
     private State _state;
     private int _size;
-    private int _value;
+    private long _value;
     private byte _flags;
     private ByteBuffer _inflated;
 
@@ -375,11 +378,12 @@ public class GZIPContentDecoder implements Destroyable
                     }
                     case ISIZE:
                     {
-                        _value += (currByte & 0xFF) << 8 * _size;
+                        _value = _value | ((currByte & 0xFFL) << (8 * _size));
                         ++_size;
                         if (_size == 4)
                         {
-                            if (_value != _inflater.getBytesWritten())
+                            // RFC 1952: Section 2.3.1; ISIZE is the input size modulo 2^32
+                            if (_value != (_inflater.getBytesWritten() & UINT_MAX))
                                 throw new ZipException("Invalid input size");
 
                             // TODO ByteBuffer result = output == null ? BufferUtil.EMPTY_BUFFER : ByteBuffer.wrap(output);

jetty-http/src/test/java/org/eclipse/jetty/http/GZIPContentDecoderTest.java

@@ -20,6 +20,8 @@ package org.eclipse.jetty.http;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -30,7 +32,11 @@ import org.eclipse.jetty.io.ArrayByteBufferPool;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -351,4 +357,81 @@ public class GZIPContentDecoderTest
         assertTrue(buffer.hasRemaining());
         assertEquals(data2, StandardCharsets.UTF_8.decode(buffer).toString());
     }
+
+    // Signed Integer Max
+    final long INT_MAX = Integer.MAX_VALUE;
+
+    // Unsigned Integer Max == 2^32
+    final long UINT_MAX = 0xFFFFFFFFL;
+
+    @ParameterizedTest
+    @ValueSource(longs = {INT_MAX, INT_MAX + 1, UINT_MAX, UINT_MAX + 1})
+    public void testLargeGzipStream(long origSize) throws IOException
+    {
+        // Size chosen for trade off between speed of I/O vs speed of Gzip
+        final int BUFSIZE = 1024 * 1024;
+
+        // Create a buffer to use over and over again to produce the uncompressed input
+        byte[] cbuf = "0123456789ABCDEFGHIJKLMOPQRSTUVWXYZ".getBytes(StandardCharsets.UTF_8);
+        byte[] buf = new byte[BUFSIZE];
+        for (int off = 0; off < buf.length; )
+        {
+            int len = Math.min(cbuf.length, buf.length - off);
+            System.arraycopy(cbuf, 0, buf, off, len);
+            off += len;
+        }
+
+        GZIPDecoderOutputStream out = new GZIPDecoderOutputStream(new GZIPContentDecoder(BUFSIZE));
+        GZIPOutputStream outputStream = new GZIPOutputStream(out, BUFSIZE);
+
+        for (long bytesLeft = origSize; bytesLeft > 0; )
+        {
+            int len = buf.length;
+            if (bytesLeft < buf.length)
+            {
+                len = (int)bytesLeft;
+            }
+            outputStream.write(buf, 0, len);
+            bytesLeft -= len;
+        }
+
+        // Close GZIPOutputStream to have it generate gzip trailer.
+        // This can cause more writes of unflushed gzip buffers
+        outputStream.close();
+
+        // out.decodedByteCount is only valid after close
+        assertThat("Decoded byte count", out.decodedByteCount, is(origSize));
+    }
+
+    public static class GZIPDecoderOutputStream extends OutputStream
+    {
+        private final GZIPContentDecoder decoder;
+        public long decodedByteCount = 0L;
+
+        public GZIPDecoderOutputStream(GZIPContentDecoder decoder)
+        {
+            this.decoder = decoder;
+        }
+
+        @Override
+        public void write(byte[] b, int off, int len) throws IOException
+        {
+            ByteBuffer buf = ByteBuffer.wrap(b, off, len);
+            while (buf.hasRemaining())
+            {
+                ByteBuffer decoded = decoder.decode(buf);
+                if (decoded.hasRemaining())
+                {
+                    decodedByteCount += decoded.remaining();
+                }
+                decoder.release(decoded);
+            }
+        }
+
+        @Override
+        public void write(int b) throws IOException
+        {
+            write(new byte[]{(byte)b}, 0, 1);
+        }
+    }
 }