HTTPCLIENT-1550: fixed 'deflate' zlib header check

git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/branches/4.5.x@1717323 13f79535-47bb-0310-9956-ffa450edef68

httpclient/src/main/java/org/apache/http/client/entity/DeflateInputStream.java

@@ -29,179 +29,110 @@ package org.apache.http.client.entity;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PushbackInputStream;
-import java.util.zip.DataFormatException;
 import java.util.zip.Inflater;
 import java.util.zip.InflaterInputStream;
+import java.util.zip.ZipException;
+
+/**
+ * Deflate input stream.    This class includes logic needed for various Rfc's in order
+ * to reasonably implement the "deflate" compression style.
+ */
+public class DeflateInputStream extends InputStream {
 
-/** Deflate input stream.    This class includes logic needed for various Rfc's in order
-* to reasonably implement the "deflate" compression style.
-*/
-public class DeflateInputStream extends InputStream
-{
     private InputStream sourceStream;
 
-    public DeflateInputStream(final InputStream wrapped)
+    public DeflateInputStream(final InputStream wrapped) throws IOException {
-        throws IOException
-    {
-        /*
-            * A zlib stream will have a header.
-            *
-            * CMF | FLG [| DICTID ] | ...compressed data | ADLER32 |
-            *
-            * * CMF is one byte.
-            *
-            * * FLG is one byte.
-            *
-            * * DICTID is four bytes, and only present if FLG.FDICT is set.
-            *
-            * Sniff the content. Does it look like a zlib stream, with a CMF, etc? c.f. RFC1950,
-            * section 2.2. http://tools.ietf.org/html/rfc1950#page-4
-            *
-            * We need to see if it looks like a proper zlib stream, or whether it is just a deflate
-            * stream. RFC2616 calls zlib streams deflate. Confusing, isn't it? That's why some servers
-            * implement deflate Content-Encoding using deflate streams, rather than zlib streams.
-            *
-            * We could start looking at the bytes, but to be honest, someone else has already read
-            * the RFCs and implemented that for us. So we'll just use the JDK libraries and exception
-            * handling to do this. If that proves slow, then we could potentially change this to check
-            * the first byte - does it look like a CMF? What about the second byte - does it look like
-            * a FLG, etc.
-            */
 
-        /* We read a small buffer to sniff the content. */
+        final PushbackInputStream pushback = new PushbackInputStream(wrapped, 2);
-        final byte[] peeked = new byte[6];
+        final int i1 = pushback.read();
-
+        final int i2 = pushback.read();
-        final PushbackInputStream pushback = new PushbackInputStream(wrapped, peeked.length);
+        if (i1 == -1 || i2 == -1) {
-
+            throw new ZipException("Unexpected end of stream");
-        final int headerLength = pushback.read(peeked);
-
-        if (headerLength == -1) {
-            throw new IOException("Unable to read the response");
         }
 
-        /* We try to read the first uncompressed byte. */
+        pushback.unread(i2);
-        final byte[] dummy = new byte[1];
+        pushback.unread(i1);
 
-        final Inflater inf = new Inflater();
+        boolean nowrap = true;
-
+        final int b1 = i1 & 0xFF;
-        try {
+        final int compressionMethod = b1 & 0xF;
-            int n;
+        final int compressionInfo = b1 >> 4 & 0xF;
-            while ((n = inf.inflate(dummy)) == 0) {
+        final int b2 = i2 & 0xFF;
-                if (inf.finished()) {
+        if (compressionMethod == 8 && compressionInfo <= 7 && ((b1 << 8) | b2) % 31 == 0) {
-
+            nowrap = false;
-                    /* Not expecting this, so fail loudly. */
-                    throw new IOException("Unable to read the response");
-                }
-
-                if (inf.needsDictionary()) {
-
-                    /* Need dictionary - then it must be zlib stream with DICTID part? */
-                    break;
-                }
-
-                if (inf.needsInput()) {
-                    inf.setInput(peeked);
-                }
-            }
-
-            if (n == -1) {
-                throw new IOException("Unable to read the response");
-            }
-
-            /*
-                * We read something without a problem, so it's a valid zlib stream. Just need to reset
-                * and return an unused InputStream now.
-                */
-            pushback.unread(peeked, 0, headerLength);
-            sourceStream = new DeflateStream(pushback, new Inflater());
-        } catch (final DataFormatException e) {
-
-            /* Presume that it's an RFC1951 deflate stream rather than RFC1950 zlib stream and try
-                * again. */
-            pushback.unread(peeked, 0, headerLength);
-            sourceStream = new DeflateStream(pushback, new Inflater(true));
-        } finally {
-            inf.end();
         }
-
+        sourceStream = new DeflateStream(pushback, new Inflater(nowrap));
     }
 
-    /** Read a byte.
+    /**
-    */
+     * Read a byte.
+     */
     @Override
-    public int read()
+    public int read() throws IOException {
-        throws IOException
-    {
         return sourceStream.read();
     }
 
-    /** Read lots of bytes.
+    /**
-    */
+     * Read lots of bytes.
+     */
     @Override
-    public int read(final byte[] b)
+    public int read(final byte[] b) throws IOException {
-        throws IOException
-    {
         return sourceStream.read(b);
     }
 
-    /** Read lots of specific bytes.
+    /**
-    */
+     * Read lots of specific bytes.
+     */
     @Override
-    public int read(final byte[] b, final int off, final int len)
+    public int read(final byte[] b, final int off, final int len) throws IOException {
-        throws IOException
+        return sourceStream.read(b, off, len);
-    {
-        return sourceStream.read(b,off,len);
     }
 
-    /** Skip
+    /**
-    */
+     * Skip
+     */
     @Override
-    public long skip(final long n)
+    public long skip(final long n) throws IOException {
-        throws IOException
-    {
         return sourceStream.skip(n);
     }
 
-    /** Get available.
+    /**
-    */
+     * Get available.
+     */
     @Override
-    public int available()
+    public int available() throws IOException {
-        throws IOException
-    {
         return sourceStream.available();
     }
 
-    /** Mark.
+    /**
-    */
+     * Mark.
+     */
     @Override
-    public void mark(final int readLimit)
+    public void mark(final int readLimit) {
-    {
         sourceStream.mark(readLimit);
     }
 
-    /** Reset.
+    /**
-    */
+     * Reset.
+     */
     @Override
-    public void reset()
+    public void reset() throws IOException {
-        throws IOException
-    {
         sourceStream.reset();
     }
 
-    /** Check if mark is supported.
+    /**
-    */
+     * Check if mark is supported.
+     */
     @Override
-    public boolean markSupported()
+    public boolean markSupported() {
-    {
         return sourceStream.markSupported();
     }
 
-    /** Close.
+    /**
-    */
+     * Close.
+     */
     @Override
-    public void close()
+    public void close() throws IOException {
-        throws IOException
-    {
         sourceStream.close();
     }
 

httpclient/src/test/java/org/apache/http/client/entity/TestDeflate.java

@@ -0,0 +1,58 @@
+/*
+ * ====================================================================
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+
+package org.apache.http.client.entity;
+
+import java.util.zip.Deflater;
+
+import org.apache.http.Consts;
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.util.EntityUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestDeflate {
+
+    @Test
+    public void testCompressDecompress() throws Exception {
+
+        final String s = "some kind of text";
+        final byte[] input = s.getBytes(Consts.ASCII);
+
+        // Compress the bytes
+        final byte[] compressed = new byte[input.length * 2];
+        final Deflater compresser = new Deflater();
+        compresser.setInput(input);
+        compresser.finish();
+        final int len = compresser.deflate(compressed);
+
+        final HttpEntity entity = new DeflateDecompressingEntity(new ByteArrayEntity(compressed, 0, len));
+        Assert.assertEquals(s, EntityUtils.toString(entity));
+    }
+
+}

httpclient/src/test/java/org/apache/http/client/entity/TestGZip.java

@@ -29,15 +29,12 @@ package org.apache.http.client.entity;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.OutputStream;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.http.Consts;
 import org.apache.http.HttpEntity;
 import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.entity.ContentType;
-import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 import org.junit.Assert;
@@ -69,32 +66,6 @@ public class TestGZip {
         Assert.assertEquals("some kind of text", EntityUtils.toString(gunzipe, Consts.ASCII));
     }
 
-    @Test
-    public void testGzipDecompressingEntityDoesNotCrashInConstructorAndLeaveInputStreamOpen()
-            throws Exception {
-        final AtomicBoolean inputStreamIsClosed = new AtomicBoolean(false);
-        final HttpEntity in = new InputStreamEntity(new InputStream() {
-            @Override
-            public int read() throws IOException {
-                throw new IOException("An exception occurred");
-            }
-
-            @Override
-            public void close() throws IOException {
-                inputStreamIsClosed.set(true);
-            }
-
-        }, 123);
-        final GzipDecompressingEntity gunzipe = new GzipDecompressingEntity(in);
-        try {
-            gunzipe.getContent();
-        } catch (final IOException e) {
-            // As I cannot get the content, GzipDecompressingEntity is supposed
-            // to have released everything
-            Assert.assertTrue(inputStreamIsClosed.get());
-        }
-    }
-
     @Test
     public void testCompressionIOExceptionLeavesOutputStreamOpen() throws Exception {
         final HttpEntity in = Mockito.mock(HttpEntity.class);