Issue #2815 HPack Opaque Bytes

HPack as specified allows for values to be opaque bytes, even though HTTP only allows ISO-8859-1 within a header.
This updates HPack, so that strings are converted to UTF-8 and then encoded as bytes iff non ISO-8859-1 characters are discovered.

Signed-off-by: Greg Wilkins <gregw@webtide.com>

jetty-http2/http2-hpack/src/main/java/org/eclipse/jetty/http2/hpack/HpackEncoder.java

@@ -20,6 +20,7 @@
 package org.eclipse.jetty.http2.hpack;
 
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.EnumSet;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -390,20 +391,39 @@ public class HpackEncoder
         if (huffman)
         {
             // huffman literal value
-            buffer.put((byte)0x80);
-            NBitInteger.encode(buffer,7,Huffman.octetsNeeded(value));
-            Huffman.encode(buffer,value);
+            buffer.put((byte)0x80).mark();
+
+            try
+            {
+                NBitInteger.encode(buffer,7,Huffman.octetsNeeded(value));
+                Huffman.encode(buffer,value);
+            }
+            catch(Throwable th)
+            {
+                LOG.ignore(th);
+                byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+
+                NBitInteger.encode(buffer,7,Huffman.octetsNeeded(bytes));
+                Huffman.encode(buffer,bytes);
+            }
         }
         else
         {
             // add literal assuming iso_8859_1
-            buffer.put((byte)0x00);
+            buffer.put((byte)0x00).mark();
             NBitInteger.encode(buffer,7,value.length());
             for (int i=0;i<value.length();i++)
             {
                 char c=value.charAt(i);
                 if (c<' '|| c>127)
-                    throw new IllegalArgumentException();
+                {
+                    // Not iso_8859_1, so re-encode remaining as UTF-8
+                    buffer.reset();
+                    byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
+                    NBitInteger.encode(buffer,7,bytes.length);
+                    buffer.put(bytes,0,bytes.length);
+                    return;
+                }
                 buffer.put((byte)c);
             }
         }

jetty-http2/http2-hpack/src/main/java/org/eclipse/jetty/http2/hpack/Huffman.java

@@ -20,6 +20,8 @@ package org.eclipse.jetty.http2.hpack;
 
 import java.nio.ByteBuffer;
 
+import org.eclipse.jetty.util.Utf8StringBuilder;
+
 public class Huffman
 {
 
@@ -352,7 +354,7 @@ public class Huffman
 
     public static String decode(ByteBuffer buffer,int length) throws HpackException.CompressionException
     {        
-        StringBuilder out = new StringBuilder(length*2);
+        Utf8StringBuilder utf8 = new Utf8StringBuilder(length*2);
         int node = 0;
         int current = 0;
         int bits = 0;
@@ -379,7 +381,7 @@ public class Huffman
                         throw new HpackException.CompressionException("EOS in content");
 
                     // terminal node
-                    out.append(rowsym[node]);
+                    utf8.append((byte)(0xFF&rowsym[node]));
                     bits -= rowbits[node];
                     node = 0;
                 } 
@@ -410,7 +412,7 @@ public class Huffman
                 break;
             }
 
-            out.append(rowsym[node]);
+            utf8.append((byte)(0xFF&rowsym[node]));
             bits -= rowbits[node];
             node = 0;
         }
@@ -418,7 +420,7 @@ public class Huffman
         if(node != 0)
             throw new HpackException.CompressionException("Bad termination");
 
-        return out.toString();
+        return utf8.toString();
     }
 
     public static int octetsNeeded(String s)
@@ -426,11 +428,21 @@ public class Huffman
         return octetsNeeded(CODES,s);
     }
     
+    public static int octetsNeeded(byte[] b)
+    {   
+        return octetsNeeded(CODES,b);
+    }
+    
     public static void encode(ByteBuffer buffer,String s)
     {
         encode(CODES,buffer,s);
     }
     
+    public static void encode(ByteBuffer buffer,byte[] b)
+    {
+        encode(CODES,buffer,b);
+    }
+    
     public static int octetsNeededLC(String s)
     {
         return octetsNeeded(LCCODES,s);
@@ -456,6 +468,18 @@ public class Huffman
         return (needed+7) / 8;
     }
 
+    private static int octetsNeeded(final int[][] table,byte[] b)
+    {   
+        int needed=0;
+        int len = b.length;
+        for (int i=0;i<len;i++)
+        {
+            int c=0xFF&b[i];
+            needed += table[c][1];
+        }
+        return (needed+7) / 8;
+    }
+
     private static void encode(final int[][] table,ByteBuffer buffer,String s)
     {
         long current = 0;
@@ -494,4 +518,39 @@ public class Huffman
         buffer.position(p-buffer.arrayOffset());
     }
 
+    private static void encode(final int[][] table,ByteBuffer buffer,byte[] b)
+    {
+        long current = 0;
+        int n = 0;
+
+        byte[] array = buffer.array();
+        int p=buffer.arrayOffset()+buffer.position();
+
+        int len = b.length;
+        for (int i=0;i<len;i++)
+        {
+            int c=0xFF&b[i];
+            int code = table[c][0];
+            int bits = table[c][1];
+
+            current <<= bits;
+            current |= code;
+            n += bits;
+
+            while (n >= 8) 
+            {
+                n -= 8;
+                array[p++]=(byte)(current >> n);
+            }
+        }
+
+        if (n > 0) 
+        {
+          current <<= (8 - n);
+          current |= (0xFF >>> n); 
+          array[p++]=(byte)current;
+        }
+        
+        buffer.position(p-buffer.arrayOffset());
+    }
 }

jetty-http2/http2-hpack/src/test/java/org/eclipse/jetty/http2/hpack/HpackTest.java

@@ -136,9 +136,30 @@ public class HpackTest
         {
             assertThat(e.getMessage(),containsString("Header too large"));
         }
-
     }
     
+
+    @Test
+    public void encodeDecodeNonAscii() throws Exception
+    {
+        HpackEncoder encoder = new HpackEncoder();
+        HpackDecoder decoder = new HpackDecoder(4096,8192);
+        ByteBuffer buffer = BufferUtil.allocate(16*1024);
+        
+        HttpFields fields0 = new HttpFields();
+        fields0.add("Cookie","[\uD842\uDF9F]");
+        fields0.add("custom-key","[\uD842\uDF9F]");
+        Response original0 = new MetaData.Response(HttpVersion.HTTP_2,200,fields0);
+        
+        BufferUtil.clearToFill(buffer);
+        encoder.encode(buffer,original0);
+        BufferUtil.flipToFlush(buffer,0);
+        Response decoded0 = (Response)decoder.decode(buffer);
+        
+        assertMetadataSame(original0,decoded0);
+    }
+    
+
     @Test
     public void evictReferencedFieldTest() throws Exception
     {