near 100% test coverage of NBitInteger, Huffman and HpackContext

jetty-hpack/src/main/java/org/eclipse/jetty/hpack/HpackContext.java

@@ -20,10 +20,11 @@ package org.eclipse.jetty.hpack;
 
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map;
+import java.util.NoSuchElementException;
 import java.util.Set;
 
-import org.eclipse.jetty.hpack.Field.NameKey;
 import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.util.ArrayQueue;
 import org.eclipse.jetty.util.ArrayTrie;
@@ -108,8 +109,7 @@ public class HpackContext
         Set<String> added = new HashSet<>();
         for (int i=1;i<STATIC_TABLE.length;i++)
         {
-            Entry entry=new Entry(i,STATIC_TABLE[i][0],STATIC_TABLE[i][1],true);
+            Entry entry=__staticTable[i]=new Entry(i,STATIC_TABLE[i][0],STATIC_TABLE[i][1],true);
-            __staticTable[i]=entry;
             if (entry._field.getValue()!=null)
                 __staticFieldMap.put(entry._field,entry);
             if (!added.contains(entry._field.getName()))
@@ -120,33 +120,46 @@ public class HpackContext
         }
     }
     
-    private int _maxHeaderTableSize;
+    private int _maxHeaderTableSizeInBytes;
-    private int _headerTableSize;
+    private int _headerTableSizeInBytes;
     private final Entry _refSet=new Entry(true);
-    private final ArrayQueue<Entry> _headerTable;
+    private final HeaderTable _headerTable;
     private final Map<HttpField,Entry> _fieldMap = new HashMap<>();
     private final Map<String,Entry> _nameMap = new HashMap<>();
+    private Iterable<Entry> referenceSet = new Iterable<Entry>()
+    {
+        @Override
+        public Iterator<Entry> iterator()
+        {
+            return iterateReferenceSet();
+        }
+    };
     
     
     HpackContext(int maxHeaderTableSize)
     {
-        _maxHeaderTableSize=maxHeaderTableSize;
+        _maxHeaderTableSizeInBytes=maxHeaderTableSize;
         int guesstimateEntries = 10+maxHeaderTableSize/(32+10+10);
         _headerTable=new HeaderTable(guesstimateEntries,guesstimateEntries+10);
     }
     
+    public void resize(int maxHeaderTableSize)
+    {
+        _maxHeaderTableSizeInBytes=maxHeaderTableSize;
+        int guesstimateEntries = 10+maxHeaderTableSize/(32+10+10);
+        evict();
+        _headerTable.resizeUnsafe(guesstimateEntries);
+    }
+    
     public Entry get(HttpField field)
     {
-        System.err.println(field);
-        System.err.println(_fieldMap);
-        System.err.println(__staticFieldMap);
         Entry entry = _fieldMap.get(field);
         if (entry==null)
             entry=__staticFieldMap.get(field);
         return entry;
     }
     
-    public Entry getNameEntry(String name)
+    public Entry get(String name)
     {
         Entry entry = __staticNameMap.get(name);
         if (entry!=null)
@@ -154,14 +167,27 @@ public class HpackContext
         return _nameMap.get(StringUtil.asciiToLowerCase(name));
     }
     
+    public Entry get(int index)
+    {
+        index=index-_headerTable.size();
+        if (index>0)
+        {
+            if (index>=__staticTable.length)
+                return null;
+            return __staticTable[index];
+        }
+        
+        return _headerTable.getUnsafe(-index);
+    }
+    
     public Entry add(HttpField field)
     {
         int i=_headerTable.getNextIndexUnsafe();
         Entry entry=new Entry(i,field,false);
         int size = entry.getSize();
-        if (size>_maxHeaderTableSize)
+        if (size>_maxHeaderTableSizeInBytes)
             return null;
-        _headerTableSize+=size;
+        _headerTableSizeInBytes+=size;
         _headerTable.addUnsafe(entry);
         _fieldMap.put(field,entry);
         _nameMap.put(StringUtil.asciiToLowerCase(field.getName()),entry);
@@ -169,14 +195,72 @@ public class HpackContext
         evict();
         return entry;
     }
-    
+
-    public void evict()
+    public Object size()
     {
-        while (_headerTableSize>_maxHeaderTableSize)
+        return _headerTable.size();
+    }
+    
+    public int index(Entry entry)
+    {
+        if (entry._index<0)
+            return 0;
+        if (entry.isStatic())
+            return _headerTable.size() + entry._index;
+        return _headerTable.index(entry);
+    }
+    
+    public void addToRefSet(Entry entry)
+    {
+        entry.addToRefSet(this);
+    }
+    
+    public Iterable<Entry> getReferenceSet()
+    {
+        return referenceSet;
+    }
+    
+    public Iterator<Entry> iterateReferenceSet()
+    {
+        return new Iterator<Entry>()
+        {
+            Entry _next = _refSet._refSetNext;
+
+            @Override
+            public boolean hasNext()
+            {
+                return _next!=_refSet;
+            }
+
+            @Override
+            public Entry next()
+            {
+                if (_next==_refSet)
+                    throw new NoSuchElementException();
+                Entry next=_next;
+                _next=_next._refSetNext;
+                return next;
+            }
+
+            @Override
+            public void remove()
+            {
+                if (_next._refSetPrev==_refSet)
+                    throw new NoSuchElementException();
+                    
+                _next._refSetPrev.removeFromRefSet();
+            }
+        };
+    }
+    
+    private void evict()
+    {
+        while (_headerTableSizeInBytes>_maxHeaderTableSizeInBytes)
         {
             Entry entry = _headerTable.pollUnsafe();
-            _headerTableSize-=entry.getSize();
+            _headerTableSizeInBytes-=entry.getSize();
             entry.removeFromRefSet();
+            entry._index=-1;
             _fieldMap.remove(entry.getHttpField());
             String lc=StringUtil.asciiToLowerCase(entry.getHttpField().getName());
             if (entry==_nameMap.get(lc))
@@ -207,10 +291,10 @@ public class HpackContext
          * @see org.eclipse.jetty.util.ArrayQueue#growUnsafe()
          */
         @Override
-        protected void growUnsafe(int newCapacity)
+        protected void resizeUnsafe(int newCapacity)
         {
             // Relay on super.growUnsafe to pack all entries 0 to _nextSlot
-            super.growUnsafe(newCapacity);
+            super.resizeUnsafe(newCapacity);
             for (int i=0;i<_nextSlot;i++)
                 ((Entry)_elements[i])._index=i;
         }
@@ -234,6 +318,17 @@ public class HpackContext
         {
             return super.dequeue();
         }
+        
+        /* ------------------------------------------------------------ */
+        /**
+         * @param entry
+         * @return
+         */
+        private int index(Entry entry)
+        {
+            return entry._index>=_nextE?_size-entry._index+_nextE:_nextSlot-entry._index;
+        }
+
     }
 
 
@@ -271,8 +366,15 @@ public class HpackContext
             _field=field;
         }
         
-        public void addToRefSet(HpackContext ctx)
+        private void addToRefSet(HpackContext ctx)
         {
+            if (_static)
+                throw new IllegalStateException("static");
+            if (_index<0)
+                throw new IllegalStateException("evicted");
+            if (_refSetNext!=this)
+                return;
+            
             _refSetNext=ctx._refSet;
             _refSetPrev=ctx._refSet._refSetPrev;
             ctx._refSet._refSetPrev._refSetNext=this;
@@ -316,9 +418,7 @@ public class HpackContext
             return String.format("{%s,%d,%s,%x}",_static?"S":"D",_index,_field,hashCode());
         }
     }
-    
+
-    
+
-    
-    
 
 }

jetty-hpack/src/main/java/org/eclipse/jetty/hpack/Huffman.java

@@ -316,8 +316,6 @@ public class Huffman
             {
                 len -= 8;
                 int i = ((code >>> len) & 0xFF);
-                if (rowbits[current]!=0)
-                    throw new IllegalStateException("invalid dictionary: prefix not unique");
 
                 int t=current*256+i;
                 current = tree[t];
@@ -404,7 +402,7 @@ public class Huffman
         for (int i=0;i<len;i++)
         {
             char c=s.charAt(i);
-            if (c>=128)
+            if (c>=128 || c<' ')
                 throw new IllegalArgumentException();
             needed += CODES[c][1];
         }
@@ -424,7 +422,7 @@ public class Huffman
         for (int i=0;i<len;i++)
         {
             char c=s.charAt(i);
-            if (c>=128)
+            if (c>=128 || c<' ')
                 throw new IllegalArgumentException();
             int code = CODES[c][0];
             int bits = CODES[c][1];

jetty-hpack/src/main/java/org/eclipse/jetty/hpack/NBitInteger.java

@@ -24,12 +24,25 @@ public class NBitInteger
 {
     public static int octectsNeeded(int n,int i)
     {
+        if (n==8)
+        {
+            int nbits = 0xFF;
+            i=i-nbits;
+            if (i<0)
+                return 1;
+            if (i==0)
+                return 2;
+            int lz=Integer.numberOfLeadingZeros(i);
+            int log=32-lz;
+            return 1+(log+6)/7;
+        }
+        
         int nbits = 0xFF >>> (8 - n);
         i=i-nbits;
         if (i<0)
-            return n==8?1:0;
+            return 0;
         if (i==0)
-            return n==8?2:1;
+            return 1;
         int lz=Integer.numberOfLeadingZeros(i);
         int log=32-lz;
         return (log+6)/7;
@@ -96,9 +109,30 @@ public class NBitInteger
 
     public static int decode(ByteBuffer buf, int n)
     {
+        if (n==8)
+        {
+            int nbits = 0xFF;
+
+            int i=buf.get()&0xff;
+            
+            if (i == nbits)
+            {       
+                int m=1;
+                int b;
+                do
+                {
+                    b = 0xff&buf.get();
+                    i = i + (b&127) * m;
+                    m = m*128;
+                }
+                while ((b&128) == 128);
+            }
+            return i;
+        }
+        
         int nbits = 0xFF >>> (8 - n);
 
-        int i=buf.get(buf.position()-(n==8?0:1))&nbits;
+        int i=buf.get(buf.position()-1)&nbits;
         
         if (i == nbits)
         {       

jetty-hpack/src/test/java/org/eclipse/jetty/hpack/HpackContextTest.java

@@ -24,9 +24,15 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
 
 import org.eclipse.jetty.hpack.HpackContext.Entry;
 import org.eclipse.jetty.http.HttpField;
+import org.hamcrest.Matchers;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -41,9 +47,10 @@ public class HpackContextTest
     public void testStaticName()
     {
         HpackContext ctx = new HpackContext(4096);
-        Entry entry=ctx.getNameEntry(":method");
+        Entry entry=ctx.get(":method");
         assertEquals(":method",entry.getHttpField().getName());
         Assert.assertTrue(entry.isStatic());
+        Assert.assertThat(entry.toString(),Matchers.startsWith("{S,2,:method: "));
     }
     
     @Test
@@ -67,7 +74,9 @@ public class HpackContextTest
     {
         HpackContext ctx = new HpackContext(38);
         HttpField field = new HttpField("foo","bar");
-        Assert.assertNotNull(ctx.add(field));
+        Entry entry=ctx.add(field);
+        Assert.assertNotNull(entry);
+        Assert.assertThat(entry.toString(),Matchers.startsWith("{D,0,foo: bar,"));
     }
     
     @Test
@@ -77,18 +86,53 @@ public class HpackContextTest
         HttpField field0 = new HttpField("foo","bar");
         
         assertEquals(field0,ctx.add(field0).getHttpField());
-        assertEquals(field0,ctx.getNameEntry("foo").getHttpField());
+        assertEquals(field0,ctx.get("foo").getHttpField());
         
         HttpField field1 = new HttpField("xxx","yyy");
         assertEquals(field1,ctx.add(field1).getHttpField());
 
         assertNull(ctx.get(field0));
-        assertNull(ctx.getNameEntry("foo"));
+        assertNull(ctx.get("foo"));
         assertEquals(field1,ctx.get(field1).getHttpField());
-        assertEquals(field1,ctx.getNameEntry("xxx").getHttpField());
+        assertEquals(field1,ctx.get("xxx").getHttpField());
         
     }
-    
+
+    @Test
+    public void testEvictNames()
+    {
+        HpackContext ctx = new HpackContext(38*2);
+        HttpField[] field = 
+        {
+           new HttpField("name","v0"),
+           new HttpField("name","v1"),
+           new HttpField("name","v2"),
+           new HttpField("name","v3"),
+           new HttpField("name","v4"),
+           new HttpField("name","v5"),
+        };
+        
+        Entry[] entry = new Entry[field.length];
+        
+        // Add 2 name entries to fill table
+        for (int i=0;i<=1;i++)
+            entry[i]=ctx.add(field[i]);
+        
+        // check there is a name reference and it is the most recent added
+        assertEquals(entry[1],ctx.get("name"));
+
+        // Add 1 other entry to table and evict 1
+        ctx.add(new HttpField("xxx","yyy"));
+        
+        // check the name reference has been not been evicted
+        assertEquals(entry[1],ctx.get("name"));
+        
+        // Add 1 other entry to table and evict 1
+        ctx.add(new HttpField("foo","bar"));
+        
+        // name is evicted
+        assertNull(ctx.get("name"));
+    }
     @Test
     public void testGetAddStatic()
     {
@@ -116,5 +160,446 @@ public class HpackContextTest
         assertFalse(ctx.get(methodGet).isStatic());
         assertFalse(e0==e1);
     }
+    
+    @Test
+    public void testGetAddStaticName()
+    {
+        HpackContext ctx = new HpackContext(4096);
+        HttpField methodOther = new HttpField(":method","OTHER");
 
+        // Look for the field by name.  Should find static version.
+        assertEquals(":method",ctx.get(":method").getHttpField().getName());
+        assertTrue(ctx.get(":method").isStatic());
+        
+        // Add dynamic entry with method
+        ctx.add(methodOther);
+        
+        // Look for the field by name.  Should find static version.
+        assertEquals(":method",ctx.get(":method").getHttpField().getName());
+        assertTrue(ctx.get(":method").isStatic()); 
+    }
+
+    @Test
+    public void testIndexes()
+    {
+        // Only enough space for 5 entries
+        HpackContext ctx = new HpackContext(38*5);
+        
+        HttpField methodPost = new HttpField(":method","POST");
+        HttpField[] field = 
+        {
+           new HttpField("fo0","b0r"),
+           new HttpField("fo1","b1r"),
+           new HttpField("fo2","b2r"),
+           new HttpField("fo3","b3r"),
+           new HttpField("fo4","b4r"),
+           new HttpField("fo5","b5r"),
+           new HttpField("fo6","b6r"),
+           new HttpField("fo7","b7r"),
+           new HttpField("fo8","b8r"),
+           new HttpField("fo9","b9r"),
+           new HttpField("foA","bAr"),
+        };
+        
+        Entry[] entry = new Entry[100];
+        
+        // Lookup the index of a static field
+        assertEquals(0,ctx.size());
+        assertEquals(":authority",ctx.get(1).getHttpField().getName());
+        assertEquals(3,ctx.index(ctx.get(methodPost)));
+        assertEquals(methodPost,ctx.get(3).getHttpField());
+        assertEquals("www-authenticate",ctx.get(61).getHttpField().getName());
+        assertEquals(null,ctx.get(62));
+        
+        // Add a single entry  
+        entry[0]=ctx.add(field[0]);
+        
+        // Check new entry is 1 
+        assertEquals(1,ctx.size());
+        assertEquals(1,ctx.index(entry[0]));
+        assertEquals(entry[0],ctx.get(1));
+        
+        // and statics have moved up 1
+        assertEquals(":authority",ctx.get(1+1).getHttpField().getName());
+        assertEquals(3+1,ctx.index(ctx.get(methodPost)));
+        assertEquals(methodPost,ctx.get(3+1).getHttpField());
+        assertEquals("www-authenticate",ctx.get(61+1).getHttpField().getName());
+        assertEquals(null,ctx.get(62+1));
+        
+
+        // Add 4 more entries
+        for (int i=1;i<=4;i++)  
+            entry[i]=ctx.add(field[i]);
+
+        // Check newest entry is at 1 oldest at 5
+        assertEquals(5,ctx.size());
+        int index=5;
+        for (int i=0;i<=4;i++)  
+        {
+            assertEquals(index,ctx.index(entry[i]));
+            assertEquals(entry[i],ctx.get(index));
+            index--;
+        }
+
+        // and statics have moved up 5
+        assertEquals(":authority",ctx.get(1+5).getHttpField().getName());
+        assertEquals(3+5,ctx.index(ctx.get(methodPost)));
+        assertEquals(methodPost,ctx.get(3+5).getHttpField());
+        assertEquals("www-authenticate",ctx.get(61+5).getHttpField().getName());
+        assertEquals(null,ctx.get(62+5));
+        
+        // add 1 more entry and this should cause an eviction!
+        entry[5]=ctx.add(field[5]);
+
+        // Check newest entry is at 1 oldest at 5
+        index=5;
+        for (int i=1;i<=5;i++)  
+        {
+            assertEquals(index,ctx.index(entry[i]));
+            assertEquals(entry[i],ctx.get(index));
+            index--;
+        }
+        // check entry 0 evicted
+        assertNull(ctx.get(field[0]));
+        assertEquals(0,ctx.index(entry[0]));
+
+        // and statics have moved up just 5
+        assertEquals(":authority",ctx.get(1+5).getHttpField().getName());
+        assertEquals(3+5,ctx.index(ctx.get(methodPost)));
+        assertEquals(methodPost,ctx.get(3+5).getHttpField());
+        assertEquals("www-authenticate",ctx.get(61+5).getHttpField().getName());
+        assertEquals(null,ctx.get(62+5));
+        
+        // Add 4 more entries
+        for (int i=6;i<=9;i++)  
+            entry[i]=ctx.add(field[i]);
+        
+        // Check newest entry is at 1 oldest at 5
+        index=5;
+        for (int i=5;i<=9;i++)  
+        {
+            assertEquals(index,ctx.index(entry[i]));
+            assertEquals(entry[i],ctx.get(index));
+            index--;
+        }
+        // check entry 0-4 evicted
+        for (int i=0;i<=4;i++) 
+        {
+            assertNull(ctx.get(field[i]));
+            assertEquals(0,ctx.index(entry[i]));
+        }
+        
+
+        // Add new entries enough so that array queue will wrap
+        for (int i=10;i<=52;i++)
+            entry[i]=ctx.add(new HttpField("n"+i,"v"+i));
+
+        index=5;
+        for (int i=48;i<=52;i++)  
+        {
+            assertEquals(index,ctx.index(entry[i]));
+            assertEquals(entry[i],ctx.get(index));
+            index--;
+        }
+    }
+    
+    
+    @Test
+    public void testRefSetAddStatic()
+    {
+        try
+        {
+            HpackContext ctx = new HpackContext(4096);
+
+            HttpField methodGet = new HttpField(":method","GET");
+            Entry entry = ctx.get(methodGet);
+            ctx.addToRefSet(entry);
+            fail();
+        }
+        catch(IllegalStateException e)
+        {}
+    }
+    
+    @Test
+    public void testRefSetAddEvicted()
+    {
+        try
+        {
+            HpackContext ctx = new HpackContext(38);
+            HttpField field0 = new HttpField("foo","bar");
+            HttpField field1 = new HttpField("xxx","yyy");
+            Entry entry = ctx.add(field0);
+            ctx.add(field1);
+            ctx.addToRefSet(entry);
+            fail();
+        }
+        catch(IllegalStateException e)
+        {}
+    }
+
+    @Test
+    public void testRefSetEmptyIteration()
+    {
+        HpackContext ctx = new HpackContext(4096);
+        for (Entry entry: ctx.getReferenceSet() )
+            fail("unexpected:"+entry);
+        Iterator<Entry> iter = ctx.iterateReferenceSet();
+        assertFalse(iter.hasNext());
+        
+        try
+        {
+            iter.next();
+            fail();
+        }
+        catch(NoSuchElementException e)
+        {
+        }
+        
+        try
+        {
+            iter.remove();
+            fail();
+        }
+        catch(NoSuchElementException e)
+        {
+        }
+        
+    }
+    
+
+    @Test
+    public void testRefSet()
+    {
+        // Only enough space for 5 entries
+        HpackContext ctx = new HpackContext(38*5);
+        
+        HttpField[] field = 
+        {
+           new HttpField("fo0","b0r"),
+           new HttpField("fo1","b1r"),
+           new HttpField("fo2","b2r"),
+           new HttpField("fo3","b3r"),
+           new HttpField("fo4","b4r"),
+           new HttpField("fo5","b5r"),
+           new HttpField("fo6","b6r"),
+           new HttpField("fo7","b7r"),
+           new HttpField("fo8","b8r"),
+           new HttpField("fo9","b9r"),
+           new HttpField("foA","bAr"),
+        };
+        Entry[] entry = new Entry[field.length];
+        
+        // Add 5 entries
+        for (int i=0;i<=4;i++)  
+            entry[i]=ctx.add(field[i]);
+        
+        // Add 3 entries to reference set
+        ctx.addToRefSet(ctx.get(3));
+        ctx.addToRefSet(ctx.get(1));
+        ctx.addToRefSet(ctx.get(5));
+
+        // iterate ref set
+        HashSet<HttpField> fields = new HashSet<>();
+        for (Entry e: ctx.getReferenceSet() )
+            fields.add(e.getHttpField());
+        assertEquals(3,fields.size());
+        assertTrue(fields.contains(field[0]));
+        assertTrue(fields.contains(field[2]));
+        assertTrue(fields.contains(field[4]));
+        
+        // duplicate add ignored
+        ctx.addToRefSet(ctx.get(1));
+        fields.clear();
+        for (Entry e: ctx.getReferenceSet() )
+            fields.add(e.getHttpField());
+        assertEquals(3,fields.size());
+        assertTrue(fields.contains(field[0]));
+        assertTrue(fields.contains(field[2]));
+        assertTrue(fields.contains(field[4]));
+        
+        // remove entry
+        ctx.get(3).removeFromRefSet();
+        fields.clear();
+        for (Entry e: ctx.getReferenceSet() )
+            fields.add(e.getHttpField());
+        assertEquals(2,fields.size());
+        assertTrue(fields.contains(field[0]));
+        assertTrue(fields.contains(field[4]));
+
+        // iterator remove
+        Iterator<Entry> iter=ctx.iterateReferenceSet();
+        iter.next();
+        iter.remove();
+        fields.clear();
+        for (Entry e: ctx.getReferenceSet() )
+            fields.add(e.getHttpField());
+        assertEquals(1,fields.size());
+
+        // Add 5 new entries to cause evictions
+        for (int i=5;i<=9;i++)  
+            entry[i]=ctx.add(field[i]);
+        fields.clear();
+        for (Entry e: ctx.getReferenceSet() )
+            fields.add(e.getHttpField());
+        assertEquals(0,fields.size());
+        
+    }
+
+    @Test
+    public void testResize()
+    {
+        // Only enough space for 5 entries
+        HpackContext ctx = new HpackContext(38*5);
+        HttpField methodPost = new HttpField(":method","POST");
+        
+        HttpField[] field = 
+        {
+           new HttpField("fo0","b0r"),
+           new HttpField("fo1","b1r"),
+           new HttpField("fo2","b2r"),
+           new HttpField("fo3","b3r"),
+           new HttpField("fo4","b4r"),
+           new HttpField("fo5","b5r"),
+           new HttpField("fo6","b6r"),
+           new HttpField("fo7","b7r"),
+           new HttpField("fo8","b8r"),
+           new HttpField("fo9","b9r"),
+           new HttpField("foA","bAr"),
+        };
+        Entry[] entry = new Entry[field.length];
+        
+        // Add 5 entries
+        for (int i=0;i<=4;i++)  
+            entry[i]=ctx.add(field[i]);
+        
+        assertEquals(5,ctx.size());
+        
+        // check indexes
+        int index=5;
+        for (int i=0;i<=4;i++)  
+        {
+            assertEquals(index,ctx.index(entry[i]));
+            assertEquals(entry[i],ctx.get(index));
+            index--;
+        }
+
+        // and statics have moved up 5
+        assertEquals(":authority",ctx.get(1+5).getHttpField().getName());
+        assertEquals(3+5,ctx.index(ctx.get(methodPost)));
+        assertEquals(methodPost,ctx.get(3+5).getHttpField());
+        assertEquals("www-authenticate",ctx.get(61+5).getHttpField().getName());
+        assertEquals(null,ctx.get(62+5));
+        
+        
+        // Add 3 entries to reference set
+        ctx.addToRefSet(ctx.get(3));
+        ctx.addToRefSet(ctx.get(1));
+        ctx.addToRefSet(ctx.get(5));
+
+        
+        
+        // iterate ref set
+        HashSet<HttpField> fields = new HashSet<>();
+        for (Entry e: ctx.getReferenceSet() )
+            fields.add(e.getHttpField());
+        assertEquals(3,fields.size());
+        assertTrue(fields.contains(field[0]));
+        assertTrue(fields.contains(field[2]));
+        assertTrue(fields.contains(field[4]));
+        
+        // resize so that only 2 entries may be held
+        ctx.resize(38*2);
+        assertEquals(2,ctx.size());
+        
+        // check indexes
+        index=2;
+        for (int i=3;i<=4;i++)  
+        {
+            assertEquals(index,ctx.index(entry[i]));
+            assertEquals(entry[i],ctx.get(index));
+            index--;
+        }
+        
+        // and statics have moved up 2
+        assertEquals(":authority",ctx.get(1+2).getHttpField().getName());
+        assertEquals(3+2,ctx.index(ctx.get(methodPost)));
+        assertEquals(methodPost,ctx.get(3+2).getHttpField());
+        assertEquals("www-authenticate",ctx.get(61+2).getHttpField().getName());
+        assertEquals(null,ctx.get(62+2));
+        
+        // check reference set 
+        fields.clear();
+        for (Entry e: ctx.getReferenceSet() )
+            fields.add(e.getHttpField());
+        assertEquals(1,fields.size());
+        assertFalse(fields.contains(field[0]));
+        assertFalse(fields.contains(field[2]));
+        assertTrue(fields.contains(field[4]));
+        
+
+        // resize so that 6.5 entries may be held
+        ctx.resize(38*6+19);
+        assertEquals(2,ctx.size());
+        
+
+        // check indexes
+        index=2;
+        for (int i=3;i<=4;i++)  
+        {
+            assertEquals(index,ctx.index(entry[i]));
+            assertEquals(entry[i],ctx.get(index));
+            index--;
+        }
+        
+        // and statics have moved up 2
+        assertEquals(":authority",ctx.get(1+2).getHttpField().getName());
+        assertEquals(3+2,ctx.index(ctx.get(methodPost)));
+        assertEquals(methodPost,ctx.get(3+2).getHttpField());
+        assertEquals("www-authenticate",ctx.get(61+2).getHttpField().getName());
+        assertEquals(null,ctx.get(62+2));
+        
+        // check reference set 
+        fields.clear();
+        for (Entry e: ctx.getReferenceSet() )
+            fields.add(e.getHttpField());
+        assertEquals(1,fields.size());
+        assertFalse(fields.contains(field[0]));
+        assertFalse(fields.contains(field[2]));
+        assertTrue(fields.contains(field[4]));
+        
+
+        // Add 5 entries
+        for (int i=5;i<=9;i++)  
+            entry[i]=ctx.add(field[i]);
+        
+        assertEquals(6,ctx.size());
+
+        // check indexes
+        index=6;
+        for (int i=4;i<=9;i++)  
+        {
+            assertEquals(index,ctx.index(entry[i]));
+            assertEquals(entry[i],ctx.get(index));
+            index--;
+        }
+        
+        // and statics have moved up 6
+        assertEquals(":authority",ctx.get(1+6).getHttpField().getName());
+        assertEquals(3+6,ctx.index(ctx.get(methodPost)));
+        assertEquals(methodPost,ctx.get(3+6).getHttpField());
+        assertEquals("www-authenticate",ctx.get(61+6).getHttpField().getName());
+        assertEquals(null,ctx.get(62+6));
+        
+        // check reference set 
+        fields.clear();
+        for (Entry e: ctx.getReferenceSet() )
+            fields.add(e.getHttpField());
+        assertEquals(1,fields.size());
+        assertFalse(fields.contains(field[0]));
+        assertFalse(fields.contains(field[2]));
+        assertTrue(fields.contains(field[4]));
+        
+
+        
+    }
 }

jetty-hpack/src/test/java/org/eclipse/jetty/hpack/HuffmanTest.java

@@ -51,6 +51,17 @@ public class HuffmanTest
         }
     }
     
+    @Test
+    public void testDecodeTrailingFF() throws Exception
+    {
+        for (String[] test:tests)
+        {
+            byte[] encoded=TypeUtil.fromHexString(test[1]+"FF");
+            String decoded=Huffman.decode(ByteBuffer.wrap(encoded));
+            Assert.assertEquals(test[0],test[2],decoded);
+        }
+    }
+    
     @Test
     public void testEncode() throws Exception
     {
@@ -62,8 +73,37 @@ public class HuffmanTest
             BufferUtil.flipToFlush(buf,pos);
             String encoded=TypeUtil.toHexString(BufferUtil.toArray(buf)).toLowerCase();
             Assert.assertEquals(test[0],test[1],encoded);
+            Assert.assertEquals(test[1].length()/2,Huffman.octetsNeeded(test[2]));
+        }
+    }
+
+    @Test
+    public void testEncode8859Only() throws Exception
+    {
+        char bad[] = {(char)128,(char)0,(char)-1,' '-1};
+        for (int i=0;i<bad.length;i++)
+        {
+            String s="bad '"+bad[i]+"'";
+            
+            try
+            {
+                Huffman.octetsNeeded(s);
+                Assert.fail("i="+i);
+            }
+            catch(IllegalArgumentException e)
+            {
+            }
+            
+            try
+            {
+                Huffman.encode(BufferUtil.allocate(32),s);
+                Assert.fail("i="+i);
+            }
+            catch(IllegalArgumentException e)
+            {
+            }
         }
-        
     }
     
+    
 }

jetty-hpack/src/test/java/org/eclipse/jetty/hpack/NBitIntegerTest.java

@@ -36,6 +36,7 @@ public class NBitIntegerTest
         assertEquals(0,NBitInteger.octectsNeeded(5,10));
         assertEquals(2,NBitInteger.octectsNeeded(5,1337));
         assertEquals(1,NBitInteger.octectsNeeded(8,42));
+        assertEquals(3,NBitInteger.octectsNeeded(8,1337));
 
         assertEquals(0,NBitInteger.octectsNeeded(6,62));
         assertEquals(1,NBitInteger.octectsNeeded(6,63));
@@ -64,19 +65,31 @@ public class NBitIntegerTest
         testEncode(6,63+0x00+0x80*0x80,     "3f808001");
         testEncode(6,63+0x7f+0x80*0x80*0x7f,"3fFf807f");
         testEncode(6,63+0x00+0x80*0x80*0x80,"3f80808001");
+
+        testEncode(8,0,"00");
+        testEncode(8,1,"01");
+        testEncode(8,128,"80");
+        testEncode(8,254,"Fe");
+        testEncode(8,255,"Ff00");
+        testEncode(8,255+1,"Ff01");
+        testEncode(8,255+0x7e,"Ff7e");
+        testEncode(8,255+0x7f,"Ff7f");
+        testEncode(8,255+0x80,"Ff8001");
+        testEncode(8,255+0x00+0x80*0x80,"Ff808001");
     }
 
     public void testEncode(int n,int i,String expected)
     {
         ByteBuffer buf = BufferUtil.allocate(16);
         int p=BufferUtil.flipToFill(buf);
-        buf.put((byte)0x00);
+        if (n<8)
+            buf.put((byte)0x00);
         NBitInteger.encode(buf,n,i);
         BufferUtil.flipToFlush(buf,p);
         String r=TypeUtil.toHexString(BufferUtil.toArray(buf));
         assertEquals(expected,r);
         
-        assertEquals(expected.length()/2,1+NBitInteger.octectsNeeded(n,i));
+        assertEquals(expected.length()/2,(n<8?1:0)+NBitInteger.octectsNeeded(n,i));
     }
     
     @Test
@@ -98,6 +111,17 @@ public class NBitIntegerTest
         testDecode(6,63+0x00+0x80*0x80,     "3f808001");
         testDecode(6,63+0x7f+0x80*0x80*0x7f,"3fFf807f");
         testDecode(6,63+0x00+0x80*0x80*0x80,"3f80808001");
+        
+        testDecode(8,0,"00");
+        testDecode(8,1,"01");
+        testDecode(8,128,"80");
+        testDecode(8,254,"Fe");
+        testDecode(8,255,"Ff00");
+        testDecode(8,255+1,"Ff01");
+        testDecode(8,255+0x7e,"Ff7e");
+        testDecode(8,255+0x7f,"Ff7f");
+        testDecode(8,255+0x80,"Ff8001");
+        testDecode(8,255+0x00+0x80*0x80,"Ff808001");
     }
     
     

jetty-util/src/main/java/org/eclipse/jetty/util/ArrayQueue.java

@@ -393,16 +393,22 @@ public class ArrayQueue<E> extends AbstractList<E> implements Queue<E>
     }
 
     /* ------------------------------------------------------------ */
-    protected void growUnsafe(int newCapacity)
+    protected void resizeUnsafe(int newCapacity)
     {
+        newCapacity = Math.max(newCapacity,_size);
         Object[] elements = new Object[newCapacity];
 
-        int split = _elements.length - _nextE;
+        if (_size>0)
-        if (split > 0)
+        {
-            System.arraycopy(_elements, _nextE, elements, 0, split);
+            if (_nextSlot>_nextE)
-        if (_nextE != 0)
+                System.arraycopy(_elements, _nextE, elements, 0, _size);
-            System.arraycopy(_elements, 0, elements, split, _nextSlot);
+            else
-
+            {
+                int split = _elements.length - _nextE;
+                System.arraycopy(_elements, _nextE, elements, 0, split);
+                System.arraycopy(_elements, 0, elements, split, _nextSlot);
+            }
+        }
         _elements = elements;
         _nextE = 0;
         _nextSlot = _size;
@@ -413,8 +419,7 @@ public class ArrayQueue<E> extends AbstractList<E> implements Queue<E>
     {
         if (_growCapacity <= 0)
             return false;
-        growUnsafe(_elements.length+_growCapacity);
+        resizeUnsafe(_elements.length+_growCapacity);
         return true;
     }
-
 }