430418 - Jetty 9.1.3 and Chrome 33 permessage-deflate do not work together

+ Updating Compress/PerMessageDeflate extensions for latest spec
  document, http://tools.ietf.org/html/draft-ietf-hybi-permessage-compression-18
  Intra-frame tail 0000FFFF is now being preserved for
  permessage-deflate

jetty-websocket/websocket-common/src/main/java/org/eclipse/jetty/websocket/common/extensions/compress/CompressExtension.java

@@ -42,18 +42,39 @@ public abstract class CompressExtension extends AbstractExtension
 {
     protected static final byte[] TAIL_BYTES = new byte[]{0x00, 0x00, (byte)0xFF, (byte)0xFF};
     private static final Logger LOG = Log.getLogger(CompressExtension.class);
+    
+    /** Never drop tail bytes 0000FFFF, from any frame type */
+    protected static final int TAIL_DROP_NEVER = 0;
+    /** Always drop tail bytes 0000FFFF, from all frame types */
+    protected static final int TAIL_DROP_ALWAYS = 1;
+    /** Only drop tail bytes 0000FFFF, from fin==true frames */
+    protected static final int TAIL_DROP_FIN_ONLY = 2;
+
+    /** Always set RSV flag, on all frame types */
+    protected static final int RSV_USE_ALWAYS = 0;
+    /** 
+     * Only set RSV flag on first frame in multi-frame messages.
+     * <p>
+     * Note: this automatically means no-continuation frames have
+     * the RSV bit set 
+     */
+    protected static final int RSV_USE_ONLY_FIRST = 1;
 
     private final Queue<FrameEntry> entries = new ConcurrentArrayQueue<>();
     private final IteratingCallback flusher = new Flusher();
     private final Deflater compressor;
     private final Inflater decompressor;
+    private int tailDrop = TAIL_DROP_NEVER;
+    private int rsvUse = RSV_USE_ALWAYS;
 
     protected CompressExtension()
     {
         compressor = new Deflater(Deflater.BEST_COMPRESSION, true);
         decompressor = new Inflater(true);
+        tailDrop = getTailDropMode();
+        rsvUse = getRsvUseMode();
     }
-
+    
     public Deflater getDeflater()
     {
         return compressor;
@@ -72,6 +93,20 @@ public abstract class CompressExtension extends AbstractExtension
     {
         return true;
     }
+    
+    /**
+     * Return the mode of operation for dropping (or keeping) tail bytes in frames generated by compress (outgoing)
+     * 
+     * @return either {@link #TAIL_DROP_ALWAYS}, {@link #TAIL_DROP_FIN_ONLY}, or {@link #TAIL_DROP_NEVER}
+     */
+    abstract int getTailDropMode();
+
+    /**
+     * Return the mode of operation for RSV flag use in frames generate by compress (outgoing)
+     * 
+     * @return either {@link #RSV_USE_ALWAYS} or {@link #RSV_USE_ONLY_FIRST}
+     */
+    abstract int getRsvUseMode();
 
     protected void forwardIncoming(Frame frame, ByteAccumulator accumulator)
     {
@@ -301,15 +336,31 @@ public abstract class CompressExtension extends AbstractExtension
                 }
             }
 
-            // Skip the last tail bytes bytes generated by SYNC_FLUSH.
+            boolean fin = frame.isFin() && finished;
-            payload = ByteBuffer.wrap(output, 0, outputLength - TAIL_BYTES.length);
+
-            LOG.debug("Compressed {}: {}->{} chunk bytes", entry, inputLength, outputLength);
+            // Handle tail bytes generated by SYNC_FLUSH.
+            if(tailDrop == TAIL_DROP_ALWAYS) {
+                payload = ByteBuffer.wrap(output, 0, outputLength - TAIL_BYTES.length);
+            } else if(tailDrop == TAIL_DROP_FIN_ONLY) {
+                payload = ByteBuffer.wrap(output, 0, outputLength - (fin?TAIL_BYTES.length:0));
+            } else {
+                // always include
+                payload = ByteBuffer.wrap(output, 0, outputLength);
+            }
+            if (LOG.isDebugEnabled())
+            {
+                LOG.debug("Compressed {}: {}->{} chunk bytes",entry,inputLength,outputLength);
+            }
 
             boolean continuation = frame.getType().isContinuation() || !first;
             DataFrame chunk = new DataFrame(frame, continuation);
-            chunk.setRsv1(true);
+            if(rsvUse == RSV_USE_ONLY_FIRST) {
+                chunk.setRsv1(!continuation);
+            } else {
+                // always set
+                chunk.setRsv1(true);
+            }
             chunk.setPayload(payload);
-            boolean fin = frame.isFin() && finished;
             chunk.setFin(fin);
 
             nextOutgoingFrame(chunk, this, entry.batchMode);

jetty-websocket/websocket-common/src/main/java/org/eclipse/jetty/websocket/common/extensions/compress/DeflateFrameExtension.java

@@ -35,6 +35,18 @@ public class DeflateFrameExtension extends CompressExtension
     {
         return "deflate-frame";
     }
+    
+    @Override
+    int getRsvUseMode()
+    {
+        return RSV_USE_ALWAYS;
+    }
+    
+    @Override
+    int getTailDropMode()
+    {
+        return TAIL_DROP_ALWAYS;
+    }
 
     @Override
     public void incomingFrame(Frame frame)

jetty-websocket/websocket-common/src/main/java/org/eclipse/jetty/websocket/common/extensions/compress/PerMessageDeflateExtension.java

@@ -102,13 +102,25 @@ public class PerMessageDeflateExtension extends CompressExtension
         }
         super.nextOutgoingFrame(frame, callback, batchMode);
     }
+    
+    @Override
+    int getRsvUseMode()
+    {
+        return RSV_USE_ONLY_FIRST;
+    }
+    
+    @Override
+    int getTailDropMode()
+    {
+        return TAIL_DROP_FIN_ONLY;
+    }
 
     @Override
     public void setConfig(final ExtensionConfig config)
     {
         configRequested = new ExtensionConfig(config);
         configNegotiated = new ExtensionConfig(config.getName());
-
+        
         for (String key : config.getParameterKeys())
         {
             key = key.trim();

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/browser/BrowserDebugTool.java

@@ -23,6 +23,7 @@ import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.handler.ResourceHandler;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
+import org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension;
 import org.eclipse.jetty.websocket.server.WebSocketHandler;
 import org.eclipse.jetty.websocket.servlet.ServletUpgradeRequest;
 import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse;
@@ -83,21 +84,15 @@ public class BrowserDebugTool implements WebSocketCreator
 
         String ua = req.getHeader("User-Agent");
         String rexts = req.getHeader("Sec-WebSocket-Extensions");
-        
+
-        LOG.debug("User-Agent: {}", ua);
+        LOG.debug("User-Agent: {}",ua);
-        LOG.debug("Sec-WebSocket-Extensions (Request) : {}", rexts);
+        LOG.debug("Sec-WebSocket-Extensions (Request) : {}",rexts);
         return new BrowserSocket(ua,rexts);
     }
 
-    public void start() throws Exception
+    public int getPort()
     {
-        server.start();
+        return connector.getLocalPort();
-        LOG.info("Server available on port {}", getPort());
-    }
-
-    public void stop() throws Exception
-    {
-        server.stop();
     }
 
     public void prepare(int port)
@@ -116,6 +111,7 @@ public class BrowserDebugTool implements WebSocketCreator
 
                 // factory.getExtensionFactory().unregister("deflate-frame");
                 // factory.getExtensionFactory().unregister("permessage-deflate");
+                factory.getExtensionFactory().register("permessage-deflate",PerMessageDeflateExtension.class);
                 // factory.getExtensionFactory().unregister("x-webkit-deflate-frame");
 
                 // Setup the desired Socket to use for all incoming upgrade requests
@@ -123,6 +119,9 @@ public class BrowserDebugTool implements WebSocketCreator
 
                 // Set the timeout
                 factory.getPolicy().setIdleTimeout(30000);
+
+                // Set top end message size
+                factory.getPolicy().setMaxTextMessageSize(15 * 1024 * 1024);
             }
         };
 
@@ -138,8 +137,14 @@ public class BrowserDebugTool implements WebSocketCreator
         LOG.info("{} setup on port {}",this.getClass().getName(),port);
     }
 
-    public int getPort()
+    public void start() throws Exception
     {
-        return connector.getLocalPort();
+        server.start();
+        LOG.info("Server available on port {}",getPort());
+    }
+
+    public void stop() throws Exception
+    {
+        server.stop();
     }
 }

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/browser/BrowserSocket.java

@@ -109,7 +109,15 @@ public class BrowserSocket
     @OnWebSocketMessage
     public void onTextMessage(String message)
     {
-        LOG.info("onTextMessage({})",message);
+        if (message.length() > 300)
+        {
+            int len = message.length();
+            LOG.info("onTextMessage({} ... {}) size:{}",message.substring(0,15),message.substring(len - 15,len).replaceAll("[\r\n]*",""),len);
+        }
+        else
+        {
+            LOG.info("onTextMessage({})",message);
+        }
 
         int idx = message.indexOf(':');
         if (idx > 0)

jetty-websocket/websocket-server/src/test/resources/browser-debug-tool/index.html

@@ -17,6 +17,10 @@
       <input id="hello" class="button" type="submit" name="hello" value="hello" disabled="disabled"/>
       <input id="there" class="button" type="submit" name="there" value="there" disabled="disabled"/>
       <input id="json" class="button" type="submit" name="json" value="json" disabled="disabled"/>
+      <input id="send10k" class="button" type="submit" name="send10k" value="send10k" disabled="disabled"/>
+      <input id="send100k" class="button" type="submit" name="send100k" value="send100k" disabled="disabled"/>
+      <input id="send1000k" class="button" type="submit" name="send1000k" value="send1000k" disabled="disabled"/>
+      <input id="send10m" class="button" type="submit" name="send10m" value="send10m" disabled="disabled"/>
     </div>
     <script type="text/javascript">
     $("connect").onclick = function(event) { wstool.connect(); return false; }
@@ -32,6 +36,21 @@
             + " 12 Sep 2013 19:42:30 GMT\"},{\"channel\":\"/chat/demo\",\"data\":{\"user\":\"ch\",\"membership\":\"join\",\"chat\":\"ch"
             + " has joined\"},\"id\":\"4\",\"clientId\":\"81dwnxwbgs0h0bq8968b0a0gyl\",\"timestamp\":\"Thu,"
             + " 12 Sep 2013 19:42:30 GMT\"}]"); return false; }
+    $("send10k").onclick =   function(event) {wstool.write(randomString(  10*1024)); return false;}
+    $("send100k").onclick =  function(event) {wstool.write(randomString( 100*1024)); return false;}
+    $("send1000k").onclick = function(event) {wstool.write(randomString(1000*1024)); return false;}
+    $("send10m").onclick =   function(event) {wstool.write(randomString(  10*1024*1024)); return false;}
+    
+    function randomString(len, charSet) {
+        charSet = charSet || 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789{}	":;<>,.()[]';
+        var randomString = '';
+        var charLen = charSet.length;
+        for (var i = 0; i < len; i++) {
+        	var randomPoz = Math.floor(Math.random() * charLen);
+        	randomString += charSet.substring(randomPoz,randomPoz+1);
+        }
+        return randomString;
+    }
     </script>
   </body>
 </html>

jetty-websocket/websocket-server/src/test/resources/browser-debug-tool/websocket.js

@@ -59,12 +59,20 @@ var wstool = {
     },
 
     infoc : function(message) {
-        wstool._out("client", "[c] " + message);
+        if(message.length > 300) {
+            wstool._out("client", "[c] [big message: " + message.length + " characters]");
+        } else {
+            wstool._out("client", "[c] " + message);
+        }
     },
     
     infos : function(message) {
         this._scount++;
-        wstool._out("server", "[s" + this._scount + "] " + message);
+        if(message.length > 300) {
+            wstool._out("server", "[s" + this._scount + "] [big message: " + message.length + " characters]");
+        } else {
+            wstool._out("server", "[s" + this._scount + "] " + message);
+        }
     },
 
     setState : function(enabled) {
@@ -77,6 +85,10 @@ var wstool = {
         $('hello').disabled = !enabled;
         $('there').disabled = !enabled;
         $('json').disabled = !enabled;
+        $('send10k').disabled = !enabled;
+        $('send100k').disabled = !enabled;
+        $('send1000k').disabled = !enabled;
+        $('send10m').disabled = !enabled;
     },
     
     _onopen : function() {