Merge branch 'jetty-9' into jetty-9-oneconnector

Conflicts:
	jetty-server/src/main/java/org/eclipse/jetty/server/ssl/SslSelectChannelConnector.java

jetty-io/src/main/java/org/eclipse/jetty/io/EndPoint.java

@@ -13,6 +13,7 @@
 
 package org.eclipse.jetty.io;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
@@ -23,7 +24,7 @@ import java.nio.ByteBuffer;
  *
  * A transport EndPoint
  */
-public interface EndPoint
+public interface EndPoint extends Closeable
 {
     /* ------------------------------------------------------------ */
     /**

jetty-io/src/main/java/org/eclipse/jetty/io/SelectChannelEndPoint.java

@@ -54,8 +54,7 @@ public class SelectChannelEndPoint extends ChannelEndPoint implements Runnable,
         @Override
         protected boolean needsFill()
         {
-            updateKey(SelectionKey.OP_READ, true);
+            return SelectChannelEndPoint.this.needsFill();
-            return false;
         }
     };
     private final WriteFlusher _writeFlusher = new WriteFlusher(this)
@@ -63,7 +62,7 @@ public class SelectChannelEndPoint extends ChannelEndPoint implements Runnable,
         @Override
         protected void onIncompleteFlushed()
         {
-            updateKey(SelectionKey.OP_WRITE, true);
+            SelectChannelEndPoint.this.onIncompleteFlush();
         }
     };
     private final SelectorManager.ManagedSelector _selector;
@@ -91,9 +90,29 @@ public class SelectChannelEndPoint extends ChannelEndPoint implements Runnable,
         scheduleIdleTimeout(idleTimeout);
     }
 
+    protected boolean needsFill()
+    {
+        updateLocalInterests(SelectionKey.OP_READ, true);
+        return false;
+    }
+
+    protected void onIncompleteFlush()
+    {
+        updateLocalInterests(SelectionKey.OP_WRITE, true);
+    }
+
     private void scheduleIdleTimeout(long delay)
     {
-        Future<?> newTimeout = isOpen() && delay > 0 ? _scheduler.schedule(_idleTask, delay, TimeUnit.MILLISECONDS) : null;
+        Future<?> newTimeout = null;
+        if (isOpen() && delay > 0)
+        {
+            LOG.debug("{} scheduling idle timeout in {} ms", this, delay);
+            newTimeout = _scheduler.schedule(_idleTask, delay, TimeUnit.MILLISECONDS);
+        }
+        else
+        {
+            LOG.debug("{} skipped scheduling idle timeout ({} ms)", this, delay);
+        }
         Future<?> oldTimeout = _timeout.getAndSet(newTimeout);
         if (oldTimeout != null)
             oldTimeout.cancel(false);
@@ -129,7 +148,11 @@ public class SelectChannelEndPoint extends ChannelEndPoint implements Runnable,
     @Override
     public void onSelected()
     {
-        _interestOps = 0;
+        int oldInterestOps = _key.interestOps();
+        int readyOps = _key.readyOps();
+        int newInterestOps = oldInterestOps & ~readyOps;
+        setKeyInterests(oldInterestOps, newInterestOps);
+        updateLocalInterests(readyOps, false);
         if (_key.isReadable())
             _readInterest.readable();
         if (_key.isWritable())
@@ -145,12 +168,16 @@ public class SelectChannelEndPoint extends ChannelEndPoint implements Runnable,
             long idleElapsed = System.currentTimeMillis() - idleTimestamp;
             long idleLeft = idleTimeout - idleElapsed;
 
+            LOG.debug("{} idle timeout check, elapsed: {} ms, remaining: {} ms", this, idleElapsed, idleLeft);
+
             if (isOutputShutdown() || _readInterest.isInterested() || _writeFlusher.isWriting())
             {
                 if (idleTimestamp != 0 && idleTimeout > 0)
                 {
-                    if (idleLeft < 0)
+                    if (idleLeft <= 0)
                     {
+                        LOG.debug("{} idle timeout expired", this);
+
                         if (isOutputShutdown())
                             close();
                         notIdle();
@@ -165,7 +192,7 @@ public class SelectChannelEndPoint extends ChannelEndPoint implements Runnable,
         }
     }
 
-    private void updateKey(int operation, boolean add)
+    private void updateLocalInterests(int operation, boolean add)
     {
         int oldInterestOps = _interestOps;
         int newInterestOps;
@@ -182,12 +209,12 @@ public class SelectChannelEndPoint extends ChannelEndPoint implements Runnable,
         if (newInterestOps != oldInterestOps)
         {
             _interestOps = newInterestOps;
-            LOG.debug("Key update {} -> {} for {}", oldInterestOps, newInterestOps, this);
+            LOG.debug("Local interests updated {} -> {} for {}", oldInterestOps, newInterestOps, this);
             _selector.submit(this);
         }
         else
         {
-            LOG.debug("Ignoring key update {} -> {} for {}", oldInterestOps, newInterestOps, this);
+            LOG.debug("Ignoring local interests update {} -> {} for {}", oldInterestOps, newInterestOps, this);
         }
     }
 
@@ -201,13 +228,25 @@ public class SelectChannelEndPoint extends ChannelEndPoint implements Runnable,
                 int oldInterestOps = _key.interestOps();
                 int newInterestOps = _interestOps;
                 if (newInterestOps != oldInterestOps)
-                    _key.interestOps(newInterestOps);
+                    setKeyInterests(oldInterestOps, newInterestOps);
             }
         }
         catch (CancelledKeyException x)
         {
             LOG.debug("Ignoring key update for concurrently closed channel {}", this);
+            close();
         }
+        catch (Exception x)
+        {
+            LOG.warn("Ignoring key update for " + this, x);
+            close();
+        }
+    }
+
+    private void setKeyInterests(int oldInterestOps, int newInterestOps)
+    {
+        LOG.debug("Key interests updated {} -> {}", oldInterestOps, newInterestOps);
+        _key.interestOps(newInterestOps);
     }
 
     @Override

jetty-io/src/main/java/org/eclipse/jetty/io/SelectorManager.java

@@ -13,13 +13,13 @@
 
 package org.eclipse.jetty.io;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.net.ConnectException;
 import java.net.Socket;
 import java.net.SocketAddress;
 import java.nio.channels.CancelledKeyException;
 import java.nio.channels.ClosedChannelException;
-import java.nio.channels.ClosedSelectorException;
 import java.nio.channels.SelectionKey;
 import java.nio.channels.Selector;
 import java.nio.channels.ServerSocketChannel;
@@ -218,13 +218,13 @@ public abstract class SelectorManager extends AbstractLifeCycle implements Dumpa
      * or {@link #accept(SocketChannel)}.</p>
      *
      * @param channel   the channel associated to the endpoint
-     * @param selectSet the selector the channel is registered to
+     * @param selector the selector the channel is registered to
      * @param selectionKey      the selection key
      * @return a new endpoint
      * @throws IOException if the endPoint cannot be created
      * @see #newConnection(SocketChannel, AsyncEndPoint, Object)
      */
-    protected abstract AsyncEndPoint newEndPoint(SocketChannel channel, SelectorManager.ManagedSelector selectSet, SelectionKey selectionKey) throws IOException;
+    protected abstract AsyncEndPoint newEndPoint(SocketChannel channel, SelectorManager.ManagedSelector selector, SelectionKey selectionKey) throws IOException;
 
     /**
      * <p>Factory method to create {@link AsyncConnection}.</p>
@@ -337,16 +337,7 @@ public abstract class SelectorManager extends AbstractLifeCycle implements Dumpa
                 _thread.setName(name + " Selector" + _id);
                 LOG.debug("Starting {} on {}", _thread, this);
                 while (isRunning())
-                {
-                    try
-                    {
                     select();
-                    }
-                    catch (IOException e)
-                    {
-                        LOG.warn(e);
-                    }
-                }
                 processChanges();
             }
             finally
@@ -359,10 +350,9 @@ public abstract class SelectorManager extends AbstractLifeCycle implements Dumpa
         /**
          * <p>Process changes and waits on {@link Selector#select()}.</p>
          *
-         * @throws IOException if the select operation fails
          * @see #submit(Runnable)
          */
-        public void select() throws IOException
+        public void select()
         {
             boolean debug = LOG.isDebugEnabled();
             try
@@ -380,32 +370,22 @@ public abstract class SelectorManager extends AbstractLifeCycle implements Dumpa
                 Set<SelectionKey> selectedKeys = _selector.selectedKeys();
                 for (SelectionKey key : selectedKeys)
                 {
-                    try
+                    if (key.isValid())
                     {
-                        if (!key.isValid())
+                        processKey(key);
+                    }
+                    else
                     {
                         if (debug)
                             LOG.debug("Selector loop ignoring invalid key for channel {}", key.channel());
-                            continue;
+                        Object attachment = key.attachment();
-                        }
+                        if (attachment instanceof EndPoint)
-
+                            ((EndPoint)attachment).close();
-                        processKey(key);
-                    }
-                    catch (Exception x)
-                    {
-                        if (isRunning())
-                            LOG.warn(x);
-                        else
-                            LOG.debug(x);
-
-                        execute(new Close(key));
                     }
                 }
-
-                // Everything always handled
                 selectedKeys.clear();
             }
-            catch (ClosedSelectorException x)
+            catch (Exception x)
             {
                 if (isRunning())
                     LOG.warn(x);
@@ -430,12 +410,11 @@ public abstract class SelectorManager extends AbstractLifeCycle implements Dumpa
 
         private void processKey(SelectionKey key)
         {
+            Object attachment = key.attachment();
             try
             {
-                Object attachment = key.attachment();
                 if (attachment instanceof SelectableAsyncEndPoint)
                 {
-                    key.interestOps(0);
                     ((SelectableAsyncEndPoint)attachment).onSelected();
                 }
                 else if (key.isConnectable())
@@ -459,7 +438,7 @@ public abstract class SelectorManager extends AbstractLifeCycle implements Dumpa
                     catch (Exception x)
                     {
                         connectionFailed(channel, x, attachment);
-                        key.cancel();
+                        closeNoExceptions(channel);
                     }
                 }
                 else
@@ -469,7 +448,27 @@ public abstract class SelectorManager extends AbstractLifeCycle implements Dumpa
             }
             catch (CancelledKeyException x)
             {
-                LOG.debug("Ignoring cancelled key for channel", key.channel());
+                LOG.debug("Ignoring cancelled key for channel {}", key.channel());
+                if (attachment instanceof EndPoint)
+                    ((EndPoint)attachment).close();
+            }
+            catch (Exception x)
+            {
+                LOG.warn("Could not process key for channel " + key.channel(), x);
+                if (attachment instanceof EndPoint)
+                    ((EndPoint)attachment).close();
+            }
+        }
+
+        private void closeNoExceptions(Closeable closeable)
+        {
+            try
+            {
+                closeable.close();
+            }
+            catch (IOException x)
+            {
+                LOG.ignore(x);
             }
         }
 
@@ -644,29 +643,6 @@ public abstract class SelectorManager extends AbstractLifeCycle implements Dumpa
             }
         }
 
-        private class Close implements Runnable
-        {
-            private final SelectionKey key;
-
-            private Close(SelectionKey key)
-            {
-                this.key = key;
-            }
-
-            @Override
-            public void run()
-            {
-                try
-                {
-                    key.channel().close();
-                }
-                catch (IOException x)
-                {
-                    LOG.ignore(x);
-                }
-            }
-        }
-
         private class Stop implements Runnable
         {
             private final CountDownLatch latch = new CountDownLatch(1);
@@ -686,11 +662,7 @@ public abstract class SelectorManager extends AbstractLifeCycle implements Dumpa
                         }
                     }
 
-                    _selector.close();
+                    closeNoExceptions(_selector);
-                }
-                catch (IOException x)
-                {
-                    LOG.ignore(x);
                 }
                 finally
                 {

jetty-io/src/main/java/org/eclipse/jetty/io/WriteFlusher.java

@@ -19,6 +19,7 @@ import org.eclipse.jetty.util.Callback;
  * written after a call to flush and should organise for the {@link #completeWrite()}
  * method to be called when a subsequent call to flush should be able to make more progress.
  *
+ * TODO remove synchronisation
  */
 abstract public class WriteFlusher
 {
@@ -36,7 +37,7 @@ abstract public class WriteFlusher
     }
 
     /* ------------------------------------------------------------ */
-    public <C> void write(C context, Callback<C> callback, ByteBuffer... buffers)
+    public synchronized <C> void write(C context, Callback<C> callback, ByteBuffer... buffers)
     {
         if (callback==null)
             throw new IllegalArgumentException();
@@ -86,7 +87,7 @@ abstract public class WriteFlusher
     /* ------------------------------------------------------------ */
     /* Remove empty buffers from the start of a multi buffer array
      */
-    private ByteBuffer[] compact(ByteBuffer[] buffers)
+    private synchronized ByteBuffer[] compact(ByteBuffer[] buffers)
     {
         if (buffers.length<2)
             return buffers;
@@ -110,7 +111,7 @@ abstract public class WriteFlusher
      * method when a call to {@link EndPoint#flush(ByteBuffer...)}
      * is likely to be able to progress.
      */
-    public void completeWrite()
+    public synchronized void completeWrite()
     {
         if (!isWriting())
             return; // TODO throw?
@@ -163,7 +164,7 @@ abstract public class WriteFlusher
      * the cause wrapped as an execution exception.
      * @return true if a write was in progress
      */
-    public boolean failed(Throwable cause)
+    public synchronized boolean failed(Throwable cause)
     {
         if (!_writing.compareAndSet(true,false))
             return false;
@@ -183,7 +184,7 @@ abstract public class WriteFlusher
      * not instantiated unless a write was in progress.
      * @return true if a write was in progress
      */
-    public boolean close()
+    public synchronized boolean close()
     {
         if (!_writing.compareAndSet(true,false))
             return false;
@@ -197,7 +198,7 @@ abstract public class WriteFlusher
     }
 
     /* ------------------------------------------------------------ */
-    public boolean isWriting()
+    public synchronized boolean isWriting()
     {
         return _writing.get();
     }

jetty-io/src/main/java/org/eclipse/jetty/io/ssl/SslConnection.java

@@ -15,10 +15,12 @@ package org.eclipse.jetty.io.ssl;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 import java.util.concurrent.Executor;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
 import javax.net.ssl.SSLEngineResult.HandshakeStatus;
+import javax.net.ssl.SSLEngineResult.Status;
 import javax.net.ssl.SSLException;
 
 import org.eclipse.jetty.io.AbstractAsyncConnection;
@@ -38,33 +40,54 @@ import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 
 /**
- * An AsyncConnection that acts as an interceptor between and EndPoint and another
+ * An AsyncConnection that acts as an intercepter between an AsyncEndPoint providing SSL encrypted data
- * Connection, that implements TLS encryption using an {@link SSLEngine}.
+ * and another consumer of an  AsyncEndPoint (typically an {@link AsyncConnection} like HttpConnection) that 
- * <p/>
+ * wants unencrypted data.
- * The connector uses an {@link EndPoint} (like {@link SelectChannelEndPoint}) as
+ * <p>
- * it's source/sink of encrypted data.   It then provides {@link #getSslEndPoint()} to
+ * The connector uses an {@link AsyncEndPoint} (typically {@link SelectChannelEndPoint}) as
+ * it's source/sink of encrypted data.   It then provides an endpoint via {@link #getSslEndPoint()} to
  * expose a source/sink of unencrypted data to another connection (eg HttpConnection).
+ * <p>
+ * The design of this class is based on a clear separation between the passive methods, which do not block nor schedule any 
+ * asynchronous callbacks, and active methods that do schedule asynchronous callbacks.
+ * <p>
+ * The passive methods are {@link DecryptedEndPoint#fill(ByteBuffer)} and {@link DecryptedEndPoint#flush(ByteBuffer...)}. They make best
+ * effort attempts to progress the connection using only calls to the encrypted {@link AsyncEndPoint#fill(ByteBuffer)} and {@link AsyncEndPoint#flush(ByteBuffer...)}
+ * methods.  They will never block nor schedule any readInterest or write callbacks.   If a fill/flush cannot progress either because
+ * of network congestion or waiting for an SSL handshake message, then the fill/flush will simply return with zero bytes filled/flushed.
+ * Specifically, if a flush cannot proceed because it needs to receive a handshake message, then the flush will attempt to fill bytes from the 
+ * encrypted endpoint, but if insufficient bytes are read it will NOT call {@link AsyncEndPoint#fillInterested(Object, Callback)}.
+ * <p>
+ * It is only the active methods : {@link DecryptedEndPoint#fillInterested(Object, Callback)} and 
+ * {@link DecryptedEndPoint#write(Object, Callback, ByteBuffer...)} that may schedule callbacks by calling the encrypted 
+ * {@link AsyncEndPoint#fillInterested(Object, Callback)} and {@link AsyncEndPoint#write(Object, Callback, ByteBuffer...)}
+ * methods.  For normal data handling, the decrypted fillInterest method will result in an encrypted fillInterest and a decrypted
+ * write will result in an encrypted write. However, due to SSL handshaking requirements, it is also possible for a decrypted fill 
+ * to call the encrypted write and for the decrypted flush to call the encrypted fillInterested methods.
+ * <p>
+ * MOST IMPORTANTLY, the encrypted callbacks from the active methods (#onFillable() and WriteFlusher#completeWrite()) do no filling or flushing 
+ * themselves.  Instead they simple make the callbacks to the decrypted callbacks, so that the passive encyrpted fill/flush will
+ * be called again and make another best effort attempt to progress the connection.
+ * 
  */
 public class SslConnection extends AbstractAsyncConnection
 {
     private static final Logger LOG = Log.getLogger(SslConnection.class);
     private final ByteBufferPool _bufferPool;
     private final SSLEngine _sslEngine;
-    private final SslEndPoint _appEndPoint;
+    private final DecryptedEndPoint _decryptedEndPoint;
-    private ByteBuffer _appIn;
+    private ByteBuffer _decryptedInput;
-    private ByteBuffer _netIn;
+    private ByteBuffer _encryptedInput;
-    private ByteBuffer _netOut;
+    private ByteBuffer _encryptedOutput;
-    private final boolean _netDirect = false;
+    private final boolean _encryptedDirectBuffers = false;
-    private final boolean _appDirect = false;
+    private final boolean _decryptedDirectBuffers = false;
-    private SSLEngineResult _unwrapResult;
-    private SSLEngineResult _wrapResult;
 
     public SslConnection(ByteBufferPool byteBufferPool, Executor executor, AsyncEndPoint endPoint, SSLEngine sslEngine)
     {
         super(endPoint, executor, true);
         this._bufferPool = byteBufferPool;
         this._sslEngine = sslEngine;
-        this._appEndPoint = new SslEndPoint();
+        this._decryptedEndPoint = new DecryptedEndPoint();
     }
 
     public SSLEngine getSSLEngine()
@@ -74,7 +97,7 @@ public class SslConnection extends AbstractAsyncConnection
 
     public AsyncEndPoint getSslEndPoint()
     {
-        return _appEndPoint;
+        return _decryptedEndPoint;
     }
 
     @Override
@@ -88,7 +111,7 @@ public class SslConnection extends AbstractAsyncConnection
             _sslEngine.beginHandshake();
 
             if (_sslEngine.getUseClientMode())
-                _appEndPoint.write(null, new Callback.Empty<>(), BufferUtil.EMPTY_BUFFER);
+                _decryptedEndPoint.write(null, new Callback.Empty<>(), BufferUtil.EMPTY_BUFFER);
         }
         catch (SSLException x)
         {
@@ -101,18 +124,27 @@ public class SslConnection extends AbstractAsyncConnection
     @Override
     public void onFillable()
     {
+        // onFillable means that there are encrypted bytes ready to be filled.
+        // however we do not fill them here on this callback, but instead wakeup
+        // the decrypted readInterest and/or writeFlusher so that they will attempt
+        // to do the fill and/or flush again and these calls will do the actually
+        // filling. 
+        
         LOG.debug("{} onReadable", this);
 
+        synchronized(_decryptedEndPoint)
+        {
             // wake up whoever is doing the fill or the flush so they can
             // do all the filling, unwrapping ,wrapping and flushing
-        if (_appEndPoint._readInterest.isInterested())
+            if (_decryptedEndPoint._readInterest.isInterested())
-            _appEndPoint._readInterest.readable();
+                _decryptedEndPoint._readInterest.readable();
 
             // If we are handshaking, then wake up any waiting write as well as it may have been blocked on the read
-        if (_appEndPoint._writeFlusher.isWriting() && _appEndPoint._flushUnwrap)
+            if ( _decryptedEndPoint._flushRequiresFillToProgress)
             {
-            _appEndPoint._flushUnwrap = false;
+                _decryptedEndPoint._flushRequiresFillToProgress = false;
-            _appEndPoint._writeFlusher.completeWrite();
+                _decryptedEndPoint._writeFlusher.completeWrite();
+            }
         }
     }
 
@@ -120,17 +152,25 @@ public class SslConnection extends AbstractAsyncConnection
     @Override
     public void onFillInterestedFailed(Throwable cause)
     {
+        // this means that the fill interest in encrypted bytes has failed.
+        // However we do not handle that here on this callback, but instead wakeup
+        // the decrypted readInterest and/or writeFlusher so that they will attempt
+        // to do the fill and/or flush again and these calls will do the actually
+        // handle the cause.
+        
         super.onFillInterestedFailed(cause);
 
-        if (_appEndPoint._readInterest.isInterested())
+        synchronized(_decryptedEndPoint)
-            _appEndPoint._readInterest.failed(cause);
-
-        if (_appEndPoint._writeFlusher.isWriting() && _appEndPoint._flushUnwrap)
         {
-            _appEndPoint._flushUnwrap = false;
+            if (_decryptedEndPoint._readInterest.isInterested())
-            _appEndPoint._writeFlusher.failed(cause);
+                _decryptedEndPoint._readInterest.failed(cause);
-        }
 
+            if (_decryptedEndPoint._flushRequiresFillToProgress)
+            {
+                _decryptedEndPoint._flushRequiresFillToProgress = false;
+                _decryptedEndPoint._writeFlusher.failed(cause);
+            }
+        }
     }
 
     /* ------------------------------------------------------------ */
@@ -140,18 +180,18 @@ public class SslConnection extends AbstractAsyncConnection
         return String.format("SslConnection@%x{%s,%s%s}",
                 hashCode(),
                 _sslEngine.getHandshakeStatus(),
-                _appEndPoint._readInterest.isInterested() ? "R" : "",
+                _decryptedEndPoint._readInterest.isInterested() ? "R" : "",
-                _appEndPoint._writeFlusher.isWriting() ? "W" : "");
+                _decryptedEndPoint._writeFlusher.isWriting() ? "W" : "");
     }
 
     /* ------------------------------------------------------------ */
-    public class SslEndPoint extends AbstractEndPoint implements AsyncEndPoint
+    public class DecryptedEndPoint extends AbstractEndPoint implements AsyncEndPoint
     {
         private AsyncConnection _connection;
-        private boolean _fillWrap;
+        private boolean _fillRequiresFlushToProgress;
-        private boolean _flushUnwrap;
+        private boolean _flushRequiresFillToProgress;
-        private boolean _netWriting;
+        private boolean _cannotAcceptMoreAppDataToFlush;
-        private boolean _underflown;
+        private boolean _needToFillMoreDataToProgress;
         private boolean _ishut = false;
 
         @Override
@@ -170,16 +210,20 @@ public class SslConnection extends AbstractAsyncConnection
             @Override
             public void completed(Void context)
             {
-                synchronized (SslEndPoint.this)
+                // This means that a write of data has completed.  Writes are done
+                // only if there is an active writeflusher or a read needed to write
+                // data.  In either case the appropriate callback is passed on.
+                synchronized (DecryptedEndPoint.this)
                 {
-                    LOG.debug("{} write.complete {}", SslConnection.this, _netWriting ? (_fillWrap ? "FW" : "F") : (_fillWrap ? "W" : ""));
+                    LOG.debug("{} write.complete {}", SslConnection.this, _cannotAcceptMoreAppDataToFlush ? (_fillRequiresFlushToProgress ? "FW" : "F") : (_fillRequiresFlushToProgress ? "W" : ""));
 
                     releaseNetOut();
 
-                    _netWriting = false;
+                    _cannotAcceptMoreAppDataToFlush = false;
-                    if (_fillWrap)
+                    
+                    if (_fillRequiresFlushToProgress)
                     {
-                        _fillWrap = false;
+                        _fillRequiresFlushToProgress = false;
                         _readInterest.readable();
                     }
 
@@ -191,16 +235,21 @@ public class SslConnection extends AbstractAsyncConnection
             @Override
             public void failed(Void context, Throwable x)
             {
-                synchronized (SslEndPoint.this)
+                // This means that a write of data has failed.  Writes are done
+                // only if there is an active writeflusher or a read needed to write
+                // data.  In either case the appropriate callback is passed on.
+                synchronized (DecryptedEndPoint.this)
                 {
                     LOG.debug("{} write.failed", SslConnection.this, x);
-                    if (_netOut != null)
+                    if (_encryptedOutput != null)
-                        BufferUtil.clear(_netOut);
+                        BufferUtil.clear(_encryptedOutput);
                     releaseNetOut();
-                    _netWriting = false;
+                    
-                    if (_fillWrap)
+                    _cannotAcceptMoreAppDataToFlush = false;
+                    
+                    if (_fillRequiresFlushToProgress)
                     {
-                        _fillWrap = false;
+                        _fillRequiresFlushToProgress = false;
                         _readInterest.failed(x);
                     }
 
@@ -217,40 +266,55 @@ public class SslConnection extends AbstractAsyncConnection
             @Override
             protected boolean needsFill() throws IOException
             {
-                synchronized (SslEndPoint.this)
+                // This means that the decrypted data consumer has called the fillInterested
+                // method on the DecryptedEndPoint, so we have to work out if there is 
+                // decrypted data to be filled or what callbacks to setup to be told when there
+                // might be more encrypted data available to attempt another call to fill
+                
+                synchronized (DecryptedEndPoint.this)
                 {
-                    // Do we already have some app data
+                    // Do we already have some app data, then app can fill now so return true
-                    if (BufferUtil.hasContent(_appIn))
+                    if (BufferUtil.hasContent(_decryptedInput))
                         return true;
 
-                    // If we are not underflown and have net data
+                    // If we have no encrypted data to decrypt OR we have some, but it is not enough
-                    if (!_underflown && BufferUtil.hasContent(_netIn))
+                    if (BufferUtil.isEmpty(_encryptedInput) || _needToFillMoreDataToProgress)
-                        return true;
+                    {
-
+                        // We are not ready to read data
-                    // So we are not read ready
 
                         // Are we actually write blocked?
-                    if (_fillWrap)
+                        if (_fillRequiresFlushToProgress)
                         {
                             // we must be blocked trying to write before we can read
-                        // If we have written the net data
+                            
-                        if (BufferUtil.isEmpty(_netOut))
+                            // Do we have data to write
+                            if (BufferUtil.hasContent(_encryptedOutput))
                             {
+                                // write it
+                                _cannotAcceptMoreAppDataToFlush = true;
+                                getEndPoint().write(null, _writeCallback, _encryptedOutput);
+                            }
+                            else
+                            {
+                                // we have already written the net data
                                 // pretend we are readable so the wrap is done by next readable callback
-                            _fillWrap = false;
+                                _fillRequiresFlushToProgress = false;
                                 return true;
                             }
-
-                        // otherwise write the net data
-                        _netWriting = true;
-                        getEndPoint().write(null, _writeCallback, _netOut);
                         }
                         else
                             // Normal readable callback
+                            // Get called back on onfillable when then is more data to fill
                             SslConnection.this.fillInterested();
 
                         return false;
                     }
+                    else
+                    {
+                        // We are ready to read data
+                        return true;
+                    }
+                }
             }
         };
 
@@ -259,16 +323,19 @@ public class SslConnection extends AbstractAsyncConnection
             @Override
             protected void onIncompleteFlushed()
             {
-                synchronized (SslEndPoint.this)
+                // This means that the decripted endpoint write method was called and not
+                // all data could be wrapped. So either we need to write some encrypted data,
+                // OR if we are handshaking we need to read some encrypted data OR
+                // if neither than we should just try the flush again.
+                synchronized (DecryptedEndPoint.this)
                 {
                     // If we have pending output data,
-                    if (BufferUtil.hasContent(_netOut))
+                    if (BufferUtil.hasContent(_encryptedOutput))
                     {
                         // write it
-                        _netWriting = true;
+                        _cannotAcceptMoreAppDataToFlush = true;
-                        getEndPoint().write(null, _writeCallback, _netOut);
+                        getEndPoint().write(null, _writeCallback, _encryptedOutput);
                     }
-                    // TODO test this with _flushInwrap
                     else if (_sslEngine.getHandshakeStatus() == HandshakeStatus.NEED_UNWRAP)
                         // we are actually read blocked in order to write
                         SslConnection.this.fillInterested();
@@ -279,7 +346,7 @@ public class SslConnection extends AbstractAsyncConnection
             }
         };
 
-        public SslEndPoint()
+        public DecryptedEndPoint()
         {
             super(getEndPoint().getLocalAddress(), getEndPoint().getRemoteAddress());
         }
@@ -308,51 +375,52 @@ public class SslConnection extends AbstractAsyncConnection
             try
             {
                 // Do we already have some decrypted data?
-                if (BufferUtil.hasContent(_appIn))
+                if (BufferUtil.hasContent(_decryptedInput))
-                    return BufferUtil.append(_appIn, buffer);
+                    return BufferUtil.append(_decryptedInput, buffer);
 
                 // We will need a network buffer
-                if (_netIn == null)
+                if (_encryptedInput == null)
-                    _netIn = _bufferPool.acquire(_sslEngine.getSession().getPacketBufferSize(), _netDirect);
+                    _encryptedInput = _bufferPool.acquire(_sslEngine.getSession().getPacketBufferSize(), _encryptedDirectBuffers);
                 else
-                    BufferUtil.compact(_netIn);
+                    BufferUtil.compact(_encryptedInput);
 
                 // We also need an app buffer, but can use the passed buffer if it is big enough
                 ByteBuffer app_in;
                 if (BufferUtil.space(buffer) > _sslEngine.getSession().getApplicationBufferSize())
                     app_in = buffer;
-                else if (_appIn == null)
+                else if (_decryptedInput == null)
-                    app_in = _appIn = _bufferPool.acquire(_sslEngine.getSession().getApplicationBufferSize(), _appDirect);
+                    app_in = _decryptedInput = _bufferPool.acquire(_sslEngine.getSession().getApplicationBufferSize(), _decryptedDirectBuffers);
                 else
-                    app_in = _appIn;
+                    app_in = _decryptedInput;
 
                 // loop filling and unwrapping until we have something
                 while (true)
                 {
                     // Let's try reading some encrypted data... even if we have some already.
-                    int net_filled = getEndPoint().fill(_netIn);
+                    int net_filled = getEndPoint().fill(_encryptedInput);
                     LOG.debug("{} filled {} encrypted bytes", SslConnection.this, net_filled);
                     if (net_filled > 0)
-                        _underflown = false;
+                        _needToFillMoreDataToProgress = false;
 
                     // Let's try the SSL thang even if we have no net data because in that
                     // case we want to fall through to the handshake handling
                     int pos = BufferUtil.flipToFill(app_in);
-                    _unwrapResult = _sslEngine.unwrap(_netIn, app_in);
+                    SSLEngineResult unwrapResult = _sslEngine.unwrap(_encryptedInput, app_in);
-                    LOG.debug("{} unwrap {}", SslConnection.this, _unwrapResult);
+                    LOG.debug("{} unwrap {}", SslConnection.this, unwrapResult);
                     BufferUtil.flipToFlush(app_in, pos);
 
                     // and deal with the results
-                    switch (_unwrapResult.getStatus())
+                    switch (unwrapResult.getStatus())
                     {
                         case BUFFER_OVERFLOW:
                             throw new IllegalStateException();
 
                         case CLOSED:
-                            // Dang! we have to care about the handshake state
+                            // Dang! we have to care about the handshake state specially for close
                             switch (_sslEngine.getHandshakeStatus())
                             {
                                 case NOT_HANDSHAKING:
+                                    // We were not handshaking, so just tell the app we are closed
                                     return -1;
 
                                 case NEED_TASK:
@@ -361,40 +429,47 @@ public class SslConnection extends AbstractAsyncConnection
                                     continue;
 
                                 case NEED_WRAP:
-                                    // we need to send some handshake data
+                                    // we need to send some handshake data (probably to send a close handshake).
-                                    if (!_flushUnwrap)
+                                    if (_flushRequiresFillToProgress)
-                                    {
+                                        return -1; // we were called from flush, so it can deal with sending the close handshake
-                                        _fillWrap = true;
+
+                                    // We need to call flush to cause the wrap to happen
+                                    _fillRequiresFlushToProgress = true;
                                     try
                                     {
+                                        // flushing an empty buffer will invoke the wrap mechanisms
                                         flush(BufferUtil.EMPTY_BUFFER);
+                                        // If encrypted output is all written, we can proceed with close
+                                        if (BufferUtil.isEmpty(_encryptedOutput))
+                                        {
+                                            _fillRequiresFlushToProgress = false;
+                                            return -1;
+                                        }
+                                        
+                                        // Otherwise return as if a normal fill and let a subsequent call
+                                        // return -1 to the caller.
+                                        return unwrapResult.bytesProduced();
                                     }
                                     catch(IOException e)
                                     {
+                                        LOG.debug(e);
+                                        // The flush failed, oh well nothing more to do than tell the app
+                                        // that the connection is closed.
                                         return -1;
                                     }
-                                        if (BufferUtil.hasContent(_netOut))
-                                            return 0;
-                                        _fillWrap = false;
                             }
-                                    return -1;
-
-                                default:
                             throw new IllegalStateException();
-                            }
-
-                        case BUFFER_UNDERFLOW:
-                            _underflown = true;
-
-                            //$FALL-THROUGH$ to deal with handshaking stuff
                             
                         default:
-                            // if we produced bytes, we don't care about the handshake state
+                            if (unwrapResult.getStatus()==Status.BUFFER_UNDERFLOW)
-                            if (_unwrapResult.bytesProduced() > 0)
+                                _needToFillMoreDataToProgress=true;
+                            
+                            // if we produced bytes, we don't care about the handshake state for now and it can be dealt with on another call to fill or flush
+                            if (unwrapResult.bytesProduced() > 0)
                             {
                                 if (app_in == buffer)
-                                    return _unwrapResult.bytesProduced();
+                                    return unwrapResult.bytesProduced();
-                                return BufferUtil.append(_appIn, buffer);
+                                return BufferUtil.append(_decryptedInput, buffer);
                             }
 
                             // Dang! we have to care about the handshake state
@@ -413,14 +488,17 @@ public class SslConnection extends AbstractAsyncConnection
 
                                 case NEED_WRAP:
                                     // we need to send some handshake data
-                                    if (_flushUnwrap)
+                                    if (_flushRequiresFillToProgress)
                                         return 0;
-                                    _fillWrap = true;
+                                    _fillRequiresFlushToProgress = true;
                                     flush(BufferUtil.EMPTY_BUFFER);
-                                    if (BufferUtil.hasContent(_netOut))
+                                    if (BufferUtil.isEmpty(_encryptedOutput))
-                                        return 0;
+                                    {
-                                    _fillWrap = false;
+                                        // the flush completed so continue 
+                                        _fillRequiresFlushToProgress = false;
                                         continue;
+                                    }
+                                    return 0;
 
                                 case NEED_UNWRAP:
                                     // if we just filled some net data
@@ -451,15 +529,15 @@ public class SslConnection extends AbstractAsyncConnection
             }
             finally
             {
-                if (_netIn != null && !_netIn.hasRemaining())
+                if (_encryptedInput != null && !_encryptedInput.hasRemaining())
                 {
-                    _bufferPool.release(_netIn);
+                    _bufferPool.release(_encryptedInput);
-                    _netIn = null;
+                    _encryptedInput = null;
                 }
-                if (_appIn != null && !_appIn.hasRemaining())
+                if (_decryptedInput != null && !_decryptedInput.hasRemaining())
                 {
-                    _bufferPool.release(_appIn);
+                    _bufferPool.release(_decryptedInput);
-                    _appIn = null;
+                    _decryptedInput = null;
                 }
                 LOG.debug("{} fill exit", SslConnection.this);
             }
@@ -468,85 +546,96 @@ public class SslConnection extends AbstractAsyncConnection
         @Override
         public synchronized int flush(ByteBuffer... appOuts) throws IOException
         {
-            // TODO: it is possible that an application flushes during the SSL handshake,
+            // The contract for flush does not require that all appOuts bytes are written
-            // TODO: the flush wraps 0 application bytes, and then a need for unwrap is
+            // or even that any appOut bytes are written!  If the connection is write block
-            // TODO: triggered. In that case, we need to save the appOuts and re-attempt
+            // or busy handshaking, then zero bytes may be taken from appOuts and this method
-            // TODO: to flush it at the first occasion (which may be on a fill ?)
+            // will return 0 (even if some handshake bytes were flushed and filled).
+            // it is the applications responsibility to call flush again - either in a busy loop
+            // or better yet by using AsyncEndPoint#write to do the flushing.
 
-            LOG.debug("{} flush enter {}", SslConnection.this, appOuts);
+            LOG.debug("{} flush enter {}", SslConnection.this, Arrays.toString(appOuts));
             try
             {
-                if (_netWriting)
+                if (_cannotAcceptMoreAppDataToFlush)
                     return 0;
 
                 // We will need a network buffer
-                if (_netOut == null)
+                if (_encryptedOutput == null)
-                    _netOut = _bufferPool.acquire(_sslEngine.getSession().getPacketBufferSize() * 2, _netDirect);
+                    _encryptedOutput = _bufferPool.acquire(_sslEngine.getSession().getPacketBufferSize() * 2, _encryptedDirectBuffers);
 
+                int consumed=0;
                 while (true)
                 {
                     // do the funky SSL thang!
-                    BufferUtil.compact(_netOut);
+                    // We call sslEngine.wrap to try to take bytes from appOut buffers and encrypt them into the _netOut buffer
-                    int pos = BufferUtil.flipToFill(_netOut);
+                    BufferUtil.compact(_encryptedOutput);
-                    _wrapResult = _sslEngine.wrap(appOuts, _netOut);
+                    int pos = BufferUtil.flipToFill(_encryptedOutput);
-                    LOG.debug("{} wrap {}", SslConnection.this, _wrapResult);
+                    SSLEngineResult wrapResult = _sslEngine.wrap(appOuts, _encryptedOutput);
-                    BufferUtil.flipToFlush(_netOut, pos);
+                    LOG.debug("{} wrap {}", SslConnection.this, wrapResult);
+                    BufferUtil.flipToFlush(_encryptedOutput, pos);
+                    consumed+=wrapResult.bytesConsumed();
 
-                    // and deal with the results
+                    // and deal with the results returned from the sslEngineWrap
-                    switch (_wrapResult.getStatus())
+                    switch (wrapResult.getStatus())
                     {
                         case CLOSED:
-                            if (BufferUtil.hasContent(_netOut))
+                            // The SSL engine has close, but there may be close handshake that needs to be written
+                            if (BufferUtil.hasContent(_encryptedOutput))
                             {
-                                _netWriting = true;
+                                _cannotAcceptMoreAppDataToFlush = true;
-                                getEndPoint().flush(_netOut);
+                                getEndPoint().flush(_encryptedOutput);
-                                if (BufferUtil.hasContent(_netOut))
+                                // If we failed to flush the close handshake then we will just pretend that 
-                                    return 0;
+                                // the write has progressed normally and let a subsequent call to flush (or WriteFlusher#onIncompleteFlushed)
+                                // to finish writing the close handshake.   The caller will find out about the close on a subsequent flush or fill.
+                                if (BufferUtil.hasContent(_encryptedOutput))
+                                    return consumed;
                             }
-                            if (_fillWrap)
+                            
-                                return 0;
+                            // If we we flushing because of a fill needing to wrap, return normally and it will handle the closed state.
+                            if (_fillRequiresFlushToProgress)
+                                return consumed;
+                            
+                            // otherwise it is an exception to write to a closed endpoint
                             throw new EofException();
 
                         case BUFFER_UNDERFLOW:
                             throw new IllegalStateException();
 
-                        case BUFFER_OVERFLOW:
-                            if (LOG.isDebugEnabled())
-                                LOG.debug("{} OVERFLOW {}", this, BufferUtil.toDetailString(_netOut));
-
-                            //$FALL-THROUGH$
                         default:
-                            // if we have net bytes, let's try to flush them
+                            if (LOG.isDebugEnabled())
-                            if (BufferUtil.hasContent(_netOut))
+                                LOG.debug("{} {} {}", this, wrapResult.getStatus(), BufferUtil.toDetailString(_encryptedOutput));
-                            {
-                                getEndPoint().flush(_netOut);
-                                return _wrapResult.bytesConsumed();
-                            }
                             
-                            // Dang! we have to deal with handshake state
+                            // if we have net bytes, let's try to flush them
+                            if (BufferUtil.hasContent(_encryptedOutput))
+                                getEndPoint().flush(_encryptedOutput);
+
+                            // But we also might have more to do for the handshaking state.
                             switch (_sslEngine.getHandshakeStatus())
                             {
                                 case NOT_HANDSHAKING:
-                                    // we just didn't write anything. Strange?
+                                    // Return with the number of bytes consumed (which may be 0)
-                                    return 0;
+                                    return consumed;
 
                                 case NEED_TASK:
-                                    // run the task
+                                    // run the task and continue
                                     _sslEngine.getDelegatedTask().run();
                                     continue;
 
                                 case NEED_WRAP:
-                                    // Hey we just wrapped!
+                                    // Hey we just wrapped! Oh well who knows what the sslEngine is thinking, so continue and we will wrap again
                                     continue;
 
                                 case NEED_UNWRAP:
-                                    // Were we were not called from fill and not reading anyway
+                                    // Ah we need to fill some data so we can write.
-                                    if (!_fillWrap && !_readInterest.isInterested())
+                                    // So if we were not called from fill and the app is not reading anyway
+                                    if (!_fillRequiresFlushToProgress && !_readInterest.isInterested())
                                     {
-                                        _flushUnwrap = true;
+                                        // Tell the onFillable method that there might be a write to complete 
+                                        // TODO move this to the writeFlusher?
+                                        _flushRequiresFillToProgress = true;
                                         fill(BufferUtil.EMPTY_BUFFER);
                                     }
-                                    return 0;
+                                    return consumed;
 
                                 case FINISHED:
                                     throw new IllegalStateException();
@@ -569,10 +658,10 @@ public class SslConnection extends AbstractAsyncConnection
 
         private void releaseNetOut()
         {
-            if (_netOut != null && !_netOut.hasRemaining())
+            if (_encryptedOutput != null && !_encryptedOutput.hasRemaining())
             {
-                _bufferPool.release(_netOut);
+                _bufferPool.release(_encryptedOutput);
-                _netOut = null;
+                _encryptedOutput = null;
                 if (_sslEngine.isOutboundDone())
                     getEndPoint().shutdownOutput();
             }
@@ -638,7 +727,7 @@ public class SslConnection extends AbstractAsyncConnection
         @Override
         public String toString()
         {
-            return String.format("%s{%s%s%s}", super.toString(), _readInterest.isInterested() ? "R" : "", _writeFlusher.isWriting() ? "W" : "", _netWriting ? "w" : "");
+            return String.format("%s{%s%s%s}", super.toString(), _readInterest.isInterested() ? "R" : "", _writeFlusher.isWriting() ? "W" : "", _cannotAcceptMoreAppDataToFlush ? "w" : "");
         }
 
     }

jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointInterestsTest.java

@@ -0,0 +1,192 @@
+// ========================================================================
+// Copyright (c) 2012-2012 Mort Bay Consulting Pty. Ltd.
+// ------------------------------------------------------------------------
+// All rights reserved. This program and the accompanying materials
+// are made available under the terms of the Eclipse Public License v1.0
+// and Apache License v2.0 which accompanies this distribution.
+// The Eclipse Public License is available at
+// http://www.eclipse.org/legal/epl-v10.html
+// The Apache License v2.0 is available at
+// http://www.opensource.org/licenses/apache2.0.php
+// You may elect to redistribute this code under either of these licenses.
+// ========================================================================
+
+package org.eclipse.jetty.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SelectChannelEndPointInterestsTest
+{
+    private QueuedThreadPool threadPool;
+    private ScheduledExecutorService scheduler;
+    private ServerSocketChannel connector;
+    private SelectorManager selectorManager;
+
+    public void init(final Interested interested) throws Exception
+    {
+        threadPool = new QueuedThreadPool();
+        threadPool.start();
+
+        scheduler = Executors.newSingleThreadScheduledExecutor();
+
+        connector = ServerSocketChannel.open();
+        connector.bind(new InetSocketAddress("localhost", 0));
+
+        selectorManager = new SelectorManager()
+        {
+            @Override
+            protected void execute(Runnable task)
+            {
+                threadPool.execute(task);
+            }
+
+            @Override
+            protected AsyncEndPoint newEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey selectionKey) throws IOException
+            {
+                return new SelectChannelEndPoint(channel, selector, selectionKey, scheduler, 60000)
+                {
+                    @Override
+                    protected void onIncompleteFlush()
+                    {
+                        super.onIncompleteFlush();
+                        interested.onIncompleteFlush();
+                    }
+                };
+            }
+
+            @Override
+            public AsyncConnection newConnection(SocketChannel channel, final AsyncEndPoint endPoint, Object attachment)
+            {
+                return new AbstractAsyncConnection(endPoint, threadPool)
+                {
+                    @Override
+                    public void onFillable()
+                    {
+                        interested.onFillable(endPoint, this);
+                    }
+                };
+            }
+        };
+        selectorManager.start();
+    }
+
+    @Test
+    public void testReadBlockedThenWriteBlockedThenReadableThenWritable() throws Exception
+    {
+        final AtomicInteger size = new AtomicInteger(1024 * 1024);
+        final AtomicReference<Exception> failure = new AtomicReference<>();
+        final CountDownLatch latch1 = new CountDownLatch(1);
+        final CountDownLatch latch2 = new CountDownLatch(1);
+        final AtomicBoolean writeBlocked = new AtomicBoolean();
+        init(new Interested()
+        {
+            @Override
+            public void onFillable(AsyncEndPoint endPoint, AbstractAsyncConnection connection)
+            {
+                ByteBuffer input = BufferUtil.allocate(2);
+                int read = fill(endPoint, input);
+
+                if (read == 1)
+                {
+                    byte b = input.get();
+                    if (b == 1)
+                    {
+                        connection.fillInterested();
+
+                        ByteBuffer output = ByteBuffer.allocate(size.get());
+                        endPoint.write(null, new Callback.Empty<>(), output);
+
+                        latch1.countDown();
+                    }
+                    else
+                    {
+                        latch2.countDown();
+                    }
+                }
+                else
+                {
+                    failure.set(new Exception("Unexpectedly read " + read + " bytes"));
+                }
+            }
+
+            @Override
+            public void onIncompleteFlush()
+            {
+                writeBlocked.set(true);
+            }
+
+            private int fill(AsyncEndPoint endPoint, ByteBuffer buffer)
+            {
+                try
+                {
+                    return endPoint.fill(buffer);
+                }
+                catch (IOException x)
+                {
+                    failure.set(x);
+                    return 0;
+                }
+            }
+        });
+
+        Socket client = new Socket();
+        client.connect(connector.getLocalAddress());
+        client.setSoTimeout(5000);
+
+        SocketChannel server = connector.accept();
+        server.configureBlocking(false);
+        selectorManager.accept(server);
+
+        OutputStream clientOutput = client.getOutputStream();
+        clientOutput.write(1);
+        clientOutput.flush();
+        Assert.assertTrue(latch1.await(5, TimeUnit.SECONDS));
+
+        // We do not read to keep the socket write blocked
+
+        clientOutput.write(2);
+        clientOutput.flush();
+        Assert.assertTrue(latch2.await(5, TimeUnit.SECONDS));
+
+        // Sleep before reading to allow waking up the server only for read
+        Thread.sleep(1000);
+
+        // Now read what was written, waking up the server for write
+        InputStream clientInput = client.getInputStream();
+        while (size.getAndDecrement() > 0)
+            clientInput.read();
+
+        client.close();
+
+        Assert.assertNull(failure.get());
+    }
+
+    private interface Interested
+    {
+        void onFillable(AsyncEndPoint endPoint, AbstractAsyncConnection connection);
+
+        void onIncompleteFlush();
+    }
+
+}

jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointSslTest.java

@@ -1,18 +1,12 @@
 package org.eclipse.jetty.io;
 
-import static org.hamcrest.Matchers.greaterThan;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.Socket;
 import java.nio.ByteBuffer;
 import java.nio.channels.SocketChannel;
-
+import java.util.concurrent.TimeUnit;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
 import javax.net.ssl.SSLEngineResult.HandshakeStatus;
@@ -27,6 +21,12 @@ import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
 
+import static org.hamcrest.Matchers.greaterThan;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
 
 public class SelectChannelEndPointSslTest extends SelectChannelEndPointTest
 {
@@ -60,6 +60,8 @@ public class SelectChannelEndPointSslTest extends SelectChannelEndPointTest
 
         AsyncConnection appConnection = super.newConnection(channel,sslConnection.getSslEndPoint());
         sslConnection.getSslEndPoint().setAsyncConnection(appConnection);
+        _manager.connectionOpened(appConnection);
+
         return sslConnection;
     }
 
@@ -188,15 +190,15 @@ public class SelectChannelEndPointSslTest extends SelectChannelEndPointTest
 
     @Test
     @Override
-    public void testWriteBlock() throws Exception
+    public void testWriteBlocked() throws Exception
     {
-        super.testWriteBlock();
+        super.testWriteBlocked();
     }
 
     @Override
-    public void testBlockRead() throws Exception
+    public void testReadBlocked() throws Exception
     {
-        super.testBlockRead();
+        super.testReadBlocked();
     }
 
     @Override
@@ -229,8 +231,8 @@ public class SelectChannelEndPointSslTest extends SelectChannelEndPointTest
             assertEquals(c,(char)b);
         }
 
-        // Set Max idle
+        assertTrue(_lastEndPointLatch.await(1, TimeUnit.SECONDS));
-        _lastEndp.setIdleTimeout(500);
+        _lastEndPoint.setIdleTimeout(500);
 
         // Write 8 and cause block waiting for 10
         _blockAt=10;
@@ -247,7 +249,7 @@ public class SelectChannelEndPointSslTest extends SelectChannelEndPointTest
 
         Thread.sleep(1000);
 
-        assertFalse(_lastEndp.isOpen());
+        assertFalse(_lastEndPoint.isOpen());
     }
 
     @Test

jetty-io/src/test/java/org/eclipse/jetty/io/SelectChannelEndPointTest.java

@@ -50,7 +50,8 @@ import static org.junit.Assert.assertTrue;
 
 public class SelectChannelEndPointTest
 {
-    protected volatile AsyncEndPoint _lastEndp;
+    protected CountDownLatch _lastEndPointLatch;
+    protected volatile AsyncEndPoint _lastEndPoint;
     protected ServerSocketChannel _connector;
     protected QueuedThreadPool _threadPool = new QueuedThreadPool();
     protected ScheduledExecutorService _scheduler = Executors.newSingleThreadScheduledExecutor();
@@ -72,7 +73,8 @@ public class SelectChannelEndPointTest
         protected SelectChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey selectionKey) throws IOException
         {
             SelectChannelEndPoint endp = new SelectChannelEndPoint(channel, selectSet, selectionKey, _scheduler, 60000);
-            _lastEndp = endp;
+            _lastEndPoint = endp;
+            _lastEndPointLatch.countDown();
             return endp;
         }
     };
@@ -85,7 +87,8 @@ public class SelectChannelEndPointTest
     public void startManager() throws Exception
     {
         _writeCount = 1;
-        _lastEndp = null;
+        _lastEndPoint = null;
+        _lastEndPointLatch = new CountDownLatch(1);
         _connector = ServerSocketChannel.open();
         _connector.socket().bind(null);
         _threadPool.start();
@@ -253,13 +256,14 @@ public class SelectChannelEndPointTest
         }
         client.close();
 
-        int i = 0;
+        for (int i = 0; i < 10; ++i)
-        while (server.isOpen())
         {
+            if (server.isOpen())
                 Thread.sleep(10);
-            if (++i == 10)
+            else
-                Assert.fail();
+                break;
         }
+        assertFalse(server.isOpen());
     }
 
     @Test
@@ -301,7 +305,6 @@ public class SelectChannelEndPointTest
         client.getOutputStream().write("Goodbye Cruel TLS".getBytes("UTF-8"));
         client.shutdownOutput();
 
-
         // Verify echo server to client
         for (char c : "Goodbye Cruel TLS".toCharArray())
         {
@@ -315,7 +318,7 @@ public class SelectChannelEndPointTest
     }
 
     @Test
-    public void testBlockRead() throws Exception
+    public void testReadBlocked() throws Exception
     {
         Socket client = newClient();
 
@@ -335,11 +338,8 @@ public class SelectChannelEndPointTest
         clientOutputStream.write("12345678".getBytes("UTF-8"));
         clientOutputStream.flush();
 
-        long wait = System.currentTimeMillis() + 1000;
+        Assert.assertTrue(_lastEndPointLatch.await(1, TimeUnit.SECONDS));
-        while (_lastEndp == null && System.currentTimeMillis() < wait)
+        _lastEndPoint.setIdleTimeout(10 * specifiedTimeout);
-            Thread.yield();
-
-        _lastEndp.setIdleTimeout(10 * specifiedTimeout);
         Thread.sleep((11 * specifiedTimeout) / 10);
 
         long start = System.currentTimeMillis();
@@ -390,25 +390,28 @@ public class SelectChannelEndPointTest
             assertEquals(c, (char)b);
         }
 
-        // Set Max idle
+        Assert.assertTrue(_lastEndPointLatch.await(1, TimeUnit.SECONDS));
-        _lastEndp.setIdleTimeout(500);
+        int idleTimeout = 500;
+        _lastEndPoint.setIdleTimeout(idleTimeout);
 
         // read until idle shutdown received
         long start = System.currentTimeMillis();
         int b = client.getInputStream().read();
         assertEquals(-1, b);
         long idle = System.currentTimeMillis() - start;
-        assertTrue(idle > 400);
+        assertTrue(idle > idleTimeout / 2);
-        assertTrue(idle < 2000);
+        assertTrue(idle < idleTimeout * 2);
 
         // But endpoint may still be open for a little bit.
-        if (_lastEndp.isOpen())
+        for (int i = 0; i < 10; ++i)
-            Thread.sleep(2000);
+        {
-
+            if (_lastEndPoint.isOpen())
-        // endpoint is closed.
+                Thread.sleep(2 * idleTimeout / 10);
-        assertFalse(_lastEndp.isOpen());
+            else
+                break;
+        }
+        assertFalse(_lastEndPoint.isOpen());
     }
-
 
     @Test
     public void testBlockedReadIdle() throws Exception
@@ -434,8 +437,9 @@ public class SelectChannelEndPointTest
             assertEquals(c, (char)b);
         }
 
-        // Set Max idle
+        Assert.assertTrue(_lastEndPointLatch.await(1, TimeUnit.SECONDS));
-        _lastEndp.setIdleTimeout(500);
+        int idleTimeout = 500;
+        _lastEndPoint.setIdleTimeout(idleTimeout);
 
         // Write 8 and cause block waiting for 10
         _blockAt = 10;
@@ -447,8 +451,8 @@ public class SelectChannelEndPointTest
         int b = client.getInputStream().read();
         assertEquals('E', b);
         long idle = System.currentTimeMillis() - start;
-        assertTrue(idle > 400);
+        assertTrue(idle > idleTimeout / 2);
-        assertTrue(idle < 2000);
+        assertTrue(idle < idleTimeout * 2);
 
         for (char c : "E: 12345678".toCharArray())
         {
@@ -458,13 +462,13 @@ public class SelectChannelEndPointTest
         }
 
         // But endpoint is still open.
-        assertTrue(_lastEndp.isOpen());
+        assertTrue(_lastEndPoint.isOpen());
 
         // Wait for another idle callback
-        Thread.sleep(2000);
+        Thread.sleep(idleTimeout * 2);
         // endpoint is closed.
 
-        assertFalse(_lastEndp.isOpen());
+        assertFalse(_lastEndPoint.isOpen());
     }
 
     @Test
@@ -489,9 +493,8 @@ public class SelectChannelEndPointTest
         out.write(count);
         out.flush();
 
-        while (_lastEndp == null)
+        Assert.assertTrue(_lastEndPointLatch.await(1, TimeUnit.SECONDS));
-            Thread.sleep(10);
+        _lastEndPoint.setIdleTimeout(5000);
-        _lastEndp.setIdleTimeout(5000);
 
         new Thread()
         {
@@ -536,8 +539,8 @@ public class SelectChannelEndPointTest
                     System.err.println("latch=" + latch.getCount());
                     System.err.println("time=" + (now - start));
                     System.err.println("last=" + (now - last));
-                    System.err.println("endp=" + _lastEndp);
+                    System.err.println("endp=" + _lastEndPoint);
-                    System.err.println("conn=" + _lastEndp.getAsyncConnection());
+                    System.err.println("conn=" + _lastEndPoint.getAsyncConnection());
 
                     e.printStackTrace();
                 }
@@ -572,7 +575,7 @@ public class SelectChannelEndPointTest
     }
 
     @Test
-    public void testWriteBlock() throws Exception
+    public void testWriteBlocked() throws Exception
     {
         Socket client = newClient();
 
@@ -592,10 +595,8 @@ public class SelectChannelEndPointTest
         for (int i = 0; i < _writeCount; i++)
         {
             if (i % 1000 == 0)
-            {
-                //System.out.println(i);
                 TimeUnit.MILLISECONDS.sleep(200);
-            }
+
             // Verify echo server to client
             for (int j = 0; j < data.length(); j++)
             {
@@ -604,18 +605,20 @@ public class SelectChannelEndPointTest
                 assertTrue(b > 0);
                 assertEquals("test-" + i + "/" + j, c, (char)b);
             }
-            if (i == 0)
-                _lastEndp.setIdleTimeout(60000);
-        }
 
+            if (i == 0)
+                _lastEndPoint.setIdleTimeout(60000);
+        }
 
         client.close();
 
-        int i = 0;
+        for (int i = 0; i < 10; ++i)
-        while (server.isOpen())
         {
-            assert (i++ < 10);
+            if (server.isOpen())
                 Thread.sleep(10);
-        }
+            else
+                break;
+        }
+        assertFalse(server.isOpen());
     }
 }

jetty-io/src/test/java/org/eclipse/jetty/io/SslConnectionTest.java

@@ -56,9 +56,9 @@ public class SslConnectionTest
 
             AsyncConnection appConnection = new TestConnection(sslConnection.getSslEndPoint());
             sslConnection.getSslEndPoint().setAsyncConnection(appConnection);
+            connectionOpened(appConnection);
 
             return sslConnection;
-
         }
 
         @Override

jetty-io/src/test/java/org/eclipse/jetty/io/WriteFlusherTest.java

@@ -0,0 +1,191 @@
+package org.eclipse.jetty.io;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.assertFalse;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.FutureCallback;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class WriteFlusherTest
+{
+    ByteArrayEndPoint _endp;
+    final AtomicBoolean _flushIncomplete = new AtomicBoolean(false);
+    WriteFlusher _flusher;
+    final String _context = new String("Context");
+    
+    @Before
+    public void before()
+    {
+        _endp = new ByteArrayEndPoint(new byte[]{},10);
+        _flushIncomplete.set(false);
+        _flusher = new WriteFlusher(_endp)
+        {            
+            @Override
+            protected void onIncompleteFlushed()
+            {
+                _flushIncomplete.set(true);
+            }
+        };
+    }
+    
+    @After
+    public void after()
+    {
+        
+    }
+    
+    @Test
+    public void testCompleteNoBlocking() throws Exception
+    {
+        _endp.setGrowOutput(true);
+        
+        FutureCallback<String> callback = new FutureCallback<>();
+        _flusher.write(_context,callback,BufferUtil.toBuffer("How "),BufferUtil.toBuffer("now "),BufferUtil.toBuffer("brown "),BufferUtil.toBuffer("cow!"));
+        assertTrue(callback.isDone());
+        assertFalse(_flushIncomplete.get());
+        assertEquals(_context,callback.get());
+        assertEquals("How now brown cow!",_endp.takeOutputString());
+    }
+    
+    @Test
+    public void testClosedNoBlocking() throws Exception
+    {
+        _endp.close();
+        
+        FutureCallback<String> callback = new FutureCallback<>();
+        _flusher.write(_context,callback,BufferUtil.toBuffer("How "),BufferUtil.toBuffer("now "),BufferUtil.toBuffer("brown "),BufferUtil.toBuffer("cow!"));
+        assertTrue(callback.isDone());
+        assertFalse(_flushIncomplete.get());
+        try
+        {
+            assertEquals(_context,callback.get());
+            Assert.fail();
+        }
+        catch(ExecutionException e)
+        {
+            Throwable cause = e.getCause();
+            Assert.assertTrue(cause instanceof IOException);
+            Assert.assertThat(cause.getMessage(),Matchers.containsString("CLOSED"));
+        }
+        assertEquals("",_endp.takeOutputString());
+    }
+    
+
+    @Test
+    public void testCompleteBlocking() throws Exception
+    {        
+        FutureCallback<String> callback = new FutureCallback<>();
+        _flusher.write(_context,callback,BufferUtil.toBuffer("How "),BufferUtil.toBuffer("now "),BufferUtil.toBuffer("brown "),BufferUtil.toBuffer("cow!"));
+        assertFalse(callback.isDone());
+        assertFalse(callback.isCancelled());
+        
+        assertTrue(_flushIncomplete.get());
+        try
+        {
+            assertEquals(_context,callback.get(10,TimeUnit.MILLISECONDS));
+            Assert.fail();
+        }
+        catch (TimeoutException to)
+        {
+            _flushIncomplete.set(false);
+        }
+
+        assertEquals("How now br",_endp.takeOutputString());
+        _flusher.completeWrite();
+        assertTrue(callback.isDone());
+        assertEquals(_context,callback.get());
+        assertEquals("own cow!",_endp.takeOutputString());
+        assertFalse(_flushIncomplete.get());
+    }
+    
+    @Test
+    public void testCloseWhileBlocking() throws Exception
+    {
+        FutureCallback<String> callback = new FutureCallback<>();
+        _flusher.write(_context,callback,BufferUtil.toBuffer("How "),BufferUtil.toBuffer("now "),BufferUtil.toBuffer("brown "),BufferUtil.toBuffer("cow!"));
+
+        assertFalse(callback.isDone());
+        assertFalse(callback.isCancelled());
+        
+        assertTrue(_flushIncomplete.get());
+        try
+        {
+            assertEquals(_context,callback.get(10,TimeUnit.MILLISECONDS));
+            Assert.fail();
+        }
+        catch (TimeoutException to)
+        {
+            _flushIncomplete.set(false);
+        }
+
+        assertEquals("How now br",_endp.takeOutputString());
+        _endp.close();
+        _flusher.completeWrite();
+        assertTrue(callback.isDone());
+        assertFalse(_flushIncomplete.get());
+        try
+        {
+            assertEquals(_context,callback.get());
+            Assert.fail();
+        }
+        catch(ExecutionException e)
+        {
+            Throwable cause = e.getCause();
+            Assert.assertTrue(cause instanceof IOException);
+            Assert.assertThat(cause.getMessage(),Matchers.containsString("CLOSED"));
+        }
+        assertEquals("",_endp.takeOutputString());
+    }
+
+    @Test
+    public void testFailWhileBlocking() throws Exception
+    {
+        FutureCallback<String> callback = new FutureCallback<>();
+        _flusher.write(_context,callback,BufferUtil.toBuffer("How "),BufferUtil.toBuffer("now "),BufferUtil.toBuffer("brown "),BufferUtil.toBuffer("cow!"));
+
+        assertFalse(callback.isDone());
+        assertFalse(callback.isCancelled());
+        
+        assertTrue(_flushIncomplete.get());
+        try
+        {
+            assertEquals(_context,callback.get(10,TimeUnit.MILLISECONDS));
+            Assert.fail();
+        }
+        catch (TimeoutException to)
+        {
+            _flushIncomplete.set(false);
+        }
+
+        assertEquals("How now br",_endp.takeOutputString());
+        _flusher.failed(new IOException("Failure"));
+        _flusher.completeWrite();
+        assertTrue(callback.isDone());
+        assertFalse(_flushIncomplete.get());
+        try
+        {
+            assertEquals(_context,callback.get());
+            Assert.fail();
+        }
+        catch(ExecutionException e)
+        {
+            Throwable cause = e.getCause();
+            Assert.assertTrue(cause instanceof IOException);
+            Assert.assertThat(cause.getMessage(),Matchers.containsString("Failure"));
+        }
+        assertEquals("",_endp.takeOutputString());
+    }
+    
+}

jetty-server/src/main/java/org/eclipse/jetty/server/ssl/SslSelectChannelConnector.java

@@ -15,7 +15,6 @@ package org.eclipse.jetty.server.ssl;
 
 import java.io.IOException;
 import java.nio.channels.SocketChannel;
-
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLSession;
@@ -41,6 +40,8 @@ import org.eclipse.jetty.util.ssl.SslContextFactory;
 public class SslSelectChannelConnector extends SelectChannelConnector 
 {
     public SslSelectChannelConnector(Server server)
+        SslConnection.DecryptedEndPoint ssl_endp = (SslConnection.DecryptedEndPoint)request.getHttpChannel().getEndPoint();
+            getSelectorManager().connectionOpened(delegate);
     {
         super(server,true);
     }

jetty-server/src/test/java/org/eclipse/jetty/server/ConnectorTimeoutTest.java

@@ -149,8 +149,8 @@ public abstract class ConnectorTimeoutTest extends HttpServerTestFixture
 
         // Get the server side endpoint
         EndPoint endp = endpoint.exchange(null,10,TimeUnit.SECONDS);
-        if (endp instanceof SslConnection.SslEndPoint)
+        if (endp instanceof SslConnection.DecryptedEndPoint)
-            endp=((SslConnection.SslEndPoint)endp).getAsyncConnection().getEndPoint();
+            endp=((SslConnection.DecryptedEndPoint)endp).getAsyncConnection().getEndPoint();
         
         // read the response
         String result=IO.toString(is);
@@ -222,8 +222,8 @@ public abstract class ConnectorTimeoutTest extends HttpServerTestFixture
 
         // Get the server side endpoint
         EndPoint endp = endpoint.exchange(null,10,TimeUnit.SECONDS);
-        if (endp instanceof SslConnection.SslEndPoint)
+        if (endp instanceof SslConnection.DecryptedEndPoint)
-            endp=((SslConnection.SslEndPoint)endp).getAsyncConnection().getEndPoint();
+            endp=((SslConnection.DecryptedEndPoint)endp).getAsyncConnection().getEndPoint();
         
         // read the response
         String result=IO.toString(is);

jetty-server/src/test/java/org/eclipse/jetty/server/ssl/SslRenegotiateTest.java

@@ -1,262 +0,0 @@
-package org.eclipse.jetty.server.ssl;
-//========================================================================
-//Copyright 2011-2012 Mort Bay Consulting Pty. Ltd.
-//------------------------------------------------------------------------
-//All rights reserved. This program and the accompanying materials
-//are made available under the terms of the Eclipse Public License v1.0
-//and Apache License v2.0 which accompanies this distribution.
-//The Eclipse Public License is available at
-//http://www.eclipse.org/legal/epl-v10.html
-//The Apache License v2.0 is available at
-//http://www.opensource.org/licenses/apache2.0.php
-//You may elect to redistribute this code under either of these licenses.
-//========================================================================
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.nio.ByteBuffer;
-import java.nio.channels.SocketChannel;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLEngine;
-import javax.net.ssl.SSLEngineResult;
-import javax.net.ssl.SSLEngineResult.HandshakeStatus;
-import javax.net.ssl.SSLProtocolException;
-import javax.net.ssl.SSLSession;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.eclipse.jetty.server.Connector;
-import org.eclipse.jetty.server.Request;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.handler.AbstractHandler;
-import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
-import org.eclipse.jetty.util.BufferUtil;
-import org.eclipse.jetty.util.StringUtil;
-import org.eclipse.jetty.util.log.Log;
-import org.eclipse.jetty.util.log.Logger;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.junit.Ignore;
-import org.junit.Test;
-
-public class SslRenegotiateTest
-{
-    private static final Logger LOG = Log.getLogger(SslRenegotiateTest.class);
-
-    private ByteBuffer _outAppB;
-    private ByteBuffer _outPacketB;
-    private ByteBuffer _inAppB;
-    private ByteBuffer _inPacketB;
-    private SocketChannel _socket;
-    private SSLEngine _engine;
-
-    @Test
-    public void testRenegNIO() throws Exception
-    {
-        // TODO This test breaks on JVMs with the fix
-//        doRequests(new SslSelectChannelConnector(),true);
-    }
-
-    @Test
-    @Ignore
-    public void testNoRenegNIO() throws Exception
-    {
-        doRequests(new SslSelectChannelConnector(),false);
-    }
-
-    @Test
-    public void testRenegBIO() throws Exception
-    {
-        // TODO - this test is too non deterministic due to call back timing
-//        doRequests(new SslSocketConnector(),true);
-    }
-
-    @Test
-    public void testNoRenegBIO() throws Exception
-    {
-        // TODO - this test is too non deterministic due to call back timing
-//        doRequests(new SslSocketConnector(),false);
-    }
-
-    private void doRequests(SslConnector connector, boolean reneg) throws Exception
-    {
-        Server server=new Server();
-        try
-        {
-            String keystore = MavenTestingUtils.getTestResourceFile("keystore").getAbsolutePath();
-            SslContextFactory cf = connector.getSslContextFactory();
-            cf.setKeyStorePath(keystore);
-            cf.setKeyStorePassword("storepwd");
-            cf.setKeyManagerPassword("keypwd");
-            cf.setAllowRenegotiate(reneg);
-
-            server.setConnectors(new Connector[] { connector });
-            server.setHandler(new HelloWorldHandler());
-
-            server.start();
-
-            SocketAddress addr = new InetSocketAddress("localhost",connector.getLocalPort());
-            _socket = SocketChannel.open(addr);
-            _socket.configureBlocking(true);
-
-            SSLContext context=SSLContext.getInstance("SSL");
-            context.init( null, SslContextFactory.TRUST_ALL_CERTS, new java.security.SecureRandom() );
-
-            _engine = context.createSSLEngine();
-            _engine.setUseClientMode(true);
-            SSLSession session=_engine.getSession();
-
-            _outAppB = ByteBuffer.allocate(session.getApplicationBufferSize());
-            _outPacketB = ByteBuffer.allocate(session.getPacketBufferSize());
-            _inAppB = ByteBuffer.allocate(session.getApplicationBufferSize());
-            _inPacketB = ByteBuffer.allocate(session.getPacketBufferSize());
-
-
-            _outAppB.put("GET /1 HTTP/1.1\r\nHost: localhost\r\n\r\n".getBytes(StringUtil.__ISO_8859_1));
-            _outAppB.flip();
-
-            _engine.beginHandshake();
-
-            runHandshake();
-
-            doWrap();
-            doUnwrap();
-            _inAppB.flip();
-            String response=BufferUtil.toString(_inAppB);
-            // System.err.println(response);
-            assertTrue(response.startsWith("HTTP/1.1 200 OK"));
-
-            if (response.indexOf("HELLO WORLD")<0)
-            {
-                _inAppB.clear();
-                doUnwrap();
-                _inAppB.flip();
-                response=BufferUtil.toString(_inAppB);
-            }
-
-            assertTrue(response.indexOf("HELLO WORLD")>=0);
-
-            _inAppB.clear();
-            _outAppB.clear();
-            _outAppB.put("GET /2 HTTP/1.1\r\nHost: localhost\r\n\r\n".getBytes(StringUtil.__ISO_8859_1));
-            _outAppB.flip();
-
-            try
-            {
-                session.invalidate();
-                _engine.beginHandshake();
-                runHandshake();
-
-                doWrap();
-                doUnwrap();
-                _inAppB.flip();
-                response=BufferUtil.toString(_inAppB);
-                assertTrue(response.startsWith("HTTP/1.1 200 OK"));
-                assertTrue(response.indexOf("HELLO WORLD")>0);
-
-                assertTrue(reneg);
-            }
-            catch(IOException e)
-            {
-                if (!(e instanceof SSLProtocolException))
-                {
-                    if (reneg)
-                        LOG.warn(e);
-                    assertFalse(reneg);
-                }
-            }
-        }
-        finally
-        {
-            server.stop();
-            server.join();
-        }
-    }
-
-    void runHandshake() throws Exception
-    {
-        while (true)
-        {
-            switch(_engine.getHandshakeStatus())
-            {
-                case NEED_TASK:
-                {
-                    //System.err.println("running task");
-                    _engine.getDelegatedTask().run();
-                    break;
-                }
-
-                case NEED_WRAP:
-                {
-                    doWrap();
-                    break;
-                }
-
-                case NEED_UNWRAP:
-                {
-                    doUnwrap();
-                    break;
-                }
-
-                default:
-                    return;
-            }
-        }
-    }
-
-    private void doWrap() throws Exception
-    {
-        _engine.wrap(_outAppB,_outPacketB);
-//        System.err.println("wrapped "+result.bytesConsumed()+" to "+result.bytesProduced());
-        _outPacketB.flip();
-        while (_outPacketB.hasRemaining())
-        {
-            int p = _outPacketB.remaining();
-            int l =_socket.write(_outPacketB);
-            // System.err.println("wrote "+l+" of "+p);
-        }
-        _outPacketB.clear();
-    }
-
-    private void doUnwrap() throws Exception
-    {
-        _inPacketB.clear();
-        int l=_socket.read(_inPacketB);
-        // System.err.println("read "+l);
-        if (l<0)
-            throw new IOException("EOF");
-
-        _inPacketB.flip();
-
-        SSLEngineResult result;
-        do
-        {
-            result =_engine.unwrap(_inPacketB,_inAppB);
-//            System.err.println("unwrapped "+result.bytesConsumed()+" to "+result.bytesProduced()+" "+_engine.getHandshakeStatus());
-
-        }
-        while(result.bytesConsumed()>0 &&
-              _inPacketB.remaining()>0 &&
-              (_engine.getHandshakeStatus()==HandshakeStatus.NEED_UNWRAP || _engine.getHandshakeStatus()==HandshakeStatus.NOT_HANDSHAKING));
-    }
-
-    private static class HelloWorldHandler extends AbstractHandler
-    {
-        public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
-        {
-            baseRequest.setHandled(true);
-            //System.err.println("HELLO WORLD HANDLING");
-
-//            System.err.println("hello "+baseRequest.getUri());
-            byte[] b=("HELLO WORLD "+baseRequest.getUri()).getBytes(StringUtil.__UTF8);
-            response.setContentLength(b.length);
-            response.getOutputStream().write(b);
-            response.getOutputStream().flush();
-        }
-    }
-}

jetty-spdy/spdy-core/pom.xml

@@ -36,12 +36,6 @@
             <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>${slf4j-version}</version>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 
 </project>

jetty-spdy/spdy-core/src/main/java/org/eclipse/jetty/spdy/Controller.java

@@ -1,18 +1,15 @@
-/*
+// ========================================================================
- * Copyright (c) 2012 the original author or authors.
+// Copyright 2012-2012 Mort Bay Consulting Pty. Ltd.
- *
+// ------------------------------------------------------------------------
- * Licensed under the Apache License, Version 2.0 (the "License");
+// All rights reserved. This program and the accompanying materials
- * you may not use this file except in compliance with the License.
+// are made available under the terms of the Eclipse Public License v1.0
- * You may obtain a copy of the License at
+// and Apache License v2.0 which accompanies this distribution.
- *
+// The Eclipse Public License is available at
- *     http://www.apache.org/licenses/LICENSE-2.0
+// http://www.eclipse.org/legal/epl-v10.html
- *
+// The Apache License v2.0 is available at
- * Unless required by applicable law or agreed to in writing, software
+// http://www.opensource.org/licenses/apache2.0.php
- * distributed under the License is distributed on an "AS IS" BASIS,
+// You may elect to redistribute this code under either of these licenses.
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// ========================================================================
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 
 package org.eclipse.jetty.spdy;
 

jetty-spdy/spdy-core/src/main/java/org/eclipse/jetty/spdy/ISession.java

@@ -1,18 +1,15 @@
-/*
+// ========================================================================
- * Copyright (c) 2012 the original author or authors.
+// Copyright 2012-2012 Mort Bay Consulting Pty. Ltd.
- *
+// ------------------------------------------------------------------------
- * Licensed under the Apache License, Version 2.0 (the "License");
+// All rights reserved. This program and the accompanying materials
- * you may not use this file except in compliance with the License.
+// are made available under the terms of the Eclipse Public License v1.0
- * You may obtain a copy of the License at
+// and Apache License v2.0 which accompanies this distribution.
- *
+// The Eclipse Public License is available at
- *     http://www.apache.org/licenses/LICENSE-2.0
+// http://www.eclipse.org/legal/epl-v10.html
- *
+// The Apache License v2.0 is available at
- * Unless required by applicable law or agreed to in writing, software
+// http://www.opensource.org/licenses/apache2.0.php
- * distributed under the License is distributed on an "AS IS" BASIS,
+// You may elect to redistribute this code under either of these licenses.
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// ========================================================================
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 
 package org.eclipse.jetty.spdy;
 
@@ -35,4 +32,10 @@ public interface ISession extends Session
     public <C> void control(IStream stream, ControlFrame frame, long timeout, TimeUnit unit, Callback<C> callback, C context);
 
     public <C> void data(IStream stream, DataInfo dataInfo, long timeout, TimeUnit unit, Callback<C> callback, C context);
+
+    /**
+     * <p>Gracefully shuts down this session.</p>
+     * <p>A special item is queued that will close the connection when it will be dequeued.</p>
+     */
+    public void shutdown();
 }

jetty-spdy/spdy-core/src/main/java/org/eclipse/jetty/spdy/StandardSession.java

@@ -65,6 +65,7 @@ import org.eclipse.jetty.spdy.frames.WindowUpdateFrame;
 import org.eclipse.jetty.spdy.generator.Generator;
 import org.eclipse.jetty.spdy.parser.Parser;
 import org.eclipse.jetty.util.Atomics;
+import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.component.AggregateLifeCycle;
 import org.eclipse.jetty.util.component.Dumpable;
@@ -873,6 +874,14 @@ public class StandardSession implements ISession, Parser.Listener, Callback<Stan
         flush();
     }
 
+    @Override
+    public void shutdown()
+    {
+        FrameBytes frameBytes = new CloseFrameBytes();
+        append(frameBytes);
+        flush();
+    }
+
     private void execute(Runnable task)
     {
         threadPool.execute(task);
@@ -1300,4 +1309,25 @@ public class StandardSession implements ISession, Parser.Listener, Callback<Stan
             return String.format("DATA bytes @%x available=%d consumed=%d on %s",dataInfo.hashCode(),dataInfo.available(),dataInfo.consumed(),getStream());
         }
     }
+
+    private class CloseFrameBytes extends AbstractFrameBytes<Void>
+    {
+        private CloseFrameBytes()
+        {
+            super(null, new Empty<Void>(), null);
+        }
+
+        @Override
+        public ByteBuffer getByteBuffer()
+        {
+            return BufferUtil.EMPTY_BUFFER;
+        }
+
+        @Override
+        public void complete()
+        {
+            super.complete();
+            close();
+        }
+    }
 }

jetty-spdy/spdy-core/src/test/java/org/eclipse/jetty/spdy/StandardStreamTest.java

@@ -86,16 +86,8 @@ public class StandardStreamTest
         {
             PushSynInfo pushSynInfo = (PushSynInfo)argument;
             if (pushSynInfo.getAssociatedStreamId() != associatedStreamId)
-            {
-                System.out.println("streamIds do not match!");
                 return false;
-            }
+            return pushSynInfo.isClose() == synInfo.isClose();
-            if (pushSynInfo.isClose() != synInfo.isClose())
-            {
-                System.out.println("isClose doesn't match");
-                return false;
-            }
-            return true;
         }
     }
 

jetty-spdy/spdy-core/src/test/resources/jetty-logging.properties

@@ -0,0 +1,2 @@
+org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog
+org.eclipse.jetty.spdy.LEVEL=WARN

jetty-spdy/spdy-core/src/test/resources/log4j.properties

@@ -1,14 +0,0 @@
-# LOG4J levels: OFF, FATAL, ERROR, WARN, INFO, DEBUG, ALL
-#
-log4j.rootLogger=ALL,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-#log4j.appender.CONSOLE.threshold=INFO
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-#log4j.appender.CONSOLE.layout.ConversionPattern=%d %t [%5p][%c{1}] %m%n
-log4j.appender.CONSOLE.layout.ConversionPattern=%d %t [%5p][%c{1}] %m%n
-log4j.appender.CONSOLE.target=System.err
-
-# Level tuning
-log4j.logger.org.eclipse.jetty=INFO
-#log4j.logger.org.eclipse.jetty.spdy=DEBUG

jetty-spdy/spdy-jetty-http-webapp/src/main/resources/jetty-logging.properties

@@ -0,0 +1,2 @@
+org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog
+org.eclipse.jetty.spdy.LEVEL=WARN

jetty-spdy/spdy-jetty-http-webapp/src/main/resources/log4j.properties

@@ -1,16 +0,0 @@
-# LOG4J levels: OFF, FATAL, ERROR, WARN, INFO, DEBUG, ALL
-#
-log4j.rootLogger=ALL,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-#log4j.appender.CONSOLE.threshold=INFO
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-#log4j.appender.CONSOLE.layout.ConversionPattern=%d %t [%5p][%c{1}] %m%n
-log4j.appender.CONSOLE.layout.ConversionPattern=%d %t [%5p][%c] %m%n
-log4j.appender.CONSOLE.target=System.err
-
-# Level tuning
-log4j.logger.jndi=INFO
-log4j.logger.org.mortbay.jetty=INFO
-log4j.logger.org.eclipse.jetty=INFO
-log4j.logger.org.eclipse.jetty.spdy=DEBUG

jetty-spdy/spdy-jetty-http/pom.xml

@@ -66,12 +66,6 @@
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>${slf4j-version}</version>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>

jetty-spdy/spdy-jetty-http/src/test/java/org/eclipse/jetty/spdy/http/ServerHTTPSPDYTest.java

@@ -1,18 +1,15 @@
-/*
+// ========================================================================
- * Copyright (c) 2012 the original author or authors.
+// Copyright 2012-2012 Mort Bay Consulting Pty. Ltd.
- *
+// ------------------------------------------------------------------------
- * Licensed under the Apache License, Version 2.0 (the "License");
+// All rights reserved. This program and the accompanying materials
- * you may not use this file except in compliance with the License.
+// are made available under the terms of the Eclipse Public License v1.0
- * You may obtain a copy of the License at
+// and Apache License v2.0 which accompanies this distribution.
- *
+// The Eclipse Public License is available at
- *     http://www.apache.org/licenses/LICENSE-2.0
+// http://www.eclipse.org/legal/epl-v10.html
- *
+// The Apache License v2.0 is available at
- * Unless required by applicable law or agreed to in writing, software
+// http://www.opensource.org/licenses/apache2.0.php
- * distributed under the License is distributed on an "AS IS" BASIS,
+// You may elect to redistribute this code under either of these licenses.
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// ========================================================================
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 
 package org.eclipse.jetty.spdy.http;
 

jetty-spdy/spdy-jetty-http/src/test/resources/jetty-logging.properties

@@ -0,0 +1,2 @@
+org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog
+org.eclipse.jetty.spdy.LEVEL=WARN

jetty-spdy/spdy-jetty-http/src/test/resources/log4j.properties

@@ -1,14 +0,0 @@
-# LOG4J levels: OFF, FATAL, ERROR, WARN, INFO, DEBUG, ALL
-#
-log4j.rootLogger=ALL,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-#log4j.appender.CONSOLE.threshold=INFO
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-#log4j.appender.CONSOLE.layout.ConversionPattern=%d %t [%5p][%c{1}] %m%n
-log4j.appender.CONSOLE.layout.ConversionPattern=%d %t [%5p][%c{1}] %m%n
-log4j.appender.CONSOLE.target=System.err
-
-# Level tuning
-log4j.logger.org.eclipse.jetty=INFO
-#log4j.logger.org.eclipse.jetty.spdy=DEBUG

jetty-spdy/spdy-jetty/pom.xml

@@ -72,12 +72,6 @@
 			<artifactId>hamcrest-all</artifactId>
 			<scope>test</scope>
 		</dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>${slf4j-version}</version>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 
 </project>

jetty-spdy/spdy-jetty/src/main/java/org/eclipse/jetty/spdy/SPDYAsyncConnection.java

@@ -21,7 +21,6 @@ import org.eclipse.jetty.io.AbstractAsyncConnection;
 import org.eclipse.jetty.io.AsyncEndPoint;
 import org.eclipse.jetty.io.ByteBufferPool;
 import org.eclipse.jetty.io.RuntimeIOException;
-import org.eclipse.jetty.spdy.api.Session;
 import org.eclipse.jetty.spdy.parser.Parser;
 import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.log.Log;
@@ -32,7 +31,7 @@ public class SPDYAsyncConnection extends AbstractAsyncConnection implements Cont
     private static final Logger logger = Log.getLogger(SPDYAsyncConnection.class);
     private final ByteBufferPool bufferPool;
     private final Parser parser;
-    private volatile Session session;
+    private volatile ISession session;
     private volatile boolean idle = false;
 
     public SPDYAsyncConnection(AsyncEndPoint endPoint, ByteBufferPool bufferPool, Parser parser, Executor executor)
@@ -65,7 +64,7 @@ public class SPDYAsyncConnection extends AbstractAsyncConnection implements Cont
             }
             else if (filled < 0)
             {
-                close(false);
+                shutdown(session);
                 return -1;
             }
             else
@@ -79,6 +78,8 @@ public class SPDYAsyncConnection extends AbstractAsyncConnection implements Cont
     {
         try
         {
+            if (endPoint.isInputShutdown())
+                return -1;
             return endPoint.fill(buffer);
         }
         catch (IOException x)
@@ -92,8 +93,9 @@ public class SPDYAsyncConnection extends AbstractAsyncConnection implements Cont
     public int write(ByteBuffer buffer, final Callback<StandardSession.FrameBytes> callback, StandardSession.FrameBytes context)
     {
         AsyncEndPoint endPoint = getEndPoint();
+        int remaining = buffer.remaining();
         endPoint.write(context, callback, buffer);
-        return -1; //TODO: void or have endPoint.write return int
+        return remaining - buffer.remaining();
     }
 
     @Override
@@ -121,16 +123,28 @@ public class SPDYAsyncConnection extends AbstractAsyncConnection implements Cont
     protected boolean onReadTimeout()
     {
         if (idle)
-            session.goAway();
+            goAway(session);
-        return idle;
+        return false;
     }
 
-    protected Session getSession()
+    protected void goAway(ISession session)
+    {
+        if (session != null)
+            session.goAway();
+    }
+
+    private void shutdown(ISession session)
+    {
+        if (session != null)
+            session.shutdown();
+    }
+
+    protected ISession getSession()
     {
         return session;
     }
 
-    protected void setSession(Session session)
+    protected void setSession(ISession session)
     {
         this.session = session;
     }

jetty-spdy/spdy-jetty/src/test/java/org/eclipse/jetty/spdy/ClosedStreamTest.java

@@ -141,7 +141,6 @@ public class ClosedStreamTest extends AbstractTest
             @Override
             public void onReply(Stream stream, ReplyInfo replyInfo)
             {
-                System.out.println("ONREPLY CLIENT CALLED");
                 replyReceivedLatch.countDown();
             }
 

jetty-spdy/spdy-jetty/src/test/resources/jetty-logging.properties

@@ -0,0 +1,2 @@
+org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog
+org.eclipse.jetty.spdy.LEVEL=WARN

jetty-spdy/spdy-jetty/src/test/resources/log4j.properties

@@ -1,16 +0,0 @@
-# LOG4J levels: OFF, FATAL, ERROR, WARN, INFO, DEBUG, ALL
-#
-log4j.rootLogger=ALL,CONSOLE
-
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-#log4j.appender.CONSOLE.threshold=INFO
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-#log4j.appender.CONSOLE.layout.ConversionPattern=%d %t [%5p][%c{1}] %m%n
-log4j.appender.CONSOLE.layout.ConversionPattern=%d %t [%5p][%c{1}] %m%n
-log4j.appender.CONSOLE.target=System.err
-
-# Level tuning
-log4j.logger.org.eclipse.jetty=INFO
-#log4j.logger.org.eclipse.jetty.io=DEBUG
-log4j.logger.org.eclipse.jetty.spdy=DEBUG
-# thomas

jetty-util/src/main/java/org/eclipse/jetty/util/log/StdErrLog.java

@@ -460,6 +460,7 @@ public class StdErrLog extends AbstractLogger
             buffer.append(_abbrevname);
         }
         buffer.append(':');
+        buffer.append(Thread.currentThread().getId()).append(": ");
         if (_source)
         {
             Throwable source = new Throwable();

jetty-websocket/websocket-client/src/main/java/org/eclipse/jetty/websocket/client/io/WebSocketClientAsyncConnection.java

@@ -0,0 +1,22 @@
+package org.eclipse.jetty.websocket.client.io;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
+
+import org.eclipse.jetty.io.AsyncEndPoint;
+import org.eclipse.jetty.io.ByteBufferPool;
+import org.eclipse.jetty.websocket.api.WebSocketPolicy;
+import org.eclipse.jetty.websocket.client.WebSocketClientFactory;
+import org.eclipse.jetty.websocket.io.WebSocketAsyncConnection;
+
+public class WebSocketClientAsyncConnection extends WebSocketAsyncConnection
+{
+    private final WebSocketClientFactory factory;
+
+    public WebSocketClientAsyncConnection(AsyncEndPoint endp, Executor executor, ScheduledExecutorService scheduler, WebSocketPolicy policy,
+            ByteBufferPool bufferPool, WebSocketClientFactory factory)
+    {
+        super(endp,executor,scheduler,policy,bufferPool);
+        this.factory = factory;
+    }
+}

jetty-websocket/websocket-client/src/main/java/org/eclipse/jetty/websocket/client/io/WebSocketClientSelectorManager.java

@@ -21,6 +21,7 @@ import java.nio.channels.SocketChannel;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.atomic.AtomicReference;
+
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLException;
 
@@ -76,7 +77,7 @@ public class WebSocketClientSelectorManager extends SelectorManager
         ByteBufferPool bufferPool = factory.getBufferPool();
         ScheduledExecutorService scheduler = factory.getScheduler();
 
-        WebSocketAsyncConnection connection = new WebSocketAsyncConnection(endPoint,executor,scheduler,policy,bufferPool);
+        WebSocketAsyncConnection connection = new WebSocketClientAsyncConnection(endPoint,executor,scheduler,policy,bufferPool,factory);
         endPoint.setAsyncConnection(connection);
         connection.getParser().setIncomingFramesHandler(websocket);
 

jetty-websocket/websocket-client/src/test/java/org/eclipse/jetty/websocket/client/examples/TestClient.java

@@ -67,7 +67,7 @@ public class TestClient
             }
         }
 
-        public void send(OpCode op, byte[] data, int maxFragmentLength)
+        public void send(byte op, byte[] data, int maxFragmentLength)
         {
             _starts.add(System.nanoTime());
 
@@ -215,7 +215,7 @@ public class TestClient
             {
                 long next = System.currentTimeMillis() + delay;
 
-                OpCode op = OpCode.TEXT;
+                byte op = OpCode.TEXT;
                 if (binary)
                 {
                     op = OpCode.BINARY;
@@ -225,7 +225,7 @@ public class TestClient
 
                 switch (op)
                 {
-                    case TEXT:
+                    case OpCode.TEXT:
                     {
                         StringBuilder b = new StringBuilder();
                         while (b.length() < size)
@@ -235,7 +235,7 @@ public class TestClient
                         data = b.toString().getBytes(StringUtil.__UTF8_CHARSET);
                         break;
                     }
-                    case BINARY:
+                    case OpCode.BINARY:
                     {
                         data = new byte[size];
                         __random.nextBytes(data);
@@ -328,7 +328,7 @@ public class TestClient
         client.connect(wsUri,socket).get(10,TimeUnit.SECONDS);
     }
 
-    private void send(OpCode op, byte[] data, int fragment)
+    private void send(byte op, byte[] data, int fragment)
     {
         socket.send(op,data,fragment);
     }

jetty-websocket/websocket-core/src/main/java/org/eclipse/jetty/websocket/driver/WebSocketEventDriver.java

@@ -59,7 +59,7 @@ public class WebSocketEventDriver implements IncomingFrames
     private final WebSocketPolicy policy;
     private final EventMethods events;
     private final ByteBufferPool bufferPool;
-    private WebSocketSession connection;
+    private WebSocketSession session;
     private ByteBuffer activeMessage;
     private StreamAppender activeStream;
 
@@ -150,7 +150,7 @@ public class WebSocketEventDriver implements IncomingFrames
 
         if (events.onException != null)
         {
-            events.onException.call(websocket,connection,e);
+            events.onException.call(websocket,session,e);
         }
     }
 
@@ -171,7 +171,7 @@ public class WebSocketEventDriver implements IncomingFrames
         // Generic Read-Only Frame version
         if ((frame instanceof Frame) && (events.onFrame != null))
         {
-            events.onFrame.call(websocket,connection,frame);
+            events.onFrame.call(websocket,session,frame);
             // DO NOT return; - as this is just a read-only notification.
         }
 
@@ -179,16 +179,16 @@ public class WebSocketEventDriver implements IncomingFrames
         {
             switch (frame.getOpCode())
             {
-                case CLOSE:
+                case OpCode.CLOSE:
                 {
                     CloseInfo close = new CloseInfo(frame);
                     if (events.onClose != null)
                     {
-                        events.onClose.call(websocket,connection,close.getStatusCode(),close.getReason());
+                        events.onClose.call(websocket,session,close.getStatusCode(),close.getReason());
                     }
                     throw new CloseException(close.getStatusCode(),close.getReason());
                 }
-                case PING:
+                case OpCode.PING:
                 {
                     WebSocketFrame pong = new WebSocketFrame(OpCode.PONG);
                     if (frame.getPayloadLength() > 0)
@@ -200,10 +200,10 @@ public class WebSocketEventDriver implements IncomingFrames
                         BufferUtil.flipToFlush(pongBuf,0);
                         pong.setPayload(pongBuf);
                     }
-                    connection.output("pong",new FutureCallback<String>(),pong);
+                    session.output("pong",new FutureCallback<String>(),pong);
                     break;
                 }
-                case BINARY:
+                case OpCode.BINARY:
                 {
                     if (events.onBinary == null)
                     {
@@ -230,7 +230,7 @@ public class WebSocketEventDriver implements IncomingFrames
 
                         if (needsNotification)
                         {
-                            events.onBinary.call(websocket,connection,activeStream);
+                            events.onBinary.call(websocket,session,activeStream);
                         }
 
                         if (frame.isFin())
@@ -261,7 +261,7 @@ public class WebSocketEventDriver implements IncomingFrames
                             {
                                 BufferUtil.flipToFlush(activeMessage,0);
                                 byte buf[] = BufferUtil.toArray(activeMessage);
-                                events.onBinary.call(websocket,connection,buf,0,buf.length);
+                                events.onBinary.call(websocket,session,buf,0,buf.length);
                             }
                             finally
                             {
@@ -273,7 +273,7 @@ public class WebSocketEventDriver implements IncomingFrames
                     }
                     return;
                 }
-                case TEXT:
+                case OpCode.TEXT:
                 {
                     if (events.onText == null)
                     {
@@ -300,7 +300,7 @@ public class WebSocketEventDriver implements IncomingFrames
 
                         if (needsNotification)
                         {
-                            events.onText.call(websocket,connection,activeStream);
+                            events.onText.call(websocket,session,activeStream);
                         }
 
                         if (frame.isFin())
@@ -335,7 +335,7 @@ public class WebSocketEventDriver implements IncomingFrames
                                 // TODO: FIX EVIL COPY
                                 utf.append(data,0,data.length);
 
-                                events.onText.call(websocket,connection,utf.toString());
+                                events.onText.call(websocket,session,utf.toString());
                             }
                             finally
                             {
@@ -371,7 +371,7 @@ public class WebSocketEventDriver implements IncomingFrames
         {
             LOG.debug("{}.onConnect()",websocket.getClass().getSimpleName());
         }
-        events.onConnect.call(websocket,connection);
+        events.onConnect.call(websocket,session);
     }
 
     /**
@@ -380,9 +380,9 @@ public class WebSocketEventDriver implements IncomingFrames
      * @param conn
      *            the connection
      */
-    public void setConnection(WebSocketSession conn)
+    public void setSession(WebSocketSession conn)
     {
-        this.connection = conn;
+        this.session = conn;
     }
 
     private void terminateConnection(int statusCode, String rawreason)
@@ -399,7 +399,7 @@ public class WebSocketEventDriver implements IncomingFrames
                 }
             }
             LOG.debug("terminateConnection({},{})",statusCode,rawreason);
-            connection.close(statusCode,reason);
+            session.close(statusCode,reason);
         }
         catch (IOException e)
         {
@@ -407,6 +407,12 @@ public class WebSocketEventDriver implements IncomingFrames
         }
     }
 
+    @Override
+    public String toString()
+    {
+        return websocket.getClass().getName();
+    }
+
     private void unhandled(Throwable t)
     {
         socketLog.warn("Unhandled Error (closing connection)",t);

jetty-websocket/websocket-core/src/main/java/org/eclipse/jetty/websocket/extensions/deflate/DeflateFrameExtension.java

@@ -114,7 +114,7 @@ public class DeflateFrameExtension extends Extension
     @Override
     public void incoming(WebSocketFrame frame)
     {
-        if (frame.getOpCode().isControlFrame() || !frame.isRsv1())
+        if (frame.isControlFrame() || !frame.isRsv1())
         {
             // Cannot modify incoming control frames or ones with RSV1 set.
             super.incoming(frame);
@@ -183,7 +183,7 @@ public class DeflateFrameExtension extends Extension
     @Override
     public <C> void output(C context, Callback<C> callback, WebSocketFrame frame) throws IOException
     {
-        if (frame.getOpCode().isControlFrame())
+        if (frame.isControlFrame())
         {
             // skip, cannot compress control frames.
             nextOutput(context,callback,frame);

jetty-websocket/websocket-core/src/main/java/org/eclipse/jetty/websocket/extensions/fragment/FragmentExtension.java

@@ -31,7 +31,7 @@ public class FragmentExtension extends Extension
     @Override
     public <C> void output(C context, Callback<C> callback, WebSocketFrame frame) throws IOException
     {
-        if (frame.getOpCode().isControlFrame())
+        if (frame.isControlFrame())
         {
             // Cannot fragment Control Frames
             nextOutput(context,callback,frame);
@@ -40,7 +40,7 @@ public class FragmentExtension extends Extension
 
         int length = frame.getPayloadLength();
 
-        OpCode opcode = frame.getOpCode(); // original opcode
+        byte opcode = frame.getOpCode(); // original opcode
         ByteBuffer payload = frame.getPayload().slice();
         int originalLimit = payload.limit();
         int currentPosition = payload.position();

jetty-websocket/websocket-core/src/main/java/org/eclipse/jetty/websocket/io/DataFrameBytes.java

@@ -25,7 +25,6 @@ import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
 public class DataFrameBytes<C> extends FrameBytes<C>
 {
     private static final Logger LOG = Log.getLogger(DataFrameBytes.class);
-    private int size;
     private ByteBuffer buffer;
 
     public DataFrameBytes(WebSocketAsyncConnection connection, Callback<C> callback, C context, WebSocketFrame frame)

jetty-websocket/websocket-core/src/main/java/org/eclipse/jetty/websocket/io/FrameBytes.java

@@ -84,7 +84,7 @@ public abstract class FrameBytes<C> implements Callback<C>, Runnable
         }
         catch (IOException e)
         {
-            WebSocketAsyncConnection.LOG.ignore(e);
+            LOG.ignore(e);
         }
         failed(context, new InterruptedByTimeoutException());
     }

jetty-websocket/websocket-core/src/main/java/org/eclipse/jetty/websocket/io/WebSocketAsyncConnection.java

@@ -41,26 +41,17 @@ import org.eclipse.jetty.websocket.api.WebSocketPolicy;
 import org.eclipse.jetty.websocket.protocol.CloseInfo;
 import org.eclipse.jetty.websocket.protocol.ExtensionConfig;
 import org.eclipse.jetty.websocket.protocol.Generator;
+import org.eclipse.jetty.websocket.protocol.OpCode;
 import org.eclipse.jetty.websocket.protocol.Parser;
 import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
 
 /**
  * Provides the implementation of {@link WebSocketConnection} within the framework of the new {@link AsyncConnection} framework of jetty-io
  */
-public class WebSocketAsyncConnection extends AbstractAsyncConnection implements RawConnection, OutgoingFrames
+public abstract class WebSocketAsyncConnection extends AbstractAsyncConnection implements RawConnection, OutgoingFrames
 {
-    static final Logger LOG = Log.getLogger(WebSocketAsyncConnection.class);
+    private static final Logger LOG = Log.getLogger(WebSocketAsyncConnection.class);
-    private static final ThreadLocal<WebSocketAsyncConnection> CURRENT_CONNECTION = new ThreadLocal<WebSocketAsyncConnection>();
+    private static final Logger LOG_FRAMES = Log.getLogger("org.eclipse.jetty.websocket.io.Frames");
-
-    public static WebSocketAsyncConnection getCurrentConnection()
-    {
-        return CURRENT_CONNECTION.get();
-    }
-
-    protected static void setCurrentConnection(WebSocketAsyncConnection connection)
-    {
-        CURRENT_CONNECTION.set(connection);
-    }
 
     private final ByteBufferPool bufferPool;
     private final ScheduledExecutorService scheduler;
@@ -68,6 +59,7 @@ public class WebSocketAsyncConnection extends AbstractAsyncConnection implements
     private final Parser parser;
     private final WebSocketPolicy policy;
     private final FrameQueue queue;
+    private WebSocketSession session;
     private List<ExtensionConfig> extensions;
     private boolean flushing;
     private AtomicLong writes;
@@ -145,6 +137,10 @@ public class WebSocketAsyncConnection extends AbstractAsyncConnection implements
             }
 
             flushing = true;
+            if (LOG.isDebugEnabled())
+            {
+                LOG.debug("Flushing {}, {} frame(s) in queue",frameBytes,queue.size());
+            }
         }
         write(buffer,this,frameBytes);
     }
@@ -202,43 +198,35 @@ public class WebSocketAsyncConnection extends AbstractAsyncConnection implements
         return scheduler;
     }
 
+    public WebSocketSession getSession()
+    {
+        return session;
+    }
+
     @Override
     public boolean isOpen()
     {
         return getEndPoint().isOpen();
     }
 
-    @Override
-    public void onClose()
-    {
-        LOG.debug("onClose()");
-        super.onClose();
-    }
-
     @Override
     public void onFillable()
     {
-        setCurrentConnection(this);
         ByteBuffer buffer = bufferPool.acquire(policy.getBufferSize(),false);
         BufferUtil.clear(buffer);
+        boolean readMore = false;
         try
         {
-            read(buffer);
+            readMore = (read(buffer) != -1);
         }
         finally
         {
-            fillInterested();
-            setCurrentConnection(null);
             bufferPool.release(buffer);
         }
-    }
+        if (readMore)
-
-    @Override
-    public void onOpen()
         {
-        super.onOpen();
+            fillInterested();
-        // TODO: websocket.setConnection(this);
+        }
-        // TODO: websocket.onConnect();
     }
 
     /**
@@ -247,52 +235,76 @@ public class WebSocketAsyncConnection extends AbstractAsyncConnection implements
     @Override
     public <C> void output(C context, Callback<C> callback, WebSocketFrame frame)
     {
+        if (LOG.isDebugEnabled())
+        {
+            LOG.debug("output({}, {}, {})",context,callback,frame);
+        }
+
         synchronized (queue)
         {
-            if (frame.getOpCode().isControlFrame())
+            FrameBytes<C> bytes = null;
+
+            if (frame.isControlFrame())
             {
-                ControlFrameBytes<C> bytes = new ControlFrameBytes<C>(this,callback,context,frame);
+                bytes = new ControlFrameBytes<C>(this,callback,context,frame);
+            }
+            else
+            {
+                bytes = new DataFrameBytes<C>(this,callback,context,frame);
+            }
+
             scheduleTimeout(bytes);
+            if (frame.getOpCode() == OpCode.PING)
+            {
                 queue.prepend(bytes);
             }
             else
             {
-                DataFrameBytes<C> bytes = new DataFrameBytes<C>(this,callback,context,frame);
-                scheduleTimeout(bytes);
                 queue.append(bytes);
             }
         }
         flush();
     }
 
-    private void read(ByteBuffer buffer)
+    private int read(ByteBuffer buffer)
     {
+        AsyncEndPoint endPoint = getEndPoint();
         try
         {
             while (true)
             {
-                int filled = getEndPoint().fill(buffer);
+                int filled = endPoint.fill(buffer);
                 if (filled == 0)
                 {
-                    break;
+                    return 0;
                 }
-
+                else if (filled < 0)
-                if (LOG.isDebugEnabled() && (filled > 0))
+                {
+                    LOG.debug("read - EOF Reached");
+                    disconnect(false);
+                    return -1;
+                }
+                else
+                {
+                    if (LOG.isDebugEnabled())
                     {
                         LOG.debug("Filled {} bytes - {}",filled,BufferUtil.toDetailString(buffer));
                     }
                     parser.parse(buffer);
                 }
             }
+        }
         catch (IOException e)
         {
             LOG.warn(e);
             terminateConnection(StatusCode.PROTOCOL,e.getMessage());
+            return -1;
         }
         catch (CloseException e)
         {
             LOG.warn(e);
             terminateConnection(e.getStatusCode(),e.getMessage());
+            return -1;
         }
     }
 
@@ -317,6 +329,11 @@ public class WebSocketAsyncConnection extends AbstractAsyncConnection implements
         this.extensions = extensions;
     }
 
+    public void setSession(WebSocketSession session)
+    {
+        this.session = session;
+    }
+
     /**
      * For terminating connections forcefully.
      *
@@ -345,9 +362,9 @@ public class WebSocketAsyncConnection extends AbstractAsyncConnection implements
     {
         AsyncEndPoint endpoint = getEndPoint();
 
-        if (LOG.isDebugEnabled())
+        if (LOG_FRAMES.isDebugEnabled())
         {
-            LOG.debug("Writing {} frame bytes of {}",buffer.remaining(),frameBytes);
+            LOG_FRAMES.debug("{} Writing {} frame bytes of {}",policy.getBehavior(),buffer.remaining(),frameBytes);
         }
         try
         {

jetty-websocket/websocket-core/src/main/java/org/eclipse/jetty/websocket/io/WebSocketSession.java

@@ -93,6 +93,13 @@ public class WebSocketSession implements WebSocketConnection, IncomingFrames, Ou
         return connection.isOpen();
     }
 
+    public void onConnect()
+    {
+        LOG.debug("onConnect()");
+        websocket.setSession(this);
+        websocket.onConnect();
+    }
+
     @Override
     public <C> void output(C context, Callback<C> callback, WebSocketFrame frame) throws IOException
     {
@@ -120,6 +127,22 @@ public class WebSocketSession implements WebSocketConnection, IncomingFrames, Ou
         this.outgoing = outgoing;
     }
 
+    @Override
+    public String toString()
+    {
+        StringBuilder builder = new StringBuilder();
+        builder.append("WebSocketSession[websocket=");
+        builder.append(websocket);
+        builder.append(",connection=");
+        builder.append(connection);
+        builder.append(",subprotocol=");
+        builder.append(subprotocol);
+        builder.append(",outgoing=");
+        builder.append(outgoing);
+        builder.append("]");
+        return builder.toString();
+    }
+
     /**
      * {@inheritDoc}
      */

jetty-websocket/websocket-core/src/main/java/org/eclipse/jetty/websocket/protocol/Frame.java

@@ -26,7 +26,7 @@ public interface Frame
 {
     public byte[] getMask();
 
-    public OpCode getOpCode();
+    public byte getOpCode();
 
     public ByteBuffer getPayload();
 

jetty-websocket/websocket-core/src/main/java/org/eclipse/jetty/websocket/protocol/Generator.java

@@ -125,7 +125,7 @@ public class Generator
             throw new ProtocolException("RSV3 not allowed to be set");
         }
 
-        if (frame.getOpCode().isControlFrame())
+        if (frame.isControlFrame())
         {
             /*
              * RFC 6455 Section 5.5
@@ -229,12 +229,12 @@ public class Generator
                 b |= 0x10;
             }
 
-            byte opcode = frame.getOpCode().getCode();
+            byte opcode = frame.getOpCode();
 
             if (frame.isContinuation())
             {
                 // Continuations are not the same OPCODE
-                opcode = OpCode.CONTINUATION.getCode();
+                opcode = OpCode.CONTINUATION;
             }
 
             b |= opcode & 0x0F;

jetty-websocket/websocket-core/src/main/java/org/eclipse/jetty/websocket/protocol/OpCode.java

@@ -15,89 +15,92 @@
 //========================================================================
 package org.eclipse.jetty.websocket.protocol;
 
-import java.util.HashMap;
-import java.util.Map;
 
-public enum OpCode
+public final class OpCode
 {
     /**
      * OpCode for a Continuation Frame
      * 
      * @see <a href="https://tools.ietf.org/html/rfc6455#section-11.8">RFC 6455, Section 11.8 (WebSocket Opcode Registry</a>
      */
-    CONTINUATION((byte)0x00),
+    public static final byte CONTINUATION = (byte)0x00;
 
     /**
      * OpCode for a Text Frame
      * 
      * @see <a href="https://tools.ietf.org/html/rfc6455#section-11.8">RFC 6455, Section 11.8 (WebSocket Opcode Registry</a>
      */
-    TEXT((byte)0x01),
+    public static final byte TEXT = (byte)0x01;
 
     /**
      * OpCode for a Binary Frame
      * 
      * @see <a href="https://tools.ietf.org/html/rfc6455#section-11.8">RFC 6455, Section 11.8 (WebSocket Opcode Registry</a>
      */
-    BINARY((byte)0x02),
+    public static final byte BINARY = (byte)0x02;
 
     /**
      * OpCode for a Close Frame
      * 
      * @see <a href="https://tools.ietf.org/html/rfc6455#section-11.8">RFC 6455, Section 11.8 (WebSocket Opcode Registry</a>
      */
-    CLOSE((byte)0x08),
+    public static final byte CLOSE = (byte)0x08;
 
     /**
      * OpCode for a Ping Frame
      * 
      * @see <a href="https://tools.ietf.org/html/rfc6455#section-11.8">RFC 6455, Section 11.8 (WebSocket Opcode Registry</a>
      */
-    PING((byte)0x09),
+    public static final byte PING = (byte)0x09;
 
     /**
      * OpCode for a Pong Frame
      * 
      * @see <a href="https://tools.ietf.org/html/rfc6455#section-11.8">RFC 6455, Section 11.8 (WebSocket Opcode Registry</a>
      */
-    PONG((byte)0x0A);
+    public static final byte PONG = (byte)0x0A;
 
-    private static class Codes
+    public static boolean isControlFrame(byte opcode)
     {
-        private static final Map<Byte, OpCode> codes = new HashMap<>();
+        return (opcode >= CLOSE);
+    }
+
+    public static boolean isDataFrame(byte opcode)
+    {
+        return (opcode == TEXT) || (opcode == BINARY);
     }
 
     /**
-     * Get OpCode from specified value.
+     * Test for known opcodes (per the RFC spec)
      * 
      * @param opcode
-     * @return
+     *            the opcode to test
+     * @return true if known. false if unknown, undefined, or reserved
      */
-    public static OpCode from(byte opcode)
+    public static boolean isKnown(byte opcode)
     {
-        return Codes.codes.get(opcode);
+        return (opcode == CONTINUATION) || (opcode == TEXT) || (opcode == BINARY) || (opcode == CLOSE) || (opcode == PING) || (opcode == PONG);
     }
 
-    private byte opcode;
+    public static String name(byte opcode)
-
-    private OpCode(byte opcode)
     {
-        this.opcode = opcode;
+        switch (opcode)
-        Codes.codes.put(opcode,this);
-    }
-
-    public byte getCode()
         {
-        return this.opcode;
+            case -1:
-    }
+                return "NO-OP";
-
+            case CONTINUATION:
-    public boolean isControlFrame()
+                return "CONTINUATION";
-    {
+            case TEXT:
-        return (opcode >= CLOSE.opcode);
+                return "TEXT";
-    }
+            case BINARY: return "BINARY";
-
+            case CLOSE:
-    public boolean isDataFrame()
+                return "CLOSE";
-    {
+            case PING:
-        return (this == TEXT) || (this == BINARY);
+                return "PING";
+            case PONG:
+                return "PONG";
+            default:
+                return "NON-SPEC[" + opcode + "]";
+        }
     }
 }

jetty-websocket/websocket-core/src/main/java/org/eclipse/jetty/websocket/protocol/Parser.java

@@ -42,26 +42,31 @@ public class Parser
         PAYLOAD
     }
 
+    private static final Logger LOG_FRAMES = Log.getLogger("org.eclipse.jetty.websocket.io.Frames");
+
     // State specific
     private State state = State.START;
     private int cursor = 0;
     // Frame
     private WebSocketFrame frame;
-    private OpCode lastDataOpcode;
+    private byte lastDataOpcode;
     // payload specific
     private ByteBuffer payload;
     private int payloadLength;
 
+    /** Is there an extension using RSV1 */
+    private boolean rsv1InUse = false;
+    /** Is there an extension using RSV2 */
+    private boolean rsv2InUse = false;
+    /** Is there an extension using RSV3 */
+    private boolean rsv3InUse = false;
+
     private static final Logger LOG = Log.getLogger(Parser.class);
     private IncomingFrames incomingFramesHandler;
     private WebSocketPolicy policy;
 
     public Parser(WebSocketPolicy wspolicy)
     {
-        /*
-         * TODO: Investigate addition of decompression factory similar to SPDY work in situation of negotiated deflate extension?
-         */
-
         this.policy = wspolicy;
     }
 
@@ -78,14 +83,14 @@ public class Parser
 
         switch (frame.getOpCode())
         {
-            case CLOSE:
+            case OpCode.CLOSE:
                 if (len == 1)
                 {
                     throw new ProtocolException("Invalid close frame payload length, [" + payloadLength + "]");
                 }
                 // fall thru
-            case PING:
+            case OpCode.PING:
-            case PONG:
+            case OpCode.PONG:
                 if (len > WebSocketFrame.MAX_CONTROL_PAYLOAD)
                 {
                     throw new ProtocolException("Invalid control frame payload length, [" + payloadLength + "] cannot exceed ["
@@ -105,11 +110,30 @@ public class Parser
         return policy;
     }
 
+    public boolean isRsv1InUse()
+    {
+        return rsv1InUse;
+    }
+
+    public boolean isRsv2InUse()
+    {
+        return rsv2InUse;
+    }
+
+    public boolean isRsv3InUse()
+    {
+        return rsv3InUse;
+    }
+
     protected void notifyFrame(final WebSocketFrame f)
     {
+        if (LOG_FRAMES.isDebugEnabled())
+        {
+            LOG_FRAMES.debug("{} Read Frame: {}",policy.getBehavior(),f);
+        }
         if (LOG.isDebugEnabled())
         {
-            LOG.debug("{} Notify Frame: {} to {}",policy.getBehavior(),f,incomingFramesHandler);
+            LOG.debug("{} Notify {}",policy.getBehavior(),incomingFramesHandler);
         }
         if (incomingFramesHandler == null)
         {
@@ -213,18 +237,39 @@ public class Parser
                     boolean rsv2 = ((b & 0x20) != 0);
                     boolean rsv3 = ((b & 0x10) != 0);
                     byte opc = (byte)(b & 0x0F);
-                    OpCode opcode = OpCode.from(opc);
+                    byte opcode = opc;
 
-                    if (opcode == null)
+                    if (!OpCode.isKnown(opcode))
                     {
-                        throw new WebSocketException("Unknown opcode: " + opc);
+                        throw new ProtocolException("Unknown opcode: " + opc);
                     }
 
-                    LOG.debug("OpCode {}, fin={}",opcode.name(),fin);
+                    LOG.debug("OpCode {}, fin={}",OpCode.name(opcode),fin);
 
-                    if (opcode.isControlFrame() && !fin)
+                    /*
+                     * RFC 6455 Section 5.2
+                     * 
+                     * MUST be 0 unless an extension is negotiated that defines meanings for non-zero values. If a nonzero value is received and none of the
+                     * negotiated extensions defines the meaning of such a nonzero value, the receiving endpoint MUST _Fail the WebSocket Connection_.
+                     */
+                    if (!rsv1InUse && rsv1)
                     {
-                        throw new ProtocolException("Fragmented Control Frame [" + opcode.name() + "]");
+                        throw new ProtocolException("RSV1 not allowed to be set");
+                    }
+
+                    if (!rsv2InUse && rsv2)
+                    {
+                        throw new ProtocolException("RSV2 not allowed to be set");
+                    }
+
+                    if (!rsv3InUse && rsv3)
+                    {
+                        throw new ProtocolException("RSV3 not allowed to be set");
+                    }
+
+                    if (OpCode.isControlFrame(opcode) && !fin)
+                    {
+                        throw new ProtocolException("Fragmented Control Frame [" + OpCode.name(opcode) + "]");
                     }
 
                     if (opcode == OpCode.CONTINUATION)
@@ -245,7 +290,7 @@ public class Parser
                     frame.setRsv3(rsv3);
                     frame.setOpCode(opcode);
 
-                    if (opcode.isDataFrame())
+                    if (frame.isDataFrame())
                     {
                         lastDataOpcode = opcode;
                     }
@@ -436,6 +481,21 @@ public class Parser
         this.incomingFramesHandler = incoming;
     }
 
+    public void setRsv1InUse(boolean rsv1InUse)
+    {
+        this.rsv1InUse = rsv1InUse;
+    }
+
+    public void setRsv2InUse(boolean rsv2InUse)
+    {
+        this.rsv2InUse = rsv2InUse;
+    }
+
+    public void setRsv3InUse(boolean rsv3InUse)
+    {
+        this.rsv3InUse = rsv3InUse;
+    }
+
     @Override
     public String toString()
     {

jetty-websocket/websocket-core/src/main/java/org/eclipse/jetty/websocket/protocol/WebSocketFrame.java

@@ -84,7 +84,7 @@ public class WebSocketFrame implements Frame
     private boolean rsv1 = false;
     private boolean rsv2 = false;
     private boolean rsv3 = false;
-    private OpCode opcode = null;
+    private byte opcode = -1;
     private boolean masked = false;
     private byte mask[];
     /**
@@ -111,7 +111,7 @@ public class WebSocketFrame implements Frame
     /**
      * Construct form opcode
      */
-    public WebSocketFrame(OpCode opcode)
+    public WebSocketFrame(byte opcode)
     {
         reset();
         this.opcode = opcode;
@@ -151,7 +151,7 @@ public class WebSocketFrame implements Frame
 
     public void assertValid()
     {
-        if (opcode.isControlFrame())
+        if (OpCode.isControlFrame(opcode))
         {
             if (getPayloadLength() > WebSocketFrame.MAX_CONTROL_PAYLOAD)
             {
@@ -211,7 +211,7 @@ public class WebSocketFrame implements Frame
     }
 
     @Override
-    public final OpCode getOpCode()
+    public final byte getOpCode()
     {
         return opcode;
     }
@@ -275,6 +275,16 @@ public class WebSocketFrame implements Frame
         return continuation;
     }
 
+    public boolean isControlFrame()
+    {
+        return OpCode.isControlFrame(opcode);
+    }
+
+    public boolean isDataFrame()
+    {
+        return OpCode.isDataFrame(opcode);
+    }
+
     @Override
     public boolean isFin()
     {
@@ -348,7 +358,7 @@ public class WebSocketFrame implements Frame
         rsv1 = false;
         rsv2 = false;
         rsv3 = false;
-        opcode = null;
+        opcode = -1;
         masked = false;
         data = null;
         payloadLength = 0;
@@ -388,9 +398,9 @@ public class WebSocketFrame implements Frame
         return this;
     }
 
-    public WebSocketFrame setOpCode(OpCode opCode)
+    public WebSocketFrame setOpCode(byte op)
     {
-        this.opcode = opCode;
+        this.opcode = op;
         return this;
     }
 
@@ -408,7 +418,7 @@ public class WebSocketFrame implements Frame
             return this;
         }
 
-        if (opcode.isControlFrame())
+        if (OpCode.isControlFrame(opcode))
         {
             if (buf.length > WebSocketFrame.MAX_CONTROL_PAYLOAD)
             {
@@ -436,7 +446,7 @@ public class WebSocketFrame implements Frame
             return this;
         }
 
-        if (opcode.isControlFrame())
+        if (OpCode.isControlFrame(opcode))
         {
             if (len > WebSocketFrame.MAX_CONTROL_PAYLOAD)
             {
@@ -468,7 +478,7 @@ public class WebSocketFrame implements Frame
             return this;
         }
 
-        if (opcode.isControlFrame())
+        if (OpCode.isControlFrame(opcode))
         {
             if (buf.remaining() > WebSocketFrame.MAX_CONTROL_PAYLOAD)
             {
@@ -510,14 +520,7 @@ public class WebSocketFrame implements Frame
     public String toString()
     {
         StringBuilder b = new StringBuilder();
-        if (opcode != null)
+        b.append(OpCode.name(opcode));
-        {
-            b.append(opcode.name());
-        }
-        else
-        {
-            b.append("NO-OP");
-        }
         b.append('[');
         b.append("len=").append(payloadLength);
         b.append(",fin=").append(fin);

jetty-websocket/websocket-core/src/test/java/org/eclipse/jetty/websocket/driver/WebSocketEventDriverTest.java

@@ -58,7 +58,7 @@ public class WebSocketEventDriverTest
         WebSocketEventDriver driver = newDriver(socket);
 
         LocalWebSocketSession conn = new LocalWebSocketSession(testname);
-        driver.setConnection(conn);
+        driver.setSession(conn);
         driver.onConnect();
         driver.incoming(new CloseInfo(StatusCode.NORMAL).asFrame());
 
@@ -74,7 +74,7 @@ public class WebSocketEventDriverTest
         WebSocketEventDriver driver = newDriver(socket);
 
         LocalWebSocketSession conn = new LocalWebSocketSession(testname);
-        driver.setConnection(conn);
+        driver.setSession(conn);
         driver.onConnect();
         driver.incoming(makeBinaryFrame("Hello World",true));
         driver.incoming(new CloseInfo(StatusCode.NORMAL).asFrame());
@@ -92,7 +92,7 @@ public class WebSocketEventDriverTest
         WebSocketEventDriver driver = newDriver(socket);
 
         LocalWebSocketSession conn = new LocalWebSocketSession(testname);
-        driver.setConnection(conn);
+        driver.setSession(conn);
         driver.onConnect();
         driver.incoming(new WebSocketFrame(OpCode.PING).setPayload("PING"));
         driver.incoming(WebSocketFrame.text("Text Me"));
@@ -115,7 +115,7 @@ public class WebSocketEventDriverTest
         WebSocketEventDriver driver = newDriver(socket);
 
         LocalWebSocketSession conn = new LocalWebSocketSession(testname);
-        driver.setConnection(conn);
+        driver.setSession(conn);
         driver.onConnect();
         driver.incoming(makeBinaryFrame("Hello World",true));
         driver.incoming(new CloseInfo(StatusCode.NORMAL).asFrame());
@@ -133,7 +133,7 @@ public class WebSocketEventDriverTest
         WebSocketEventDriver driver = newDriver(socket);
 
         LocalWebSocketSession conn = new LocalWebSocketSession(testname);
-        driver.setConnection(conn);
+        driver.setSession(conn);
         driver.onConnect();
         driver.incoming(WebSocketFrame.text("Hello World"));
         driver.incoming(new CloseInfo(StatusCode.NORMAL).asFrame());

jetty-websocket/websocket-core/src/test/java/org/eclipse/jetty/websocket/protocol/ClosePayloadParserTest.java

@@ -23,9 +23,6 @@ import org.eclipse.jetty.util.StringUtil;
 import org.eclipse.jetty.websocket.api.StatusCode;
 import org.eclipse.jetty.websocket.api.WebSocketBehavior;
 import org.eclipse.jetty.websocket.api.WebSocketPolicy;
-import org.eclipse.jetty.websocket.protocol.CloseInfo;
-import org.eclipse.jetty.websocket.protocol.OpCode;
-import org.eclipse.jetty.websocket.protocol.Parser;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -43,7 +40,7 @@ public class ClosePayloadParserTest
         payload.flip();
 
         ByteBuffer buf = ByteBuffer.allocate(24);
-        buf.put((byte)(0x80 | OpCode.CLOSE.getCode())); // fin + close
+        buf.put((byte)(0x80 | OpCode.CLOSE)); // fin + close
         buf.put((byte)(0x80 | payload.remaining()));
         MaskedByteBuffer.putMask(buf);
         MaskedByteBuffer.putPayload(buf,payload);

jetty-websocket/websocket-core/src/test/java/org/eclipse/jetty/websocket/protocol/IncomingFramesCapture.java

@@ -19,6 +19,7 @@ import static org.hamcrest.Matchers.*;
 
 import java.util.LinkedList;
 
+import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 import org.eclipse.jetty.websocket.api.WebSocketException;
@@ -46,14 +47,14 @@ public class IncomingFramesCapture implements IncomingFrames
         Assert.assertThat(errorType.getSimpleName(),getErrorCount(errorType),is(expectedCount));
     }
 
-    public void assertHasFrame(OpCode op)
+    public void assertHasFrame(byte op)
     {
-        Assert.assertThat(op.name(),getFrameCount(op),greaterThanOrEqualTo(1));
+        Assert.assertThat(OpCode.name(op),getFrameCount(op),greaterThanOrEqualTo(1));
     }
 
-    public void assertHasFrame(OpCode op, int expectedCount)
+    public void assertHasFrame(byte op, int expectedCount)
     {
-        Assert.assertThat(op.name(),getFrameCount(op),is(expectedCount));
+        Assert.assertThat(OpCode.name(op),getFrameCount(op),is(expectedCount));
     }
 
     public void assertHasNoFrames()
@@ -66,6 +67,17 @@ public class IncomingFramesCapture implements IncomingFrames
         Assert.assertThat("Has no errors",errors.size(),is(0));
     }
 
+    public void dump()
+    {
+        System.out.printf("Captured %d incoming frames%n",frames.size());
+        for (int i = 0; i < frames.size(); i++)
+        {
+            WebSocketFrame frame = frames.get(i);
+            System.out.printf("[%3d] %s%n",i,frame);
+            System.out.printf("          %s%n",BufferUtil.toDetailString(frame.getPayload()));
+        }
+    }
+
     public int getErrorCount(Class<? extends WebSocketException> errorType)
     {
         int count = 0;
@@ -83,7 +95,7 @@ public class IncomingFramesCapture implements IncomingFrames
         return errors;
     }
 
-    public int getFrameCount(OpCode op)
+    public int getFrameCount(byte op)
     {
         int count = 0;
         for(WebSocketFrame frame: frames) {
@@ -112,4 +124,9 @@ public class IncomingFramesCapture implements IncomingFrames
     {
         frames.add(frame);
     }
+
+    public int size()
+    {
+        return frames.size();
+    }
 }

jetty-websocket/websocket-core/src/test/java/org/eclipse/jetty/websocket/protocol/OutgoingFramesCapture.java

@@ -40,14 +40,14 @@ public class OutgoingFramesCapture implements OutgoingFrames
         Assert.assertThat("Captured frame count",writes.size(),is(expectedCount));
     }
 
-    public void assertHasFrame(OpCode op)
+    public void assertHasFrame(byte op)
     {
-        Assert.assertThat(op.name(),getFrameCount(op),greaterThanOrEqualTo(1));
+        Assert.assertThat(OpCode.name(op),getFrameCount(op),greaterThanOrEqualTo(1));
     }
 
-    public void assertHasFrame(OpCode op, int expectedCount)
+    public void assertHasFrame(byte op, int expectedCount)
     {
-        Assert.assertThat(op.name(),getFrameCount(op),is(expectedCount));
+        Assert.assertThat(OpCode.name(op),getFrameCount(op),is(expectedCount));
     }
 
     public void assertHasNoFrames()
@@ -66,7 +66,7 @@ public class OutgoingFramesCapture implements OutgoingFrames
         }
     }
 
-    public int getFrameCount(OpCode op)
+    public int getFrameCount(byte op)
     {
         int count = 0;
         for (Write<?> write : writes)

jetty-websocket/websocket-server/src/main/java/org/eclipse/jetty/websocket/server/WebSocketServerAsyncConnection.java

@@ -0,0 +1,41 @@
+package org.eclipse.jetty.websocket.server;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
+
+import org.eclipse.jetty.io.AsyncEndPoint;
+import org.eclipse.jetty.io.ByteBufferPool;
+import org.eclipse.jetty.websocket.api.WebSocketPolicy;
+import org.eclipse.jetty.websocket.io.WebSocketAsyncConnection;
+
+public class WebSocketServerAsyncConnection extends WebSocketAsyncConnection
+{
+    private final WebSocketServerFactory factory;
+    private boolean connected;
+
+    public WebSocketServerAsyncConnection(AsyncEndPoint endp, Executor executor, ScheduledExecutorService scheduler, WebSocketPolicy policy,
+            ByteBufferPool bufferPool, WebSocketServerFactory factory)
+    {
+        super(endp,executor,scheduler,policy,bufferPool);
+        this.factory = factory;
+        this.connected = false;
+    }
+
+    @Override
+    public void onClose()
+    {
+        super.onClose();
+        factory.sessionClosed(getSession());
+    }
+
+    @Override
+    public void onOpen()
+    {
+        if (!connected)
+        {
+            factory.sessionOpened(getSession());
+            connected = true;
+        }
+        super.onOpen();
+    }
+}

jetty-websocket/websocket-server/src/main/java/org/eclipse/jetty/websocket/server/WebSocketServerFactory.java

@@ -50,7 +50,6 @@ import org.eclipse.jetty.websocket.driver.WebSocketEventDriver;
 import org.eclipse.jetty.websocket.extensions.WebSocketExtensionRegistry;
 import org.eclipse.jetty.websocket.io.IncomingFrames;
 import org.eclipse.jetty.websocket.io.OutgoingFrames;
-import org.eclipse.jetty.websocket.io.WebSocketAsyncConnection;
 import org.eclipse.jetty.websocket.io.WebSocketSession;
 import org.eclipse.jetty.websocket.protocol.ExtensionConfig;
 import org.eclipse.jetty.websocket.server.handshake.HandshakeHixie76;
@@ -62,7 +61,6 @@ import org.eclipse.jetty.websocket.server.handshake.HandshakeRFC6455;
 public class WebSocketServerFactory extends AbstractLifeCycle implements WebSocketCreator
 {
     private static final Logger LOG = Log.getLogger(WebSocketServerFactory.class);
-    private final Queue<WebSocketAsyncConnection> connections = new ConcurrentLinkedQueue<WebSocketAsyncConnection>();
 
     private final Map<Integer, WebSocketHandshake> handshakes = new HashMap<>();
     {
@@ -70,6 +68,7 @@ public class WebSocketServerFactory extends AbstractLifeCycle implements WebSock
         handshakes.put(HandshakeHixie76.VERSION,new HandshakeHixie76());
     }
 
+    private final Queue<WebSocketSession> sessions = new ConcurrentLinkedQueue<>();
     /**
      * Have the factory maintain 1 and only 1 scheduler. All connections share this scheduler.
      */
@@ -142,17 +141,20 @@ public class WebSocketServerFactory extends AbstractLifeCycle implements WebSock
         return upgrade(sockreq,sockresp,websocket);
     }
 
-    protected boolean addConnection(WebSocketAsyncConnection connection)
-    {
-        return isRunning() && connections.add(connection);
-    }
-
     protected void closeConnections()
     {
-        for (WebSocketAsyncConnection connection : connections)
+        for (WebSocketSession session : sessions)
         {
-            connection.getEndPoint().close();
+            try
+            {
+                session.close();
             }
+            catch (IOException e)
+            {
+                LOG.warn("Unable to close session",e);
+            }
+        }
+        sessions.clear();
     }
 
     @Override
@@ -182,6 +184,7 @@ public class WebSocketServerFactory extends AbstractLifeCycle implements WebSock
     protected void doStop() throws Exception
     {
         closeConnections();
+        super.doStop();
     }
 
     public WebSocketCreator getCreator()
@@ -278,9 +281,25 @@ public class WebSocketServerFactory extends AbstractLifeCycle implements WebSock
         methodsCache.register(websocketClass);
     }
 
-    protected boolean removeConnection(WebSocketAsyncConnection connection)
+    public boolean sessionClosed(WebSocketSession session)
     {
-        return connections.remove(connection);
+        return isRunning() && sessions.remove(session);
+    }
+
+    public boolean sessionOpened(WebSocketSession session)
+    {
+        if (LOG.isDebugEnabled())
+        {
+            LOG.debug("Session Opened: {}",session);
+        }
+        if (!isRunning())
+        {
+            LOG.warn("Factory is not running");
+            return false;
+        }
+        boolean ret = sessions.offer(session);
+        session.onConnect();
+        return ret;
     }
 
     public void setCreator(WebSocketCreator creator)
@@ -337,7 +356,7 @@ public class WebSocketServerFactory extends AbstractLifeCycle implements WebSock
         AsyncEndPoint endp = http.getEndPoint();
         Executor executor = http.getConnector().findExecutor();
         ByteBufferPool bufferPool = http.getConnector().getByteBufferPool();
-        WebSocketAsyncConnection connection = new WebSocketAsyncConnection(endp,executor,scheduler,websocket.getPolicy(),bufferPool);
+        WebSocketServerAsyncConnection connection = new WebSocketServerAsyncConnection(endp,executor,scheduler,websocket.getPolicy(),bufferPool,this);
         // Tell jetty about the new connection
         request.setAttribute(HttpConnection.UPGRADE_CONNECTION_ATTR,connection);
 
@@ -346,6 +365,7 @@ public class WebSocketServerFactory extends AbstractLifeCycle implements WebSock
 
         // Initialize / Negotiate Extensions
         WebSocketSession session = new WebSocketSession(websocket,connection,getPolicy(),response.getAcceptedSubProtocol());
+        connection.setSession(session);
         List<Extension> extensions = initExtensions(request.getExtensions());
 
         // Start with default routing.
@@ -368,14 +388,17 @@ public class WebSocketServerFactory extends AbstractLifeCycle implements WebSock
                 if (ext.useRsv1())
                 {
                     connection.getGenerator().setRsv1InUse(true);
+                    connection.getParser().setRsv1InUse(true);
                 }
                 if (ext.useRsv2())
                 {
                     connection.getGenerator().setRsv2InUse(true);
+                    connection.getParser().setRsv2InUse(true);
                 }
                 if (ext.useRsv3())
                 {
                     connection.getGenerator().setRsv3InUse(true);
+                    connection.getParser().setRsv3InUse(true);
                 }
             }
 
@@ -399,14 +422,6 @@ public class WebSocketServerFactory extends AbstractLifeCycle implements WebSock
         LOG.debug("Handshake Response: {}",handshaker);
         handshaker.doHandshakeResponse(request,response);
 
-        // Add connection
-        addConnection(connection);
-
-        // Notify POJO of connection
-        // TODO move to WebSocketAsyncConnection.onOpen
-        websocket.setConnection(session);
-        websocket.onConnect();
-
         LOG.debug("Websocket upgrade {} {} {} {}",request.getRequestURI(),version,response.getAcceptedSubProtocol(),connection);
         return true;
     }

jetty-websocket/websocket-server/src/main/java/org/eclipse/jetty/websocket/server/WebSocketServlet.java

@@ -16,6 +16,7 @@
 package org.eclipse.jetty.websocket.server;
 
 import java.io.IOException;
+
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
@@ -134,6 +135,8 @@ public abstract class WebSocketServlet extends HttpServlet
             webSocketFactory = new WebSocketServerFactory(policy);
 
             registerWebSockets(webSocketFactory);
+
+            webSocketFactory.start();
         }
         catch (Exception x)
         {

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/ByteBufferAssert.java

@@ -42,6 +42,11 @@ public class ByteBufferAssert
 
     public static void assertEquals(String message, ByteBuffer expectedBuffer, ByteBuffer actualBuffer)
     {
+        if (expectedBuffer == null)
+        {
+            Assert.assertThat(message,actualBuffer,nullValue());
+            return;
+        }
         byte expectedBytes[] = BufferUtil.toArray(expectedBuffer);
         byte actualBytes[] = BufferUtil.toArray(actualBuffer);
         assertEquals(message,expectedBytes,actualBytes);

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/DeflateExtensionTest.java

@@ -17,12 +17,12 @@ package org.eclipse.jetty.websocket.server;
 
 import static org.hamcrest.Matchers.*;
 
-import java.util.Queue;
 import java.util.concurrent.TimeUnit;
 
 import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
 import org.eclipse.jetty.websocket.server.blockhead.BlockheadClient;
 import org.eclipse.jetty.websocket.server.helper.EchoServlet;
+import org.eclipse.jetty.websocket.server.helper.IncomingFramesCapture;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -73,8 +73,8 @@ public class DeflateExtensionTest
 
             client.write(WebSocketFrame.text(msg.toString()));
 
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,1000);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,1000);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().get(0);
             Assert.assertThat("TEXT.payload",frame.getPayloadAsUTF8(),is(msg.toString()));
         }
         finally

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/FragmentExtensionTest.java

@@ -17,12 +17,12 @@ package org.eclipse.jetty.websocket.server;
 
 import static org.hamcrest.Matchers.*;
 
-import java.util.Queue;
 import java.util.concurrent.TimeUnit;
 
 import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
 import org.eclipse.jetty.websocket.server.blockhead.BlockheadClient;
 import org.eclipse.jetty.websocket.server.helper.EchoServlet;
+import org.eclipse.jetty.websocket.server.helper.IncomingFramesCapture;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -83,10 +83,10 @@ public class FragmentExtensionTest
             client.write(WebSocketFrame.text(msg));
 
             String parts[] = split(msg,fragSize);
-            Queue<WebSocketFrame> frames = client.readFrames(parts.length,TimeUnit.MILLISECONDS,1000);
+            IncomingFramesCapture capture = client.readFrames(parts.length,TimeUnit.MILLISECONDS,1000);
             for (int i = 0; i < parts.length; i++)
             {
-                WebSocketFrame frame = frames.remove();
+                WebSocketFrame frame = capture.getFrames().get(i);
                 Assert.assertThat("text[" + i + "].payload",frame.getPayloadAsUTF8(),is(parts[i]));
             }
         }

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/IdentityExtensionTest.java

@@ -17,12 +17,12 @@ package org.eclipse.jetty.websocket.server;
 
 import static org.hamcrest.Matchers.*;
 
-import java.util.Queue;
 import java.util.concurrent.TimeUnit;
 
 import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
 import org.eclipse.jetty.websocket.server.blockhead.BlockheadClient;
 import org.eclipse.jetty.websocket.server.helper.EchoServlet;
+import org.eclipse.jetty.websocket.server.helper.IncomingFramesCapture;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -66,8 +66,8 @@ public class IdentityExtensionTest
 
             client.write(WebSocketFrame.text("Hello"));
 
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,1000);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,1000);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().get(0);
             Assert.assertThat("TEXT.payload",frame.getPayloadAsUTF8(),is("Hello"));
         }
         finally

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/WebSocketServletRFCTest.java

@@ -20,7 +20,6 @@ import static org.hamcrest.Matchers.*;
 import java.net.SocketException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
-import java.util.Queue;
 import java.util.concurrent.TimeUnit;
 
 import org.eclipse.jetty.util.Utf8Appendable.NotUtf8Exception;
@@ -31,6 +30,7 @@ import org.eclipse.jetty.websocket.protocol.Generator;
 import org.eclipse.jetty.websocket.protocol.OpCode;
 import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
 import org.eclipse.jetty.websocket.server.blockhead.BlockheadClient;
+import org.eclipse.jetty.websocket.server.helper.IncomingFramesCapture;
 import org.eclipse.jetty.websocket.server.helper.RFCServlet;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -98,8 +98,8 @@ public class WebSocketServletRFCTest
             client.write(bin); // write buf3 (fin=true)
 
             // Read frame echo'd back (hopefully a single binary frame)
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,1000);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,1000);
-            WebSocketFrame binmsg = frames.remove();
+            WebSocketFrame binmsg = capture.getFrames().get(0);
             int expectedSize = buf1.length + buf2.length + buf3.length;
             Assert.assertThat("BinaryFrame.payloadLength",binmsg.getPayloadLength(),is(expectedSize));
 
@@ -164,8 +164,8 @@ public class WebSocketServletRFCTest
             client.write(WebSocketFrame.text(msg));
 
             // Read frame (hopefully text frame)
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame tf = frames.remove();
+            WebSocketFrame tf = capture.getFrames().get(0);
             Assert.assertThat("Text Frame.status code",tf.getPayloadAsUTF8(),is(msg));
         }
         finally
@@ -193,9 +193,9 @@ public class WebSocketServletRFCTest
 
             // now wait for the server to time out
             // should be 2 frames, the TextFrame echo, and then the Close on disconnect
-            Queue<WebSocketFrame> frames = client.readFrames(2,TimeUnit.SECONDS,2);
+            IncomingFramesCapture capture = client.readFrames(2,TimeUnit.SECONDS,2);
-            Assert.assertThat("frames[0].opcode",frames.remove().getOpCode(),is(OpCode.TEXT));
+            Assert.assertThat("frames[0].opcode",capture.getFrames().get(0).getOpCode(),is(OpCode.TEXT));
-            Assert.assertThat("frames[1].opcode",frames.remove().getOpCode(),is(OpCode.CLOSE));
+            Assert.assertThat("frames[1].opcode",capture.getFrames().get(1).getOpCode(),is(OpCode.CLOSE));
         }
         finally
         {
@@ -221,8 +221,8 @@ public class WebSocketServletRFCTest
             client.write(WebSocketFrame.text("CRASH"));
 
             // Read frame (hopefully close frame)
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame cf = frames.remove();
+            WebSocketFrame cf = capture.getFrames().get(0);
             CloseInfo close = new CloseInfo(cf);
             Assert.assertThat("Close Frame.status code",close.getStatusCode(),is(StatusCode.SERVER_ERROR));
         }
@@ -261,8 +261,8 @@ public class WebSocketServletRFCTest
                 Assert.assertThat("Exception",e.getMessage(),containsString("Broken pipe"));
             }
 
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.SECONDS,1);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.SECONDS,1);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().get(0);
             Assert.assertThat("frames[0].opcode",frame.getOpCode(),is(OpCode.CLOSE));
             CloseInfo close = new CloseInfo(frame);
             Assert.assertThat("Close Status Code",close.getStatusCode(),is(StatusCode.MESSAGE_TOO_LARGE));
@@ -302,8 +302,8 @@ public class WebSocketServletRFCTest
                 Assert.assertThat("Exception",e.getMessage(),containsString("Broken pipe"));
             }
 
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.SECONDS,1);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.SECONDS,1);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().get(0);
             Assert.assertThat("frames[0].opcode",frame.getOpCode(),is(OpCode.CLOSE));
             CloseInfo close = new CloseInfo(frame);
             Assert.assertThat("Close Status Code",close.getStatusCode(),is(StatusCode.MESSAGE_TOO_LARGE));
@@ -332,8 +332,8 @@ public class WebSocketServletRFCTest
             ByteBuffer bb = generator.generate(txt);
             client.writeRaw(bb);
 
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.SECONDS,1);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.SECONDS,1);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().get(0);
             Assert.assertThat("frames[0].opcode",frame.getOpCode(),is(OpCode.CLOSE));
             CloseInfo close = new CloseInfo(frame);
             Assert.assertThat("Close Status Code",close.getStatusCode(),is(StatusCode.BAD_PAYLOAD));

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/ab/AbstractABCase.java

@@ -2,6 +2,10 @@ package org.eclipse.jetty.websocket.server.ab;
 
 import java.nio.ByteBuffer;
 
+import org.eclipse.jetty.io.ByteBufferPool;
+import org.eclipse.jetty.io.StandardByteBufferPool;
+import org.eclipse.jetty.websocket.api.WebSocketPolicy;
+import org.eclipse.jetty.websocket.protocol.Generator;
 import org.eclipse.jetty.websocket.server.SimpleServletServer;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -11,11 +15,22 @@ public abstract class AbstractABCase
     protected static final byte FIN = (byte)0x80;
     protected static final byte NOFIN = 0x00;
     private static final byte MASKED_BIT = (byte)0x80;
-    private static final byte[] MASK =
+    protected static final byte[] MASK =
     { 0x12, 0x34, 0x56, 0x78 };
 
+    protected static Generator strictGenerator;
+    protected static Generator laxGenerator;
     protected static SimpleServletServer server;
 
+    @BeforeClass
+    public static void initGenerators()
+    {
+        WebSocketPolicy policy = WebSocketPolicy.newClientPolicy();
+        ByteBufferPool bufferPool = new StandardByteBufferPool();
+        strictGenerator = new Generator(policy,bufferPool,true);
+        laxGenerator = new Generator(policy,bufferPool,false);
+    }
+
     @BeforeClass
     public static void startServer() throws Exception
     {
@@ -29,6 +44,16 @@ public abstract class AbstractABCase
         server.stop();
     }
 
+    public Generator getLaxGenerator()
+    {
+        return laxGenerator;
+    }
+
+    public SimpleServletServer getServer()
+    {
+        return server;
+    }
+
     protected byte[] masked(final byte[] data)
     {
         int len = data.length;

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/ab/AllTests.java

@@ -5,7 +5,7 @@ import org.junit.runners.Suite;
 
 @RunWith(Suite.class)
 @Suite.SuiteClasses(
-{ TestABCase1.class, TestABCase5.class, TestABCase7_9.class })
+{ TestABCase1.class, TestABCase2.class, TestABCase3.class, TestABCase4.class, TestABCase5.class, TestABCase7_9.class })
 public class AllTests
 {
     /* let junit do the rest */

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/ab/Fuzzer.java

@@ -0,0 +1,187 @@
+package org.eclipse.jetty.websocket.server.ab;
+
+import static org.hamcrest.Matchers.*;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
+import org.eclipse.jetty.websocket.protocol.CloseInfo;
+import org.eclipse.jetty.websocket.protocol.Generator;
+import org.eclipse.jetty.websocket.protocol.OpCode;
+import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
+import org.eclipse.jetty.websocket.server.ByteBufferAssert;
+import org.eclipse.jetty.websocket.server.blockhead.BlockheadClient;
+import org.eclipse.jetty.websocket.server.helper.IncomingFramesCapture;
+import org.junit.Assert;
+
+/**
+ * Fuzzing utility for the AB tests.
+ */
+public class Fuzzer
+{
+    public static enum SendMode
+    {
+        BULK,
+        PER_FRAME,
+        SLOW
+    }
+
+    private static final Logger LOG = Log.getLogger(Fuzzer.class);
+
+    // Client side framing mask
+    protected static final byte[] MASK =
+    { 0x11, 0x22, 0x33, 0x44 };
+
+    private final BlockheadClient client;
+    private final Generator generator;
+    private SendMode sendMode = SendMode.BULK;
+    private int slowSendSegmentSize = 5;
+
+    public Fuzzer(AbstractABCase testcase) throws Exception
+    {
+        this.client = new BlockheadClient(testcase.getServer().getServerUri());
+        this.generator = testcase.getLaxGenerator();
+    }
+
+    public void close()
+    {
+        this.client.disconnect();
+    }
+
+    public void connect() throws IOException
+    {
+        if (!client.isConnected())
+        {
+            client.connect();
+            client.sendStandardRequest();
+            client.expectUpgradeResponse();
+        }
+    }
+
+    public void expect(List<WebSocketFrame> expect) throws IOException, TimeoutException
+    {
+        int expectedCount = expect.size();
+
+        // Read frames
+        IncomingFramesCapture capture = client.readFrames(expect.size(),TimeUnit.MILLISECONDS,500);
+
+        String prefix = "";
+        for (int i = 0; i < expectedCount; i++)
+        {
+            WebSocketFrame expected = expect.get(i);
+            WebSocketFrame actual = capture.getFrames().pop();
+
+            prefix = "Frame[" + i + "]";
+
+            Assert.assertThat(prefix + ".opcode",actual.getOpCode(),is(expected.getOpCode()));
+            prefix += "/" + actual.getOpCode();
+            if (expected.getOpCode() == OpCode.CLOSE)
+            {
+                CloseInfo expectedClose = new CloseInfo(expected);
+                CloseInfo actualClose = new CloseInfo(actual);
+                Assert.assertThat(prefix + ".statusCode",actualClose.getStatusCode(),is(expectedClose.getStatusCode()));
+            }
+            else
+            {
+                Assert.assertThat(prefix + ".payloadLength",actual.getPayloadLength(),is(expected.getPayloadLength()));
+                ByteBufferAssert.assertEquals(prefix + ".payload",expected.getPayload(),actual.getPayload());
+            }
+        }
+    }
+
+    public void expect(WebSocketFrame expect) throws IOException, TimeoutException
+    {
+        expect(Collections.singletonList(expect));
+    }
+
+    public SendMode getSendMode()
+    {
+        return sendMode;
+    }
+
+    public int getSlowSendSegmentSize()
+    {
+        return slowSendSegmentSize;
+    }
+
+    public void send(ByteBuffer buf) throws IOException
+    {
+        Assert.assertThat("Client connected",client.isConnected(),is(true));
+        LOG.debug("Sending bytes {}",BufferUtil.toDetailString(buf));
+        if (sendMode == SendMode.SLOW)
+        {
+            client.writeRawSlowly(buf,slowSendSegmentSize);
+        }
+        else
+        {
+            client.writeRaw(buf);
+        }
+    }
+
+    public void send(List<WebSocketFrame> send) throws IOException
+    {
+        Assert.assertThat("Client connected",client.isConnected(),is(true));
+        LOG.debug("Sending {} frames (mode {})",send.size(),sendMode);
+        if ((sendMode == SendMode.BULK) || (sendMode == SendMode.SLOW))
+        {
+            int buflen = 0;
+            for (WebSocketFrame f : send)
+            {
+                buflen += f.getPayloadLength() + Generator.OVERHEAD;
+            }
+            ByteBuffer buf = ByteBuffer.allocate(buflen);
+            BufferUtil.clearToFill(buf);
+
+            // Generate frames
+            for (WebSocketFrame f : send)
+            {
+                f.setMask(MASK); // make sure we have mask set
+                BufferUtil.put(generator.generate(f),buf);
+            }
+            BufferUtil.flipToFlush(buf,0);
+
+            // Write Data Frame
+            switch (sendMode)
+            {
+                case BULK:
+                    client.writeRaw(buf);
+                    break;
+                case SLOW:
+                    client.writeRawSlowly(buf,slowSendSegmentSize);
+                    break;
+            }
+        }
+        else if (sendMode == SendMode.PER_FRAME)
+        {
+            for (WebSocketFrame f : send)
+            {
+                f.setMask(MASK); // make sure we have mask set
+                // Using lax generator, generate and send
+                client.writeRaw(generator.generate(f));
+                client.flush();
+            }
+        }
+    }
+
+    public void send(WebSocketFrame send) throws IOException
+    {
+        send(Collections.singletonList(send));
+    }
+
+    public void setSendMode(SendMode sendMode)
+    {
+        this.sendMode = sendMode;
+    }
+
+    public void setSlowSendSegmentSize(int segmentSize)
+    {
+        this.slowSendSegmentSize = segmentSize;
+    }
+}

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/ab/TestABCase1.java

@@ -15,297 +15,517 @@
 //========================================================================
 package org.eclipse.jetty.websocket.server.ab;
 
-import static org.hamcrest.Matchers.*;
+import java.util.ArrayList;
-
-import java.nio.ByteBuffer;
 import java.util.Arrays;
-import java.util.Queue;
+import java.util.List;
-import java.util.concurrent.TimeUnit;
 
-import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.websocket.api.StatusCode;
-import org.eclipse.jetty.websocket.protocol.Generator;
+import org.eclipse.jetty.websocket.protocol.CloseInfo;
-import org.eclipse.jetty.websocket.protocol.OpCode;
 import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
-import org.eclipse.jetty.websocket.server.ByteBufferAssert;
+import org.eclipse.jetty.websocket.server.ab.Fuzzer.SendMode;
-import org.eclipse.jetty.websocket.server.blockhead.BlockheadClient;
-import org.junit.Assert;
 import org.junit.Test;
 
 public class TestABCase1 extends AbstractABCase
 {
     /**
-     * Echo 0 byte text message
+     * Echo 0 byte TEXT message
      */
     @Test
     public void testCase1_1_1() throws Exception
     {
-        BlockheadClient client = new BlockheadClient(server.getServerUri());
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text());
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text());
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
         try
         {
-            client.connect();
+            fuzzer.connect();
-            client.sendStandardRequest();
+            fuzzer.setSendMode(SendMode.BULK);
-            client.expectUpgradeResponse();
+            fuzzer.send(send);
-
+            fuzzer.expect(expect);
-            ByteBuffer buf = ByteBuffer.allocate(16);
-            BufferUtil.clearToFill(buf);
-
-            buf.put((byte)(0x00 | FIN | OpCode.TEXT.getCode()));
-            putPayloadLength(buf,0);
-            putMask(buf);
-
-            BufferUtil.flipToFlush(buf,0);
-            client.writeRaw(buf);
-
-            // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
-            Assert.assertThat("frame should be TEXT frame",frame.getOpCode(),is(OpCode.TEXT));
-            Assert.assertThat("Text.payloadLength",frame.getPayloadLength(),is(0));
         }
         finally
         {
-            client.close();
+            fuzzer.close();
         }
     }
 
     /**
-     * Echo 125 byte text message (uses small 7-bit payload length)
+     * Echo 125 byte TEXT message (uses small 7-bit payload length)
      */
     @Test
     public void testCase1_1_2() throws Exception
     {
-        BlockheadClient client = new BlockheadClient(server.getServerUri());
+        byte payload[] = new byte[125];
+        Arrays.fill(payload,(byte)'*');
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
         try
         {
-            client.connect();
+            fuzzer.connect();
-            client.sendStandardRequest();
+            fuzzer.setSendMode(SendMode.BULK);
-            client.expectUpgradeResponse();
+            fuzzer.send(send);
-
+            fuzzer.expect(expect);
-            byte msg[] = new byte[125];
-            Arrays.fill(msg,(byte)'*');
-
-            ByteBuffer buf = ByteBuffer.allocate(msg.length + Generator.OVERHEAD);
-            BufferUtil.clearToFill(buf);
-
-            buf.put((byte)(0x00 | FIN | OpCode.TEXT.getCode()));
-            putPayloadLength(buf,msg.length);
-            putMask(buf);
-            buf.put(masked(msg));
-
-            BufferUtil.flipToFlush(buf,0);
-            client.writeRaw(buf);
-
-            // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
-            Assert.assertThat("frame should be TEXT frame",frame.getOpCode(),is(OpCode.TEXT));
-            Assert.assertThat("Text.payloadLength",frame.getPayloadLength(),is(msg.length));
-            ByteBufferAssert.assertEquals("Text.payload",msg,frame.getPayload());
         }
         finally
         {
-            client.close();
+            fuzzer.close();
         }
     }
 
     /**
-     * Echo 126 byte text message (uses medium 2 byte payload length)
+     * Echo 126 byte TEXT message (uses medium 2 byte payload length)
      */
     @Test
     public void testCase1_1_3() throws Exception
     {
-        BlockheadClient client = new BlockheadClient(server.getServerUri());
+        byte payload[] = new byte[126];
+        Arrays.fill(payload,(byte)'*');
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
         try
         {
-            client.connect();
+            fuzzer.connect();
-            client.sendStandardRequest();
+            fuzzer.setSendMode(SendMode.BULK);
-            client.expectUpgradeResponse();
+            fuzzer.send(send);
-
+            fuzzer.expect(expect);
-            byte msg[] = new byte[126];
-            Arrays.fill(msg,(byte)'*');
-
-            ByteBuffer buf = ByteBuffer.allocate(msg.length + Generator.OVERHEAD);
-            BufferUtil.clearToFill(buf);
-
-            buf.put((byte)(0x00 | FIN | OpCode.TEXT.getCode()));
-            putPayloadLength(buf,msg.length);
-            putMask(buf);
-            buf.put(masked(msg));
-
-            BufferUtil.flipToFlush(buf,0);
-            client.writeRaw(buf);
-
-            // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
-            Assert.assertThat("frame should be TEXT frame",frame.getOpCode(),is(OpCode.TEXT));
-            Assert.assertThat("Text.payloadLength",frame.getPayloadLength(),is(msg.length));
-            ByteBufferAssert.assertEquals("Text.payload",msg,frame.getPayload());
         }
         finally
         {
-            client.close();
+            fuzzer.close();
         }
     }
 
     /**
-     * Echo 127 byte text message (uses medium 2 byte payload length)
+     * Echo 127 byte TEXT message (uses medium 2 byte payload length)
      */
     @Test
     public void testCase1_1_4() throws Exception
     {
-        BlockheadClient client = new BlockheadClient(server.getServerUri());
+        byte payload[] = new byte[127];
+        Arrays.fill(payload,(byte)'*');
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
         try
         {
-            client.connect();
+            fuzzer.connect();
-            client.sendStandardRequest();
+            fuzzer.setSendMode(SendMode.BULK);
-            client.expectUpgradeResponse();
+            fuzzer.send(send);
-
+            fuzzer.expect(expect);
-            byte msg[] = new byte[127];
-            Arrays.fill(msg,(byte)'*');
-
-            ByteBuffer buf = ByteBuffer.allocate(msg.length + Generator.OVERHEAD);
-            BufferUtil.clearToFill(buf);
-
-            buf.put((byte)(0x00 | FIN | OpCode.TEXT.getCode()));
-            putPayloadLength(buf,msg.length);
-            putMask(buf);
-            buf.put(masked(msg));
-
-            BufferUtil.flipToFlush(buf,0);
-            client.writeRaw(buf);
-
-            // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
-            Assert.assertThat("frame should be TEXT frame",frame.getOpCode(),is(OpCode.TEXT));
-            Assert.assertThat("Text.payloadLength",frame.getPayloadLength(),is(msg.length));
-            ByteBufferAssert.assertEquals("Text.payload",msg,frame.getPayload());
         }
         finally
         {
-            client.close();
+            fuzzer.close();
         }
     }
 
     /**
-     * Echo 128 byte text message (uses medium 2 byte payload length)
+     * Echo 128 byte TEXT message (uses medium 2 byte payload length)
      */
     @Test
     public void testCase1_1_5() throws Exception
     {
-        BlockheadClient client = new BlockheadClient(server.getServerUri());
+        byte payload[] = new byte[128];
+        Arrays.fill(payload,(byte)'*');
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
         try
         {
-            client.connect();
+            fuzzer.connect();
-            client.sendStandardRequest();
+            fuzzer.setSendMode(SendMode.BULK);
-            client.expectUpgradeResponse();
+            fuzzer.send(send);
-
+            fuzzer.expect(expect);
-            byte msg[] = new byte[128];
-            Arrays.fill(msg,(byte)'*');
-
-            ByteBuffer buf = ByteBuffer.allocate(msg.length + Generator.OVERHEAD);
-            BufferUtil.clearToFill(buf);
-
-            buf.put((byte)(0x00 | FIN | OpCode.TEXT.getCode()));
-            putPayloadLength(buf,msg.length);
-            putMask(buf);
-            buf.put(masked(msg));
-
-            BufferUtil.flipToFlush(buf,0);
-            client.writeRaw(buf);
-
-            // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
-            Assert.assertThat("frame should be TEXT frame",frame.getOpCode(),is(OpCode.TEXT));
-            Assert.assertThat("Text.payloadLength",frame.getPayloadLength(),is(msg.length));
-            ByteBufferAssert.assertEquals("Text.payload",msg,frame.getPayload());
         }
         finally
         {
-            client.close();
+            fuzzer.close();
         }
     }
 
     /**
-     * Echo 65535 byte text message (uses medium 2 byte payload length)
+     * Echo 65535 byte TEXT message (uses medium 2 byte payload length)
      */
     @Test
     public void testCase1_1_6() throws Exception
     {
-        BlockheadClient client = new BlockheadClient(server.getServerUri());
+        byte payload[] = new byte[65535];
+        Arrays.fill(payload,(byte)'*');
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
         try
         {
-            client.connect();
+            fuzzer.connect();
-            client.sendStandardRequest();
+            fuzzer.setSendMode(SendMode.BULK);
-            client.expectUpgradeResponse();
+            fuzzer.send(send);
-
+            fuzzer.expect(expect);
-            byte msg[] = new byte[65535];
-            Arrays.fill(msg,(byte)'*');
-
-            ByteBuffer buf = ByteBuffer.allocate(msg.length + Generator.OVERHEAD);
-            BufferUtil.clearToFill(buf);
-
-            buf.put((byte)(0x00 | FIN | OpCode.TEXT.getCode()));
-            putPayloadLength(buf,msg.length);
-            putMask(buf);
-            buf.put(masked(msg));
-
-            BufferUtil.flipToFlush(buf,0);
-            client.writeRaw(buf);
-
-            // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
-            Assert.assertThat("frame should be TEXT frame",frame.getOpCode(),is(OpCode.TEXT));
-            Assert.assertThat("Text.payloadLength",frame.getPayloadLength(),is(msg.length));
-            ByteBufferAssert.assertEquals("Text.payload",msg,frame.getPayload());
         }
         finally
         {
-            client.close();
+            fuzzer.close();
         }
     }
 
     /**
-     * Echo 65536 byte text message (uses large 8 byte payload length)
+     * Echo 65536 byte TEXT message (uses large 8 byte payload length)
      */
     @Test
     public void testCase1_1_7() throws Exception
     {
-        BlockheadClient client = new BlockheadClient(server.getServerUri());
+        byte payload[] = new byte[65536];
+        Arrays.fill(payload,(byte)'*');
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
         try
         {
-            client.connect();
+            fuzzer.connect();
-            client.sendStandardRequest();
+            fuzzer.setSendMode(SendMode.BULK);
-            client.expectUpgradeResponse();
+            fuzzer.send(send);
-
+            fuzzer.expect(expect);
-            byte msg[] = new byte[65536];
-            Arrays.fill(msg,(byte)'*');
-
-            ByteBuffer buf = ByteBuffer.allocate(msg.length + Generator.OVERHEAD);
-            BufferUtil.clearToFill(buf);
-
-            buf.put((byte)(0x00 | FIN | OpCode.TEXT.getCode()));
-            putPayloadLength(buf,msg.length);
-            putMask(buf);
-            buf.put(masked(msg));
-
-            BufferUtil.flipToFlush(buf,0);
-            client.writeRaw(buf);
-
-            // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
-            Assert.assertThat("frame should be TEXT frame",frame.getOpCode(),is(OpCode.TEXT));
-            Assert.assertThat("Text.payloadLength",frame.getPayloadLength(),is(msg.length));
-            ByteBufferAssert.assertEquals("Text.payload",msg,frame.getPayload());
         }
         finally
         {
-            client.close();
+            fuzzer.close();
         }
     }
 
+    /**
+     * Echo 65536 byte TEXT message (uses large 8 byte payload length).
+     * <p>
+     * Only send 1 TEXT frame from client, but in small segments (flushed after each).
+     * <p>
+     * This is done to test the parsing together of the frame on the server side.
+     */
+    @Test
+    public void testCase1_1_8() throws Exception
+    {
+        byte payload[] = new byte[65536];
+        Arrays.fill(payload,(byte)'*');
+        int segmentSize = 997;
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(SendMode.SLOW);
+            fuzzer.setSlowSendSegmentSize(segmentSize);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Echo 0 byte BINARY message
+     */
+    @Test
+    public void testCase1_2_1() throws Exception
+    {
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.binary());
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.binary());
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Echo 125 byte BINARY message (uses small 7-bit payload length)
+     */
+    @Test
+    public void testCase1_2_2() throws Exception
+    {
+        byte payload[] = new byte[125];
+        Arrays.fill(payload,(byte)0xFE);
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.binary().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.binary().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Echo 126 byte BINARY message (uses medium 2 byte payload length)
+     */
+    @Test
+    public void testCase1_2_3() throws Exception
+    {
+        byte payload[] = new byte[126];
+        Arrays.fill(payload,(byte)0xFE);
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.binary().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.binary().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Echo 127 byte BINARY message (uses medium 2 byte payload length)
+     */
+    @Test
+    public void testCase1_2_4() throws Exception
+    {
+        byte payload[] = new byte[127];
+        Arrays.fill(payload,(byte)0xFE);
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.binary().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.binary().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Echo 128 byte BINARY message (uses medium 2 byte payload length)
+     */
+    @Test
+    public void testCase1_2_5() throws Exception
+    {
+        byte payload[] = new byte[128];
+        Arrays.fill(payload,(byte)0xFE);
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.binary().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.binary().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Echo 65535 byte BINARY message (uses medium 2 byte payload length)
+     */
+    @Test
+    public void testCase1_2_6() throws Exception
+    {
+        byte payload[] = new byte[65535];
+        Arrays.fill(payload,(byte)0xFE);
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.binary().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.binary().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Echo 65536 byte BINARY message (uses large 8 byte payload length)
+     */
+    @Test
+    public void testCase1_2_7() throws Exception
+    {
+        byte payload[] = new byte[65536];
+        Arrays.fill(payload,(byte)0xFE);
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.binary().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.binary().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Echo 65536 byte BINARY message (uses large 8 byte payload length).
+     * <p>
+     * Only send 1 BINARY frame from client, but in small segments (flushed after each).
+     * <p>
+     * This is done to test the parsing together of the frame on the server side.
+     */
+    @Test
+    public void testCase1_2_8() throws Exception
+    {
+        byte payload[] = new byte[65536];
+        Arrays.fill(payload,(byte)0xFE);
+        int segmentSize = 997;
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.binary().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.binary().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(SendMode.SLOW);
+            fuzzer.setSlowSendSegmentSize(segmentSize);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
 }

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/ab/TestABCase2.java

@@ -0,0 +1,356 @@
+package org.eclipse.jetty.websocket.server.ab;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.eclipse.jetty.util.StringUtil;
+import org.eclipse.jetty.websocket.api.StatusCode;
+import org.eclipse.jetty.websocket.protocol.CloseInfo;
+import org.eclipse.jetty.websocket.protocol.OpCode;
+import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
+import org.junit.Test;
+
+public class TestABCase2 extends AbstractABCase
+{
+    /**
+     * Ping without payload
+     */
+    @Test
+    public void testCase2_1() throws Exception
+    {
+        WebSocketFrame send = WebSocketFrame.ping();
+
+        WebSocketFrame expect = WebSocketFrame.pong();
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * 10 pings
+     */
+    @Test
+    public void testCase2_10() throws Exception
+    {
+        // send 10 pings each with unique payload
+        // send close
+        // expect 10 pongs with our unique payload
+        // expect close
+
+        int pingCount = 10;
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        List<WebSocketFrame> expect = new ArrayList<>();
+
+        for (int i = 0; i < pingCount; i++)
+        {
+            String payload = String.format("ping-%d[%X]",i,i);
+            send.add(WebSocketFrame.ping().setPayload(payload));
+            expect.add(WebSocketFrame.pong().setPayload(payload));
+        }
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * 10 pings, sent slowly
+     */
+    @Test
+    public void testCase2_11() throws Exception
+    {
+        // send 10 pings (slowly) each with unique payload
+        // send close
+        // expect 10 pongs with OUR payload
+        // expect close
+
+        int pingCount = 10;
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        List<WebSocketFrame> expect = new ArrayList<>();
+
+        for (int i = 0; i < pingCount; i++)
+        {
+            String payload = String.format("ping-%d[%X]",i,i);
+            send.add(WebSocketFrame.ping().setPayload(payload));
+            expect.add(WebSocketFrame.pong().setPayload(payload));
+        }
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.SLOW);
+            fuzzer.setSlowSendSegmentSize(5);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Ping with small text payload
+     */
+    @Test
+    public void testCase2_2() throws Exception
+    {
+        byte payload[] = StringUtil.getUtf8Bytes("Hello world");
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.ping().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.pong().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Ping with small binary (non-utf8) payload
+     */
+    @Test
+    public void testCase2_3() throws Exception
+    {
+        byte payload[] = new byte[]
+        { 0x00, (byte)0xFF, (byte)0xFE, (byte)0xFD, (byte)0xFC, (byte)0xFB, 0x00, (byte)0xFF };
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.ping().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.pong().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Ping with 125 byte binary payload
+     */
+    @Test
+    public void testCase2_4() throws Exception
+    {
+        byte payload[] = new byte[125];
+        Arrays.fill(payload,(byte)0xFE);
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.ping().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.pong().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Ping with 126 byte binary payload
+     */
+    @Test
+    public void testCase2_5() throws Exception
+    {
+        byte payload[] = new byte[126]; // intentionally too big
+        Arrays.fill(payload,(byte)0xFE);
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        // trick websocket frame into making extra large payload for ping
+        send.add(WebSocketFrame.binary(payload).setOpCode(OpCode.PING));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(new CloseInfo(StatusCode.PROTOCOL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Ping with 125 byte binary payload (slow send)
+     */
+    @Test
+    public void testCase2_6() throws Exception
+    {
+        byte payload[] = new byte[125];
+        Arrays.fill(payload,(byte)0xFE);
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.ping().setPayload(payload));
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.pong().setPayload(payload));
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.SLOW);
+            fuzzer.setSlowSendSegmentSize(1);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Unsolicited pong frame without payload
+     */
+    @Test
+    public void testCase2_7() throws Exception
+    {
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.pong()); // unsolicited pong
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Unsolicited pong frame with basic payload
+     */
+    @Test
+    public void testCase2_8() throws Exception
+    {
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.pong().setPayload("unsolicited")); // unsolicited pong
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Unsolicited pong frame, then ping with basic payload
+     */
+    @Test
+    public void testCase2_9() throws Exception
+    {
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.pong().setPayload("unsolicited")); // unsolicited pong
+        send.add(WebSocketFrame.ping().setPayload("our ping")); // our ping
+        send.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.pong().setPayload("our ping")); // our pong
+        expect.add(new CloseInfo(StatusCode.NORMAL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+}

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/ab/TestABCase3.java

@@ -0,0 +1,216 @@
+package org.eclipse.jetty.websocket.server.ab;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.eclipse.jetty.websocket.api.StatusCode;
+import org.eclipse.jetty.websocket.protocol.CloseInfo;
+import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
+import org.junit.Test;
+
+public class TestABCase3 extends AbstractABCase
+{
+    /**
+     * Send small text frame, with RSV1 == true, with no extensions defined.
+     */
+    @Test
+    public void testCase3_1() throws Exception
+    {
+        WebSocketFrame send = WebSocketFrame.text("small").setRsv1(true); // intentionally bad
+
+        WebSocketFrame expect = new CloseInfo(StatusCode.PROTOCOL).asFrame();
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Send small text frame, send again with RSV2 == true, then ping, with no extensions defined.
+     */
+    @Test
+    public void testCase3_2() throws Exception
+    {
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text("small"));
+        send.add(WebSocketFrame.text("small").setRsv2(true)); // intentionally bad
+        send.add(WebSocketFrame.ping().setPayload("ping"));
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text("small")); // echo on good frame
+        expect.add(new CloseInfo(StatusCode.PROTOCOL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Send small text frame, send again with (RSV1 & RSV2), then ping, with no extensions defined.
+     */
+    @Test
+    public void testCase3_3() throws Exception
+    {
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text("small"));
+        send.add(WebSocketFrame.text("small").setRsv1(true).setRsv2(true)); // intentionally bad
+        send.add(WebSocketFrame.ping().setPayload("ping"));
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text("small")); // echo on good frame
+        expect.add(new CloseInfo(StatusCode.PROTOCOL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.PER_FRAME);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Send small text frame, send again with (RSV3), then ping, with no extensions defined.
+     */
+    @Test
+    public void testCase3_4() throws Exception
+    {
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text("small"));
+        send.add(WebSocketFrame.text("small").setRsv3(true)); // intentionally bad
+        send.add(WebSocketFrame.ping().setPayload("ping"));
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text("small")); // echo on good frame
+        expect.add(new CloseInfo(StatusCode.PROTOCOL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.SLOW);
+            fuzzer.setSlowSendSegmentSize(1);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Send binary frame with (RSV3 & RSV1), with no extensions defined.
+     */
+    @Test
+    public void testCase3_5() throws Exception
+    {
+        byte payload[] = new byte[8];
+        Arrays.fill(payload,(byte)0xFF);
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.binary(payload).setRsv3(true).setRsv1(true)); // intentionally bad
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(new CloseInfo(StatusCode.PROTOCOL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Send ping frame with (RSV3 & RSV2), with no extensions defined.
+     */
+    @Test
+    public void testCase3_6() throws Exception
+    {
+        byte payload[] = new byte[8];
+        Arrays.fill(payload,(byte)0xFF);
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.ping().setPayload(payload).setRsv3(true).setRsv2(true)); // intentionally bad
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(new CloseInfo(StatusCode.PROTOCOL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Send close frame with (RSV3 & RSV2 & RSV1), with no extensions defined.
+     */
+    @Test
+    public void testCase3_7() throws Exception
+    {
+        byte payload[] = new byte[8];
+        Arrays.fill(payload,(byte)0xFF);
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        WebSocketFrame frame = new CloseInfo(StatusCode.NORMAL).asFrame();
+        frame.setRsv1(true);
+        frame.setRsv2(true);
+        frame.setRsv3(true);
+        send.add(frame); // intentionally bad
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(new CloseInfo(StatusCode.PROTOCOL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+}

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/ab/TestABCase4.java

@@ -0,0 +1,154 @@
+package org.eclipse.jetty.websocket.server.ab;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.eclipse.jetty.util.StringUtil;
+import org.eclipse.jetty.websocket.api.StatusCode;
+import org.eclipse.jetty.websocket.protocol.CloseInfo;
+import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
+import org.junit.Test;
+
+public class TestABCase4 extends AbstractABCase
+{
+    /**
+     * Send opcode 3 (reserved)
+     */
+    @Test
+    public void testCase4_1_1() throws Exception
+    {
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(new WebSocketFrame((byte)3)); // intentionally bad
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(new CloseInfo(StatusCode.PROTOCOL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Send opcode 4 (reserved), with payload
+     */
+    @Test
+    public void testCase4_1_2() throws Exception
+    {
+        byte payload[] = StringUtil.getUtf8Bytes("reserved payload");
+
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(new WebSocketFrame((byte)4).setPayload(payload)); // intentionally bad
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(new CloseInfo(StatusCode.PROTOCOL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Send small text, then frame with opcode 5 (reserved), then ping
+     */
+    @Test
+    public void testCase4_1_3() throws Exception
+    {
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text("hello"));
+        send.add(new WebSocketFrame((byte)5)); // intentionally bad
+        send.add(WebSocketFrame.ping());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text("hello")); // echo
+        expect.add(new CloseInfo(StatusCode.PROTOCOL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Send small text, then frame with opcode 6 (reserved) w/payload, then ping
+     */
+    @Test
+    public void testCase4_1_4() throws Exception
+    {
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text("hello"));
+        send.add(new WebSocketFrame((byte)6).setPayload("bad")); // intentionally bad
+        send.add(WebSocketFrame.ping());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text("hello")); // echo
+        expect.add(new CloseInfo(StatusCode.PROTOCOL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+
+    /**
+     * Send small text, then frame with opcode 7 (reserved) w/payload, then ping
+     */
+    @Test
+    public void testCase4_1_5() throws Exception
+    {
+        List<WebSocketFrame> send = new ArrayList<>();
+        send.add(WebSocketFrame.text("hello"));
+        send.add(new WebSocketFrame((byte)7).setPayload("bad")); // intentionally bad
+        send.add(WebSocketFrame.ping());
+
+        List<WebSocketFrame> expect = new ArrayList<>();
+        expect.add(WebSocketFrame.text("hello")); // echo
+        expect.add(new CloseInfo(StatusCode.PROTOCOL).asFrame());
+
+        Fuzzer fuzzer = new Fuzzer(this);
+        try
+        {
+            fuzzer.connect();
+            fuzzer.setSendMode(Fuzzer.SendMode.BULK);
+            fuzzer.send(send);
+            fuzzer.expect(expect);
+        }
+        finally
+        {
+            fuzzer.close();
+        }
+    }
+}

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/ab/TestABCase5.java

@@ -18,7 +18,6 @@ package org.eclipse.jetty.websocket.server.ab;
 import static org.hamcrest.Matchers.*;
 
 import java.nio.ByteBuffer;
-import java.util.Queue;
 import java.util.concurrent.TimeUnit;
 
 import org.eclipse.jetty.io.ByteBufferPool;
@@ -34,6 +33,7 @@ import org.eclipse.jetty.websocket.server.ByteBufferAssert;
 import org.eclipse.jetty.websocket.server.SimpleServletServer;
 import org.eclipse.jetty.websocket.server.blockhead.BlockheadClient;
 import org.eclipse.jetty.websocket.server.examples.MyEchoServlet;
+import org.eclipse.jetty.websocket.server.helper.IncomingFramesCapture;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -85,7 +85,7 @@ public class TestABCase5
             String fragment1 = "fragment1";
 
             // Intentionally bad PING (spec says control frames must be FIN==true)
-            buf.put((byte)(NOFIN | OpCode.PING.getCode()));
+            buf.put((byte)(NOFIN | OpCode.PING));
 
             byte b = 0x00; // no masking
             b |= fragment1.length() & 0x7F;
@@ -100,7 +100,7 @@ public class TestABCase5
 
             String fragment2 = "fragment2";
 
-            buf2.put((byte)(FIN | OpCode.PING.getCode()));
+            buf2.put((byte)(FIN | OpCode.PING));
             b = 0x00; // no masking
             b |= fragment2.length() & 0x7F;
             buf2.put(b);
@@ -110,8 +110,8 @@ public class TestABCase5
             client.writeRaw(buf2);
 
             // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().get(0);
 
             Assert.assertThat("frame should be close frame",frame.getOpCode(),is(OpCode.CLOSE));
 
@@ -144,8 +144,8 @@ public class TestABCase5
             client.writeRaw(buf2);
 
             // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().pop();
 
             Assert.assertThat("frame should be close frame",frame.getOpCode(),is(OpCode.CLOSE));
 
@@ -172,7 +172,7 @@ public class TestABCase5
 
             String fragment1 = "fragment1";
 
-            buf.put((byte)(NOFIN | OpCode.PONG.getCode()));
+            buf.put((byte)(NOFIN | OpCode.PONG));
 
             byte b = 0x00; // no masking
             b |= fragment1.length() & 0x7F;
@@ -187,7 +187,7 @@ public class TestABCase5
 
             String fragment2 = "fragment2";
 
-            buf2.put((byte)(FIN | OpCode.CONTINUATION.getCode()));
+            buf2.put((byte)(FIN | OpCode.CONTINUATION));
             b = 0x00; // no masking
             b |= fragment2.length() & 0x7F;
             buf2.put(b);
@@ -197,8 +197,8 @@ public class TestABCase5
             client.writeRaw(buf2);
 
             // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().pop();
 
             Assert.assertThat("frame should be close frame",frame.getOpCode(),is(OpCode.CLOSE));
 
@@ -231,8 +231,8 @@ public class TestABCase5
             client.writeRaw(buf2);
 
             // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().pop();
 
             Assert.assertThat("frame should be close frame",frame.getOpCode(),is(OpCode.CLOSE));
             Assert.assertThat("CloseFrame.status code",new CloseInfo(frame).getStatusCode(),is(1002));
@@ -258,7 +258,7 @@ public class TestABCase5
 
             String fragment1 = "fragment1";
 
-            buf.put((byte)(NOFIN | OpCode.TEXT.getCode()));
+            buf.put((byte)(NOFIN | OpCode.TEXT));
 
             byte b = 0x00; // no masking
             b |= fragment1.length() & 0x7F;
@@ -273,7 +273,7 @@ public class TestABCase5
 
             String fragment2 = "fragment2";
 
-            buf2.put((byte)(FIN | OpCode.CONTINUATION.getCode()));
+            buf2.put((byte)(FIN | OpCode.CONTINUATION));
             b = 0x00; // no masking
             b |= fragment2.length() & 0x7F;
             buf2.put(b);
@@ -283,8 +283,8 @@ public class TestABCase5
             client.writeRaw(buf2);
 
             // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().pop();
 
             Assert.assertThat("frame should be text frame",frame.getOpCode(),is(OpCode.TEXT));
 
@@ -313,7 +313,7 @@ public class TestABCase5
 
             String fragment1 = "fragment1";
 
-            buf.put((byte)(NOFIN | OpCode.TEXT.getCode()));
+            buf.put((byte)(NOFIN | OpCode.TEXT));
 
             byte b = 0x00; // no masking
             b |= fragment1.length() & 0x7F;
@@ -330,7 +330,7 @@ public class TestABCase5
 
             String pingPayload = "ping payload";
 
-            pingBuf.put((byte)(FIN | OpCode.PING.getCode()));
+            pingBuf.put((byte)(FIN | OpCode.PING));
 
             b = 0x00; // no masking
             b |= pingPayload.length() & 0x7F;
@@ -347,7 +347,7 @@ public class TestABCase5
 
             String fragment2 = "fragment2";
 
-            buf2.put((byte)(FIN | OpCode.CONTINUATION.getCode()));
+            buf2.put((byte)(FIN | OpCode.CONTINUATION));
             b = 0x00; // no masking
             b |= fragment2.length() & 0x7F;
             buf2.put(b);
@@ -357,15 +357,15 @@ public class TestABCase5
             client.writeRaw(buf2);
 
             // Should be 2 frames, pong frame followed by combined echo'd text frame
-            Queue<WebSocketFrame> frames = client.readFrames(2,TimeUnit.SECONDS,1);
+            IncomingFramesCapture capture = client.readFrames(2,TimeUnit.SECONDS,1);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().pop();
 
             Assert.assertThat("first frame should be pong frame",frame.getOpCode(),is(OpCode.PONG));
 
             ByteBuffer payload1 = BufferUtil.toBuffer(pingPayload,StringUtil.__UTF8_CHARSET);
 
             ByteBufferAssert.assertEquals("payloads should be equal",payload1,frame.getPayload());
-            frame = frames.remove();
+            frame = capture.getFrames().pop();
 
             Assert.assertThat("second frame should be text frame",frame.getOpCode(),is(OpCode.TEXT));
             Assert.assertThat("TextFrame.payload",frame.getPayloadAsUTF8(),is(fragment1 + fragment2));
@@ -405,15 +405,15 @@ public class TestABCase5
             client.writeRaw(buf3);
 
             // Should be 2 frames, pong frame followed by combined echo'd text frame
-            Queue<WebSocketFrame> frames = client.readFrames(2,TimeUnit.MILLISECONDS,500);
+            IncomingFramesCapture capture = client.readFrames(2,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().pop();
 
             Assert.assertThat("first frame should be pong frame",frame.getOpCode(),is(OpCode.PONG));
 
             ByteBuffer payload1 = BufferUtil.toBuffer(pingPayload,StringUtil.__UTF8_CHARSET);
             ByteBufferAssert.assertEquals("Payload",payload1,frame.getPayload());
 
-            frame = frames.remove();
+            frame = capture.getFrames().pop();
 
             Assert.assertThat("second frame should be text frame",frame.getOpCode(),is(OpCode.TEXT));
 
@@ -445,7 +445,7 @@ public class TestABCase5
 
             // continuation w / FIN
 
-            buf.put((byte)(FIN | OpCode.CONTINUATION.getCode()));
+            buf.put((byte)(FIN | OpCode.CONTINUATION));
 
             byte b = 0x00; // no masking
             b |= fragment1.length() & 0x7F;
@@ -456,8 +456,8 @@ public class TestABCase5
             client.writeRaw(buf);
 
             // Read frame
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame frame = frames.remove();
+            WebSocketFrame frame = capture.getFrames().pop();
 
             Assert.assertThat("frame should be close frame",frame.getOpCode(),is(OpCode.CLOSE));
 

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/ab/TestABCase7_9.java

@@ -21,7 +21,6 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.Queue;
 import java.util.concurrent.TimeUnit;
 
 import org.eclipse.jetty.util.BufferUtil;
@@ -33,6 +32,7 @@ import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
 import org.eclipse.jetty.websocket.server.SimpleServletServer;
 import org.eclipse.jetty.websocket.server.blockhead.BlockheadClient;
 import org.eclipse.jetty.websocket.server.examples.MyEchoServlet;
+import org.eclipse.jetty.websocket.server.helper.IncomingFramesCapture;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -118,7 +118,7 @@ public class TestABCase7_9
             BufferUtil.clearToFill(buf);
 
             // Create Close Frame manually, as we are testing the server's behavior of a bad client.
-            buf.put((byte)(0x80 | OpCode.CLOSE.getCode()));
+            buf.put((byte)(0x80 | OpCode.CLOSE));
             buf.put((byte)(0x80 | 2));
             byte mask[] = new byte[]
             { 0x44, 0x44, 0x44, 0x44 };
@@ -130,8 +130,8 @@ public class TestABCase7_9
             client.writeRaw(buf);
 
             // Read frame (hopefully text frame)
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame closeFrame = frames.remove();
+            WebSocketFrame closeFrame = capture.getFrames().pop();
             Assert.assertThat("CloseFrame.status code",new CloseInfo(closeFrame).getStatusCode(),is(1002));
         }
         finally
@@ -159,7 +159,7 @@ public class TestABCase7_9
             BufferUtil.clearToFill(buf);
 
             // Create Close Frame manually, as we are testing the server's behavior of a bad client.
-            buf.put((byte)(0x80 | OpCode.CLOSE.getCode()));
+            buf.put((byte)(0x80 | OpCode.CLOSE));
             buf.put((byte)(0x80 | (2 + reason.length())));
             byte mask[] = new byte[]
             { 0x44, 0x44, 0x44, 0x44 };
@@ -172,8 +172,8 @@ public class TestABCase7_9
             client.writeRaw(buf);
 
             // Read frame (hopefully text frame)
-            Queue<WebSocketFrame> frames = client.readFrames(1,TimeUnit.MILLISECONDS,500);
+            IncomingFramesCapture capture = client.readFrames(1,TimeUnit.MILLISECONDS,500);
-            WebSocketFrame closeFrame = frames.remove();
+            WebSocketFrame closeFrame = capture.getFrames().pop();
             Assert.assertThat("CloseFrame.status code",new CloseInfo(closeFrame).getStatusCode(),is(1002));
         }
         finally

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/blockhead/BlockheadClient.java

@@ -34,8 +34,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -65,6 +63,7 @@ import org.eclipse.jetty.websocket.protocol.Generator;
 import org.eclipse.jetty.websocket.protocol.OpCode;
 import org.eclipse.jetty.websocket.protocol.Parser;
 import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
+import org.eclipse.jetty.websocket.server.helper.IncomingFramesCapture;
 import org.junit.Assert;
 
 /**
@@ -90,7 +89,7 @@ public class BlockheadClient implements IncomingFrames, OutgoingFrames
     private final WebSocketPolicy policy;
     private final Generator generator;
     private final Parser parser;
-    private final LinkedBlockingDeque<WebSocketFrame> incomingFrameQueue;
+    private final IncomingFramesCapture incomingFrameQueue;
     private final WebSocketExtensionRegistry extensionRegistry;
 
     private Socket socket;
@@ -123,7 +122,7 @@ public class BlockheadClient implements IncomingFrames, OutgoingFrames
         parser = new Parser(policy);
         parseCount = new AtomicInteger(0);
 
-        incomingFrameQueue = new LinkedBlockingDeque<>();
+        incomingFrameQueue = new IncomingFramesCapture();
 
         extensionRegistry = new WebSocketExtensionRegistry(policy,bufferPool);
     }
@@ -175,6 +174,8 @@ public class BlockheadClient implements IncomingFrames, OutgoingFrames
         LOG.debug("disconnect");
         IO.close(in);
         IO.close(out);
+        if (socket != null)
+        {
             try
             {
                 socket.close();
@@ -184,6 +185,7 @@ public class BlockheadClient implements IncomingFrames, OutgoingFrames
                 /* ignore */
             }
         }
+    }
 
     public String expectUpgradeResponse() throws IOException
     {
@@ -249,6 +251,11 @@ public class BlockheadClient implements IncomingFrames, OutgoingFrames
         return respHeader;
     }
 
+    public void flush() throws IOException
+    {
+        out.flush();
+    }
+
     public List<String> getExtensions()
     {
         return extensions;
@@ -298,7 +305,7 @@ public class BlockheadClient implements IncomingFrames, OutgoingFrames
     @Override
     public void incoming(WebSocketException e)
     {
-        LOG.warn(e);
+        incomingFrameQueue.incoming(e);
     }
 
     @Override
@@ -310,11 +317,13 @@ public class BlockheadClient implements IncomingFrames, OutgoingFrames
         {
             LOG.info("Client parsed {} frames",count);
         }
-        WebSocketFrame copy = new WebSocketFrame(frame); // make a copy
+        WebSocketFrame copy = new WebSocketFrame(frame);
-        if (!incomingFrameQueue.offerLast(copy))
+        incomingFrameQueue.incoming(copy);
-        {
-            throw new RuntimeException("Unable to queue incoming frame: " + copy);
     }
+
+    public boolean isConnected()
+    {
+        return (socket != null) && (socket.isConnected());
     }
 
     public void lookFor(String string) throws IOException
@@ -375,7 +384,7 @@ public class BlockheadClient implements IncomingFrames, OutgoingFrames
         return len;
     }
 
-    public Queue<WebSocketFrame> readFrames(int expectedCount, TimeUnit timeoutUnit, int timeoutDuration) throws IOException, TimeoutException
+    public IncomingFramesCapture readFrames(int expectedCount, TimeUnit timeoutUnit, int timeoutDuration) throws IOException, TimeoutException
     {
         LOG.debug("Read: waiting for {} frame(s) from server",expectedCount);
         int startCount = incomingFrameQueue.size();
@@ -410,7 +419,8 @@ public class BlockheadClient implements IncomingFrames, OutgoingFrames
                 }
                 if (!debug && (System.currentTimeMillis() > expireOn))
                 {
-                    throw new TimeoutException(String.format("Timeout reading all [%d] expected frames. (managed to read [%d] frames)",expectedCount,
+                    incomingFrameQueue.dump();
+                    throw new TimeoutException(String.format("Timeout reading all %d expected frames. (managed to only read %d frame(s))",expectedCount,
                             incomingFrameQueue.size()));
                 }
             }
@@ -545,4 +555,24 @@ public class BlockheadClient implements IncomingFrames, OutgoingFrames
         LOG.debug("write((String)[{}]){}{})",str.length(),'\n',str);
         out.write(StringUtil.getBytes(str,StringUtil.__ISO_8859_1));
     }
+
+    public void writeRawSlowly(ByteBuffer buf, int segmentSize) throws IOException
+    {
+        int origLimit = buf.limit();
+        int limit = buf.limit();
+        int len;
+        int pos = buf.position();
+        int overallLeft = buf.remaining();
+        while (overallLeft > 0)
+        {
+            buf.position(pos);
+            limit = Math.min(origLimit,pos + segmentSize);
+            buf.limit(limit);
+            len = buf.remaining();
+            overallLeft -= len;
+            pos += len;
+            writeRaw(buf);
+            flush();
+        }
+    }
 }

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/examples/echo/EchoFragmentSocket.java

@@ -25,6 +25,7 @@ import org.eclipse.jetty.websocket.annotations.OnWebSocketFrame;
 import org.eclipse.jetty.websocket.annotations.WebSocket;
 import org.eclipse.jetty.websocket.api.WebSocketConnection;
 import org.eclipse.jetty.websocket.protocol.Frame;
+import org.eclipse.jetty.websocket.protocol.OpCode;
 
 /**
  * Echo back the incoming text or binary as 2 frames of (roughly) equal size.
@@ -35,7 +36,7 @@ public class EchoFragmentSocket
     @OnWebSocketFrame
     public void onFrame(WebSocketConnection conn, Frame frame)
     {
-        if (!frame.getOpCode().isDataFrame())
+        if (!OpCode.isDataFrame(frame.getOpCode()))
         {
             return;
         }
@@ -55,11 +56,11 @@ public class EchoFragmentSocket
         {
             switch (frame.getOpCode())
             {
-                case BINARY:
+                case OpCode.BINARY:
                     conn.write(null,nop,buf1);
                     conn.write(null,nop,buf2);
                     break;
-                case TEXT:
+                case OpCode.TEXT:
                     // NOTE: This impl is not smart enough to split on a UTF8 boundary
                     conn.write(null,nop,BufferUtil.toUTF8String(buf1));
                     conn.write(null,nop,BufferUtil.toUTF8String(buf2));

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/helper/IncomingFramesCapture.java

@@ -17,35 +17,46 @@ package org.eclipse.jetty.websocket.server.helper;
 
 import static org.hamcrest.Matchers.*;
 
-import java.util.ArrayList;
+import java.util.LinkedList;
-import java.util.List;
 
+import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 import org.eclipse.jetty.websocket.api.WebSocketException;
 import org.eclipse.jetty.websocket.io.IncomingFrames;
+import org.eclipse.jetty.websocket.protocol.OpCode;
 import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
 import org.junit.Assert;
 
-public class FrameParseCapture implements IncomingFrames
+public class IncomingFramesCapture implements IncomingFrames
 {
-    private static final Logger LOG = Log.getLogger(FrameParseCapture.class);
+    private static final Logger LOG = Log.getLogger(IncomingFramesCapture.class);
-    private List<WebSocketFrame> frames = new ArrayList<>();
+    private LinkedList<WebSocketFrame> frames = new LinkedList<>();
-    private List<WebSocketException> errors = new ArrayList<>();
+    private LinkedList<WebSocketException> errors = new LinkedList<>();
+
+    public void assertErrorCount(int expectedCount)
+    {
+        Assert.assertThat("Captured error count",errors.size(),is(expectedCount));
+    }
+
+    public void assertFrameCount(int expectedCount)
+    {
+        Assert.assertThat("Captured frame count",frames.size(),is(expectedCount));
+    }
 
     public void assertHasErrors(Class<? extends WebSocketException> errorType, int expectedCount)
     {
         Assert.assertThat(errorType.getSimpleName(),getErrorCount(errorType),is(expectedCount));
     }
 
-    public void assertHasFrame(Class<? extends WebSocketFrame> frameType)
+    public void assertHasFrame(byte op)
     {
-        Assert.assertThat(frameType.getSimpleName(),getFrameCount(frameType),greaterThanOrEqualTo(1));
+        Assert.assertThat(OpCode.name(op),getFrameCount(op),greaterThanOrEqualTo(1));
     }
 
-    public void assertHasFrame(Class<? extends WebSocketFrame> frameType, int expectedCount)
+    public void assertHasFrame(byte op, int expectedCount)
     {
-        Assert.assertThat(frameType.getSimpleName(),getFrameCount(frameType),is(expectedCount));
+        Assert.assertThat(OpCode.name(op),getFrameCount(op),is(expectedCount));
     }
 
     public void assertHasNoFrames()
@@ -58,6 +69,17 @@ public class FrameParseCapture implements IncomingFrames
         Assert.assertThat("Has no errors",errors.size(),is(0));
     }
 
+    public void dump()
+    {
+        System.err.printf("Captured %d incoming frames%n",frames.size());
+        for (int i = 0; i < frames.size(); i++)
+        {
+            WebSocketFrame frame = frames.get(i);
+            System.err.printf("[%3d] %s%n",i,frame);
+            System.err.printf("          %s%n",BufferUtil.toDetailString(frame.getPayload()));
+        }
+    }
+
     public int getErrorCount(Class<? extends WebSocketException> errorType)
     {
         int count = 0;
@@ -71,17 +93,17 @@ public class FrameParseCapture implements IncomingFrames
         return count;
     }
 
-    public List<WebSocketException> getErrors()
+    public LinkedList<WebSocketException> getErrors()
     {
         return errors;
     }
 
-    public int getFrameCount(Class<? extends WebSocketFrame> frameType)
+    public int getFrameCount(byte op)
     {
         int count = 0;
         for (WebSocketFrame frame : frames)
         {
-            if (frameType.isInstance(frame))
+            if (frame.getOpCode() == op)
             {
                 count++;
             }
@@ -89,7 +111,7 @@ public class FrameParseCapture implements IncomingFrames
         return count;
     }
 
-    public List<WebSocketFrame> getFrames()
+    public LinkedList<WebSocketFrame> getFrames()
     {
         return frames;
     }
@@ -97,7 +119,7 @@ public class FrameParseCapture implements IncomingFrames
     @Override
     public void incoming(WebSocketException e)
     {
-        LOG.warn(e);
+        LOG.debug(e);
         errors.add(e);
     }
 
@@ -106,4 +128,9 @@ public class FrameParseCapture implements IncomingFrames
     {
         frames.add(frame);
     }
+
+    public int size()
+    {
+        return frames.size();
+    }
 }

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/helper/MessageSender.java

@@ -1,77 +0,0 @@
-// ========================================================================
-// Copyright 2011-2012 Mort Bay Consulting Pty. Ltd.
-// ------------------------------------------------------------------------
-// All rights reserved. This program and the accompanying materials
-// are made available under the terms of the Eclipse Public License v1.0
-// and Apache License v2.0 which accompanies this distribution.
-//
-//     The Eclipse Public License is available at
-//     http://www.eclipse.org/legal/epl-v10.html
-//
-//     The Apache License v2.0 is available at
-//     http://www.opensource.org/licenses/apache2.0.php
-//
-// You may elect to redistribute this code under either of these licenses.
-//========================================================================
-package org.eclipse.jetty.websocket.server.helper;
-
-import java.io.IOException;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import org.eclipse.jetty.websocket.api.StatusCode;
-import org.eclipse.jetty.websocket.api.WebSocketAdapter;
-import org.eclipse.jetty.websocket.api.WebSocketConnection;
-
-public class MessageSender extends WebSocketAdapter
-{
-    private CountDownLatch connectLatch = new CountDownLatch(1);
-    private int closeCode = -1;
-    private String closeMessage = null;
-
-    public void awaitConnect() throws InterruptedException
-    {
-        connectLatch.await(1,TimeUnit.SECONDS);
-    }
-
-    public void close()
-    {
-        try
-        {
-            getConnection().close(StatusCode.NORMAL,null);
-        }
-        catch (IOException e)
-        {
-            e.printStackTrace();
-        }
-    }
-
-    public int getCloseCode()
-    {
-        return closeCode;
-    }
-
-    public String getCloseMessage()
-    {
-        return closeMessage;
-    }
-
-    @Override
-    public void onWebSocketClose(int statusCode, String reason)
-    {
-        this.closeCode = statusCode;
-        this.closeMessage = reason;
-    }
-
-    @Override
-    public void onWebSocketConnect(WebSocketConnection connection)
-    {
-        super.onWebSocketConnect(connection);
-        connectLatch.countDown();
-    }
-
-    public void sendMessage(String format, Object... args) throws IOException
-    {
-        getBlockingConnection().write(String.format(format,args));
-    }
-}

jetty-websocket/websocket-server/src/test/java/org/eclipse/jetty/websocket/server/helper/OutgoingFramesCapture.java

@@ -0,0 +1,99 @@
+// ========================================================================
+// Copyright 2011-2012 Mort Bay Consulting Pty. Ltd.
+// ------------------------------------------------------------------------
+// All rights reserved. This program and the accompanying materials
+// are made available under the terms of the Eclipse Public License v1.0
+// and Apache License v2.0 which accompanies this distribution.
+//
+//     The Eclipse Public License is available at
+//     http://www.eclipse.org/legal/epl-v10.html
+//
+//     The Apache License v2.0 is available at
+//     http://www.opensource.org/licenses/apache2.0.php
+//
+// You may elect to redistribute this code under either of these licenses.
+//========================================================================
+package org.eclipse.jetty.websocket.server.helper;
+
+import static org.hamcrest.Matchers.*;
+
+import java.util.LinkedList;
+
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.websocket.io.OutgoingFrames;
+import org.eclipse.jetty.websocket.protocol.OpCode;
+import org.eclipse.jetty.websocket.protocol.WebSocketFrame;
+import org.junit.Assert;
+
+public class OutgoingFramesCapture implements OutgoingFrames
+{
+    public static class Write<C>
+    {
+        public C context;
+        public Callback<C> callback;
+        public WebSocketFrame frame;
+    }
+
+    private LinkedList<Write<?>> writes = new LinkedList<>();
+
+    public void assertFrameCount(int expectedCount)
+    {
+        Assert.assertThat("Captured frame count",writes.size(),is(expectedCount));
+    }
+
+    public void assertHasFrame(byte op)
+    {
+        Assert.assertThat(OpCode.name(op),getFrameCount(op),greaterThanOrEqualTo(1));
+    }
+
+    public void assertHasFrame(byte op, int expectedCount)
+    {
+        Assert.assertThat(OpCode.name(op),getFrameCount(op),is(expectedCount));
+    }
+
+    public void assertHasNoFrames()
+    {
+        Assert.assertThat("Has no frames",writes.size(),is(0));
+    }
+
+    public void dump()
+    {
+        System.out.printf("Captured %d outgoing writes%n",writes.size());
+        for (int i = 0; i < writes.size(); i++)
+        {
+            Write<?> write = writes.get(i);
+            System.out.printf("[%3d] %s | %s | %s%n",i,write.context,write.callback,write.frame);
+            System.out.printf("          %s%n",BufferUtil.toDetailString(write.frame.getPayload()));
+        }
+    }
+
+    public int getFrameCount(byte op)
+    {
+        int count = 0;
+        for (Write<?> write : writes)
+        {
+            WebSocketFrame frame = write.frame;
+            if (frame.getOpCode() == op)
+            {
+                count++;
+            }
+        }
+        return count;
+    }
+
+    public LinkedList<Write<?>> getWrites()
+    {
+        return writes;
+    }
+
+    @Override
+    public <C> void output(C context, Callback<C> callback, WebSocketFrame frame)
+    {
+        Write<C> write = new Write<C>();
+        write.context = context;
+        write.callback = callback;
+        write.frame = frame;
+        writes.add(write);
+    }
+}

jetty-websocket/websocket-server/src/test/resources/jetty-logging.properties

@@ -1,11 +1,17 @@
 org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog
-org.eclipse.jetty.io.LEVEL=WARN
+org.eclipse.jetty.LEVEL=WARN
 org.eclipse.jetty.server.LEVEL=WARN
+# org.eclipse.jetty.websocket.LEVEL=WARN
 org.eclipse.jetty.websocket.server.helper.RFCSocket.LEVEL=OFF
+# See the read/write traffic
+# org.eclipse.jetty.websocket.io.Frames.LEVEL=DEBUG
+# org.eclipse.jetty.websocket.io.LEVEL=DEBUG
+# org.eclipse.jetty.websocket.io.WebSocketAsyncConnection.LEVEL=DEBUG
 # org.eclipse.jetty.util.thread.QueuedThreadPool.LEVEL=DEBUG
 # org.eclipse.jetty.io.SelectorManager.LEVEL=INFO
-# org.eclipse.jetty.websocket.LEVEL=DEBUG
+org.eclipse.jetty.websocket.LEVEL=DEBUG
+# org.eclipse.jetty.websocket.driver.WebSocketEventDriver.LEVEL=DEBUG
 # org.eclipse.jetty.websocket.extensions.LEVEL=DEBUG
 # org.eclipse.jetty.websocket.protocol.Generator.LEVEL=INFO
-# org.eclipse.jetty.websocket.protocol.Parser.LEVEL=INFO
+# org.eclipse.jetty.websocket.protocol.Parser.LEVEL=DEBUG
 # org.eclipse.jetty.websocket.server.blockhead.LEVEL=INFO