Merge remote-tracking branch 'origin/jetty-9.4.x'

Conflicts:
	jetty-webapp/src/main/java/org/eclipse/jetty/webapp/WebAppContext.java
	jetty-webapp/src/test/java/org/eclipse/jetty/webapp/WebAppClassLoaderTest.java

jetty-client/src/main/java/org/eclipse/jetty/client/HttpRequest.java

@@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiFunction;
+import java.util.function.Supplier;
 
 import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.ContentResponse;
@@ -83,6 +84,7 @@ public class HttpRequest implements Request
     private Map<String, Object> attributes;
     private List<RequestListener> requestListeners;
     private BiFunction<Request, Request, Response.CompleteListener> pushListener;
+    private Supplier<HttpFields> trailers;
 
     protected HttpRequest(HttpClient client, HttpConversation conversation, URI uri)
     {
@@ -589,6 +591,12 @@ public class HttpRequest implements Request
         return this;
     }
 
+    public HttpRequest trailers(Supplier<HttpFields> trailers)
+    {
+        this.trailers = trailers;
+        return this;
+    }
+
     @Override
     public ContentProvider getContent()
     {
@@ -725,6 +733,11 @@ public class HttpRequest implements Request
         return pushListener;
     }
 
+    public Supplier<HttpFields> getTrailers()
+    {
+        return trailers;
+    }
+
     @Override
     public boolean abort(Throwable cause)
     {

jetty-client/src/main/java/org/eclipse/jetty/client/HttpResponse.java

@@ -23,6 +23,7 @@ import java.util.List;
 
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpVersion;
 
@@ -34,6 +35,7 @@ public class HttpResponse implements Response
     private HttpVersion version;
     private int status;
     private String reason;
+    private HttpFields trailers;
 
     public HttpResponse(Request request, List<ResponseListener> listeners)
     {
@@ -97,6 +99,19 @@ public class HttpResponse implements Response
         return result;
     }
 
+    public HttpFields getTrailers()
+    {
+        return trailers;
+    }
+
+    public HttpResponse trailer(HttpField trailer)
+    {
+        if (trailers == null)
+            trailers = new HttpFields();
+        trailers.add(trailer);
+        return this;
+    }
+
     @Override
     public boolean abort(Throwable cause)
     {

jetty-client/src/main/java/org/eclipse/jetty/client/HttpSender.java

@@ -20,10 +20,12 @@ package org.eclipse.jetty.client;
 
 import java.nio.ByteBuffer;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
 
 import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Result;
+import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpHeaderValue;
 import org.eclipse.jetty.util.BufferUtil;
@@ -31,7 +33,6 @@ import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.IteratingCallback;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
-import org.eclipse.jetty.util.thread.Invocable.InvocationType;
 
 /**
  * {@link HttpSender} abstracts the algorithm to send HTTP requests, so that subclasses only implement
@@ -42,9 +43,9 @@ import org.eclipse.jetty.util.thread.Invocable.InvocationType;
  * {@link HttpSender} governs two state machines.
  * <p>
  * The request state machine is updated by {@link HttpSender} as the various steps of sending a request
- * are executed, see <code>RequestState</code>.
+ * are executed, see {@code RequestState}.
  * At any point in time, a user thread may abort the request, which may (if the request has not been
- * completely sent yet) move the request state machine to <code>RequestState#FAILURE</code>.
+ * completely sent yet) move the request state machine to {@code RequestState#FAILURE}.
  * The request state machine guarantees that the request steps are executed (by I/O threads) only if
  * the request has not been failed already.
  * <p>
@@ -64,7 +65,8 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
     private final AtomicReference<SenderState> senderState = new AtomicReference<>(SenderState.IDLE);
     private final Callback commitCallback = new CommitCallback();
     private final IteratingCallback contentCallback = new ContentCallback();
-    private final Callback lastCallback = new LastContentCallback();
+    private final Callback trailersCallback = new TrailersCallback();
+    private final Callback lastCallback = new LastCallback();
     private final HttpChannel channel;
     private HttpContent content;
     private Throwable failure;
@@ -407,7 +409,7 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
     /**
      * Implementations should send the content at the {@link HttpContent} cursor position over the wire.
      * <p>
-     * The {@link HttpContent} cursor is advanced by {@link HttpSender} at the right time, and if more
+     * The {@link HttpContent} cursor is advanced by HttpSender at the right time, and if more
      * content needs to be sent, this method is invoked again; subclasses need only to send the content
      * at the {@link HttpContent} cursor position.
      * <p>
@@ -422,6 +424,15 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
      */
     protected abstract void sendContent(HttpExchange exchange, HttpContent content, Callback callback);
 
+    /**
+     * Implementations should send the HTTP trailers and notify the given {@code callback} of the
+     * result of this operation.
+     *
+     * @param exchange the exchange to send
+     * @param callback the callback to notify
+     */
+    protected abstract void sendTrailers(HttpExchange exchange, Callback callback);
+
     protected void reset()
     {
         HttpContent content = this.content;
@@ -674,13 +685,6 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
 
     private class CommitCallback implements Callback
     {
-
-        @Override
-        public InvocationType getInvocationType()
-        {
-            return content.getInvocationType();
-        }
-
         @Override
         public void succeeded()
         {
@@ -721,10 +725,20 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
             if (content == null)
                 return;
 
-            if (!content.hasContent())
+            HttpRequest request = exchange.getRequest();
+            Supplier<HttpFields> trailers = request.getTrailers();
+            boolean hasContent = content.hasContent();
+            if (!hasContent)
             {
-                // No content to send, we are done.
+                if (trailers == null)
-                someToSuccess(exchange);
+                {
+                    // No trailers or content to send, we are done.
+                    someToSuccess(exchange);
+                }
+                else
+                {
+                    sendTrailers(exchange, lastCallback);
+                }
             }
             else
             {
@@ -825,7 +839,9 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
 
                 if (lastContent)
                 {
-                    sendContent(exchange, content, lastCallback);
+                    HttpRequest request = exchange.getRequest();
+                    Supplier<HttpFields> trailers = request.getTrailers();
+                    sendContent(exchange, content, trailers == null ? lastCallback : trailersCallback);
                     return Action.IDLE;
                 }
 
@@ -884,19 +900,35 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         @Override
         protected void onCompleteSuccess()
         {
-            // Nothing to do, since we always return false from process().
+            // Nothing to do, since we always return IDLE from process().
-            // Termination is obtained via LastContentCallback.
+            // Termination is obtained via LastCallback.
         }
     }
 
-    private class LastContentCallback implements Callback
+    private class TrailersCallback implements Callback
     {
         @Override
-        public InvocationType getInvocationType()
+        public void succeeded()
         {
-            return content.getInvocationType();
+            HttpExchange exchange = getHttpExchange();
+            if (exchange == null)
+                return;
+            sendTrailers(exchange, lastCallback);
         }
 
+        @Override
+        public void failed(Throwable x)
+        {
+            HttpContent content = HttpSender.this.content;
+            if (content == null)
+                return;
+            content.failed(x);
+            anyToFailure(x);
+        }
+    }
+
+    private class LastCallback implements Callback
+    {
         @Override
         public void succeeded()
         {

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpReceiverOverHTTP.java

@@ -276,7 +276,17 @@ public class HttpReceiverOverHTTP extends HttpReceiver implements HttpParser.Res
     {
         return false;
     }
-    
+
+    @Override
+    public void parsedTrailer(HttpField trailer)
+    {
+        HttpExchange exchange = getHttpExchange();
+        if (exchange == null)
+            return;
+
+        exchange.getResponse().trailer(trailer);
+    }
+
     @Override
     public boolean messageComplete()
     {

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpSenderOverHTTP.java

@@ -23,26 +23,29 @@ import java.nio.ByteBuffer;
 import org.eclipse.jetty.client.HttpClient;
 import org.eclipse.jetty.client.HttpContent;
 import org.eclipse.jetty.client.HttpExchange;
+import org.eclipse.jetty.client.HttpRequest;
 import org.eclipse.jetty.client.HttpRequestException;
 import org.eclipse.jetty.client.HttpSender;
 import org.eclipse.jetty.client.api.ContentProvider;
-import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.http.HttpGenerator;
 import org.eclipse.jetty.http.HttpURI;
 import org.eclipse.jetty.http.MetaData;
 import org.eclipse.jetty.io.ByteBufferPool;
 import org.eclipse.jetty.io.EndPoint;
+import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.IteratingCallback;
 
 public class HttpSenderOverHTTP extends HttpSender
 {
     private final HttpGenerator generator = new HttpGenerator();
+    private final HttpClient httpClient;
     private boolean shutdown;
 
     public HttpSenderOverHTTP(HttpChannelOverHTTP channel)
     {
         super(channel);
+        httpClient = channel.getHttpDestination().getHttpClient();
     }
 
     @Override
@@ -71,8 +74,7 @@ public class HttpSenderOverHTTP extends HttpSender
     {
         try
         {
-            HttpClient client = getHttpChannel().getHttpDestination().getHttpClient();
+            ByteBufferPool bufferPool = httpClient.getByteBufferPool();
-            ByteBufferPool bufferPool = client.getByteBufferPool();
             ByteBuffer chunk = null;
             while (true)
             {
@@ -90,6 +92,11 @@ public class HttpSenderOverHTTP extends HttpSender
                         chunk = bufferPool.acquire(HttpGenerator.CHUNK_SIZE, false);
                         break;
                     }
+                    case NEED_CHUNK_TRAILER:
+                    {
+                        callback.succeeded();
+                        return;
+                    }
                     case FLUSH:
                     {
                         EndPoint endPoint = getHttpChannel().getHttpConnection().getEndPoint();
@@ -131,6 +138,21 @@ public class HttpSenderOverHTTP extends HttpSender
         }
     }
 
+    @Override
+    protected void sendTrailers(HttpExchange exchange, Callback callback)
+    {
+        try
+        {
+            new TrailersCallback(callback).iterate();
+        }
+        catch (Throwable x)
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug(x);
+            callback.failed(x);
+        }
+    }
+
     @Override
     protected void reset()
     {
@@ -181,7 +203,7 @@ public class HttpSenderOverHTTP extends HttpSender
             this.exchange = exchange;
             this.callback = callback;
 
-            Request request = exchange.getRequest();
+            HttpRequest request = exchange.getRequest();
             ContentProvider requestContent = request.getContent();
             long contentLength = requestContent == null ? -1 : requestContent.getLength();
             String path = request.getPath();
@@ -189,6 +211,7 @@ public class HttpSenderOverHTTP extends HttpSender
             if (query != null)
                 path += "?" + query;
             metaData = new MetaData.Request(request.getMethod(), new HttpURI(path), request.getVersion(), request.getHeaders(), contentLength);
+            metaData.setTrailerSupplier(request.getTrailers());
 
             if (!expects100Continue(request))
             {
@@ -201,9 +224,6 @@ public class HttpSenderOverHTTP extends HttpSender
         @Override
         protected Action process() throws Exception
         {
-            HttpClient client = getHttpChannel().getHttpDestination().getHttpClient();
-            ByteBufferPool bufferPool = client.getByteBufferPool();
-
             while (true)
             {
                 HttpGenerator.Result result = generator.generateRequest(metaData, headerBuffer, chunkBuffer, contentBuffer, lastContent);
@@ -217,31 +237,28 @@ public class HttpSenderOverHTTP extends HttpSender
                 {
                     case NEED_HEADER:
                     {
-                        headerBuffer = bufferPool.acquire(client.getRequestBufferSize(), false);
+                        headerBuffer = httpClient.getByteBufferPool().acquire(httpClient.getRequestBufferSize(), false);
                         break;
                     }
                     case NEED_CHUNK:
                     {
-                        chunkBuffer = bufferPool.acquire(HttpGenerator.CHUNK_SIZE, false);
+                        chunkBuffer = httpClient.getByteBufferPool().acquire(HttpGenerator.CHUNK_SIZE, false);
                         break;
                     }
+                    case NEED_CHUNK_TRAILER:
+                    {
+                        return Action.SUCCEEDED;
+                    }
                     case FLUSH:
                     {
                         EndPoint endPoint = getHttpChannel().getHttpConnection().getEndPoint();
+                        if (headerBuffer == null)
+                            headerBuffer = BufferUtil.EMPTY_BUFFER;
                         if (chunkBuffer == null)
-                        {
+                            chunkBuffer = BufferUtil.EMPTY_BUFFER;
-                            if (contentBuffer == null)
+                        if (contentBuffer == null)
-                                endPoint.write(this, headerBuffer);
+                            contentBuffer = BufferUtil.EMPTY_BUFFER;
-                            else
+                        endPoint.write(this, headerBuffer, chunkBuffer, contentBuffer);
-                                endPoint.write(this, headerBuffer, contentBuffer);
-                        }
-                        else
-                        {
-                            if (contentBuffer == null)
-                                endPoint.write(this, headerBuffer, chunkBuffer);
-                            else
-                                endPoint.write(this, headerBuffer, chunkBuffer, contentBuffer);
-                        }
                         generated = true;
                         return Action.SCHEDULED;
                     }
@@ -296,13 +313,91 @@ public class HttpSenderOverHTTP extends HttpSender
 
         private void release()
         {
-            HttpClient client = getHttpChannel().getHttpDestination().getHttpClient();
+            ByteBufferPool bufferPool = httpClient.getByteBufferPool();
-            ByteBufferPool bufferPool = client.getByteBufferPool();
+            if (headerBuffer != BufferUtil.EMPTY_BUFFER)
-            bufferPool.release(headerBuffer);
+                bufferPool.release(headerBuffer);
             headerBuffer = null;
-            if (chunkBuffer != null)
+            if (chunkBuffer != BufferUtil.EMPTY_BUFFER)
                 bufferPool.release(chunkBuffer);
             chunkBuffer = null;
+            contentBuffer = null;
+        }
+    }
+
+    private class TrailersCallback extends IteratingCallback
+    {
+        private final Callback callback;
+        private ByteBuffer chunkBuffer;
+
+        public TrailersCallback(Callback callback)
+        {
+            this.callback = callback;
+        }
+
+        @Override
+        protected Action process() throws Throwable
+        {
+            while (true)
+            {
+                HttpGenerator.Result result = generator.generateRequest(null, null, chunkBuffer, null, true);
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Generated trailers {}/{}", result, generator);
+                switch (result)
+                {
+                    case NEED_CHUNK_TRAILER:
+                    {
+                        chunkBuffer = httpClient.getByteBufferPool().acquire(httpClient.getRequestBufferSize(), false);
+                        break;
+                    }
+                    case FLUSH:
+                    {
+                        EndPoint endPoint = getHttpChannel().getHttpConnection().getEndPoint();
+                        endPoint.write(this, chunkBuffer);
+                        return Action.SCHEDULED;
+                    }
+                    case SHUTDOWN_OUT:
+                    {
+                        shutdownOutput();
+                        return Action.SUCCEEDED;
+                    }
+                    case DONE:
+                    {
+                        return Action.SUCCEEDED;
+                    }
+                    default:
+                    {
+                        throw new IllegalStateException(result.toString());
+                    }
+                }
+            }
+        }
+
+        @Override
+        public void succeeded()
+        {
+            release();
+            super.succeeded();
+        }
+
+        @Override
+        public void failed(Throwable x)
+        {
+            release();
+            callback.failed(x);
+            super.failed(x);
+        }
+
+        @Override
+        protected void onCompleteSuccess()
+        {
+            super.onCompleteSuccess();
+            callback.succeeded();
+        }
+
+        private void release()
+        {
+            httpClient.getByteBufferPool().release(chunkBuffer);
+            chunkBuffer = null;
         }
     }
 

jetty-client/src/test/java/org/eclipse/jetty/client/AbstractHttpClientServerTest.java

@@ -71,8 +71,6 @@ public abstract class AbstractHttpClientServerTest
             sslContextFactory.setEndpointIdentificationAlgorithm("");
             sslContextFactory.setKeyStorePath("src/test/resources/keystore.jks");
             sslContextFactory.setKeyStorePassword("storepwd");
-            sslContextFactory.setTrustStorePath("src/test/resources/truststore.jks");
-            sslContextFactory.setTrustStorePassword("storepwd");
         }
 
         if (server == null)

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTLSTest.java

@@ -72,8 +72,6 @@ public class HttpClientTLSTest
         sslContextFactory.setEndpointIdentificationAlgorithm("");
         sslContextFactory.setKeyStorePath("src/test/resources/keystore.jks");
         sslContextFactory.setKeyStorePassword("storepwd");
-        sslContextFactory.setTrustStorePath("src/test/resources/truststore.jks");
-        sslContextFactory.setTrustStorePassword("storepwd");
         return sslContextFactory;
     }
 

jetty-client/src/test/java/org/eclipse/jetty/client/TLSServerConnectionCloseTest.java

@@ -69,8 +69,6 @@ public class TLSServerConnectionCloseTest
         sslContextFactory.setEndpointIdentificationAlgorithm("");
         sslContextFactory.setKeyStorePath("src/test/resources/keystore.jks");
         sslContextFactory.setKeyStorePassword("storepwd");
-        sslContextFactory.setTrustStorePath("src/test/resources/truststore.jks");
-        sslContextFactory.setTrustStorePassword("storepwd");
 
         QueuedThreadPool clientThreads = new QueuedThreadPool();
         clientThreads.setName("client");

jetty-client/src/test/java/org/eclipse/jetty/client/ssl/NeedWantClientAuthTest.java

@@ -0,0 +1,233 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2017 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.client.ssl;
+
+import java.security.cert.Certificate;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLSession;
+
+import org.eclipse.jetty.client.EmptyServerHandler;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.io.ssl.SslHandshakeListener;
+import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * In order to work, client authentication needs a certificate
+ * signed by a CA that also signed the server certificate.
+ * <p>
+ * For this test, the client certificate is signed with the server
+ * certificate, and the server certificate is self-signed.
+ */
+public class NeedWantClientAuthTest
+{
+    private Server server;
+    private ServerConnector connector;
+    private HttpClient client;
+
+    private void startServer(SslContextFactory sslContextFactory, Handler handler) throws Exception
+    {
+        QueuedThreadPool serverThreads = new QueuedThreadPool();
+        serverThreads.setName("server");
+        server = new Server(serverThreads);
+
+        connector = new ServerConnector(server, sslContextFactory);
+        server.addConnector(connector);
+
+        server.setHandler(handler);
+        server.start();
+    }
+
+    private void startClient(SslContextFactory sslContextFactory) throws Exception
+    {
+        QueuedThreadPool clientThreads = new QueuedThreadPool();
+        clientThreads.setName("client");
+        client = new HttpClient(sslContextFactory);
+        client.setExecutor(clientThreads);
+        client.start();
+    }
+
+    private SslContextFactory createSslContextFactory()
+    {
+        SslContextFactory sslContextFactory = new SslContextFactory();
+        sslContextFactory.setEndpointIdentificationAlgorithm("");
+        sslContextFactory.setKeyStorePath("src/test/resources/keystore.jks");
+        sslContextFactory.setKeyStorePassword("storepwd");
+        return sslContextFactory;
+    }
+
+    @After
+    public void dispose() throws Exception
+    {
+        if (client != null)
+            client.stop();
+        if (server != null)
+            server.stop();
+    }
+
+    @Test
+    public void testWantClientAuthWithoutAuth() throws Exception
+    {
+        SslContextFactory serverSSL = new SslContextFactory();
+        serverSSL.setKeyStorePath("src/test/resources/keystore.jks");
+        serverSSL.setKeyStorePassword("storepwd");
+        serverSSL.setWantClientAuth(true);
+        startServer(serverSSL, new EmptyServerHandler());
+
+        SslContextFactory clientSSL = new SslContextFactory(true);
+        startClient(clientSSL);
+
+        ContentResponse response = client.newRequest("https://localhost:" + connector.getLocalPort())
+                .timeout(5, TimeUnit.SECONDS)
+                .send();
+
+        Assert.assertEquals(HttpStatus.OK_200, response.getStatus());
+    }
+
+    @Test
+    public void testWantClientAuthWithAuth() throws Exception
+    {
+        SslContextFactory serverSSL = new SslContextFactory();
+        serverSSL.setKeyStorePath("src/test/resources/keystore.jks");
+        serverSSL.setKeyStorePassword("storepwd");
+        serverSSL.setWantClientAuth(true);
+        startServer(serverSSL, new EmptyServerHandler());
+        CountDownLatch handshakeLatch = new CountDownLatch(1);
+        connector.addBean(new SslHandshakeListener()
+        {
+            @Override
+            public void handshakeSucceeded(Event event)
+            {
+                try
+                {
+                    SSLSession session = event.getSSLEngine().getSession();
+                    Certificate[] clientCerts = session.getPeerCertificates();
+                    Assert.assertNotNull(clientCerts);
+                    Assert.assertThat(clientCerts.length, Matchers.greaterThan(0));
+                    handshakeLatch.countDown();
+                }
+                catch (Throwable x)
+                {
+                    x.printStackTrace();
+                }
+            }
+        });
+
+        SslContextFactory clientSSL = new SslContextFactory(true);
+        clientSSL.setKeyStorePath("src/test/resources/client_keystore.jks");
+        clientSSL.setKeyStorePassword("storepwd");
+        startClient(clientSSL);
+
+        ContentResponse response = client.newRequest("https://localhost:" + connector.getLocalPort())
+                .timeout(5, TimeUnit.SECONDS)
+                .send();
+
+        Assert.assertEquals(HttpStatus.OK_200, response.getStatus());
+        Assert.assertTrue(handshakeLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testNeedClientAuthWithoutAuth() throws Exception
+    {
+        SslContextFactory serverSSL = new SslContextFactory();
+        serverSSL.setKeyStorePath("src/test/resources/keystore.jks");
+        serverSSL.setKeyStorePassword("storepwd");
+        serverSSL.setNeedClientAuth(true);
+        startServer(serverSSL, new EmptyServerHandler());
+
+        SslContextFactory clientSSL = new SslContextFactory(true);
+        startClient(clientSSL);
+        CountDownLatch handshakeLatch = new CountDownLatch(1);
+        client.addBean(new SslHandshakeListener()
+        {
+            @Override
+            public void handshakeFailed(Event event, Throwable failure)
+            {
+                Assert.assertThat(failure, Matchers.instanceOf(SSLHandshakeException.class));
+                handshakeLatch.countDown();
+            }
+        });
+
+        CountDownLatch latch = new CountDownLatch(1);
+        client.newRequest("https://localhost:" + connector.getLocalPort())
+                .timeout(5, TimeUnit.SECONDS)
+                .send(result ->
+                {
+                    if (result.isFailed())
+                        latch.countDown();
+                });
+
+        Assert.assertTrue(handshakeLatch.await(5, TimeUnit.SECONDS));
+        Assert.assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testNeedClientAuthWithAuth() throws Exception
+    {
+        SslContextFactory serverSSL = new SslContextFactory();
+        serverSSL.setKeyStorePath("src/test/resources/keystore.jks");
+        serverSSL.setKeyStorePassword("storepwd");
+        serverSSL.setNeedClientAuth(true);
+        startServer(serverSSL, new EmptyServerHandler());
+        CountDownLatch handshakeLatch = new CountDownLatch(1);
+        connector.addBean(new SslHandshakeListener()
+        {
+            @Override
+            public void handshakeSucceeded(Event event)
+            {
+                try
+                {
+                    SSLSession session = event.getSSLEngine().getSession();
+                    Certificate[] clientCerts = session.getPeerCertificates();
+                    Assert.assertNotNull(clientCerts);
+                    Assert.assertThat(clientCerts.length, Matchers.greaterThan(0));
+                    handshakeLatch.countDown();
+                }
+                catch (Throwable x)
+                {
+                    x.printStackTrace();
+                }
+            }
+        });
+
+        SslContextFactory clientSSL = new SslContextFactory(true);
+        clientSSL.setKeyStorePath("src/test/resources/client_keystore.jks");
+        clientSSL.setKeyStorePassword("storepwd");
+        startClient(clientSSL);
+
+        ContentResponse response = client.newRequest("https://localhost:" + connector.getLocalPort())
+                .timeout(5, TimeUnit.SECONDS)
+                .send();
+
+        Assert.assertEquals(HttpStatus.OK_200, response.getStatus());
+        Assert.assertTrue(handshakeLatch.await(5, TimeUnit.SECONDS));
+    }
+}

jetty-documentation/src/main/asciidoc/administration/logging/chapter.adoc

@@ -19,7 +19,7 @@
 
 This chapter discusses various options for configuring logging.
 
-include::configuring-jetty-logging.adoc[]
+//include::configuring-jetty-logging.adoc[]
 include::default-logging-with-stderrlog.adoc[]
 include::configuring-jetty-request-logs.adoc[]
 include::configuring-logging-modules.adoc[]

jetty-documentation/src/main/asciidoc/administration/logging/configuring-jetty-request-logs.adoc

@@ -58,6 +58,12 @@ INFO: Base directory was modified
 
 The above command will add a new `requestlog.ini` file to your link:#start-vs-startd[`{$jetty.base}/start.d` directory].
 
+____
+[NOTE]
+By default, request logs are not set to be appended, meaning a the log file is wiped clean upon sever restart.
+You can change this setting by editing the `requestlog.ini` and un-commenting the line that reads `jetty.requestlog.append=true`.
+____
+
 The equivalent code for embedded usages of Jetty is:
 
 [source, java, subs="{sub-order}"]

jetty-documentation/src/main/asciidoc/administration/logging/default-logging-with-stderrlog.adoc

@@ -45,6 +45,13 @@ INFO  : Base directory was modified
 
 The default configuration for logging output will create a file `${jetty.base}/logs/yyyy_mm_dd.stderrout.log` which allows configuration of the output directory by setting the `jetty.logs` property.
 
+____
+[NOTE]
+By default, logs are not set to be appended, meaning a the log file is wiped clean upon sever restart.
+You can change this setting by editing the `console-capture.ini` and un-commenting the line that reads `jetty.console-capture.append=true`.
+____
+
+
 Just enabling the `console-capture` will simply output the values of STDERR and STDOUT to a log file.
 To customize the log further, a module named `logging-jetty` is available to provides a default properties file to configure.
 As with `console-capture`, you activate the `logging-jetty` on the command line.

jetty-documentation/src/main/asciidoc/configuring/security/authentication.adoc

@@ -27,9 +27,9 @@ Authorization::
 
 ==== Configuring an Authentication mechanism
 
-The jetty server supports several standard authentication mechanisms: http://en.wikipedia.org/wiki/Basic_access_authentication[BASIC]; http://en.wikipedia.org/wiki/Digest_authentication[DIGEST]; http://en.wikipedia.org/wiki/Form-based_authentication[FORM]; CLIENT-CERT; and other mechanisms can be plugged in using the extensible http://docs.oracle.com/cd/E19462-01/819-6717/gcszc/index.html[JASPI] or http://en.wikipedia.org/wiki/SPNEGO[SPNEGO] mechanisms.
+Jetty server supports several standard authentication mechanisms: http://en.wikipedia.org/wiki/Basic_access_authentication[BASIC]; http://en.wikipedia.org/wiki/Digest_authentication[DIGEST]; http://en.wikipedia.org/wiki/Form-based_authentication[FORM]; CLIENT-CERT; and other mechanisms can be plugged in using the extensible http://docs.oracle.com/cd/E19462-01/819-6717/gcszc/index.html[JASPI] or http://en.wikipedia.org/wiki/SPNEGO[SPNEGO] mechanisms.
 
-Internally, configuring an authentication mechanism is done by setting an instance of a the link:{JDURL}/org/eclipse/jetty/security/Authenticator.html[Authenticator] interface onto the link:{JDURL}/org/eclipse/jetty/security/SecurityHandler.html[SecurityHandler] of the context, but in most cases it is done by declaring a `<    login-config>` element in the standard web.xml descriptor or via annotations.
+Internally, configuring an authentication mechanism is done by setting an instance of a the link:{JDURL}/org/eclipse/jetty/security/Authenticator.html[Authenticator] interface onto the link:{JDURL}/org/eclipse/jetty/security/SecurityHandler.html[SecurityHandler] of the context, but in most cases it is done by declaring a `<login-config>` element in the standard web.xml descriptor or via annotations.
 
 Below is an example taken from the link:{GITBROWSEURL}/tests/test-webapps/test-jetty-webapp/src/main/webapp/WEB-INF/web.xml?h=release-9[jetty-test-webapp web.xml] that configures BASIC authentication:
 
@@ -103,7 +103,7 @@ When a request is received for a protected resource, the web container checks if
 The Servlet Specification does not address how the static security information in the `WEB-INF/web.xml` file is mapped to the runtime environment of the container.
 For Jetty, the link:{JDURL}/org/eclipse/jetty/security/LoginService.html[LoginService] performs this function.
 
-A LoginService has a unique name, and gives access to information about a set of users.
+A `LoginService` has a unique name, and gives access to information about a set of users.
 Each user has authentication information (e.g. a password) and a set of roles associated with him/herself.
 
 You may configure one or many different LoginServices depending on your needs.
@@ -114,7 +114,8 @@ When a request to a web application requires authentication or authorization, Je
 
 ==== Scoping Security Realms
 
-A LoginService has a unique name, and is composed of a set of users. Each user has authentication information (for example, a password) and a set of roles associated with him/herself.
+A `LoginService` has a unique name, and is composed of a set of users.
+Each user has authentication information (for example, a password) and a set of roles associated with him/herself.
 You can configure one or many different realms depending on your needs.
 
 * Configure a single LoginService to share common security information across all of your web applications.
@@ -144,8 +145,8 @@ Here's an example of an xml file that defines an in-memory type of LoginService
 
 ----
 
-If you define more than one LoginService on a Server, you will need to specify which one you want used for each context.
+If you define more than one `LoginService` on a Server, you will need to specify which one you want used for each context.
-You can do that by telling the context the name of the LoginService, or passing it the LoginService instance.
+You can do that by telling the context the name of the `LoginService`, or passing it the `LoginService` instance.
 Here's an example of doing both of these, using a link:#deployable-descriptor-file[context xml file]:
 
 [source, xml, subs="{sub-order}"]
@@ -170,7 +171,7 @@ Here's an example of doing both of these, using a link:#deployable-descriptor-fi
 
 ===== Per-Webapp Scoped
 
-Alternatively, you can define a LoginService for just a single web application.
+Alternatively, you can define a `LoginService` for just a single web application.
 Here's how to define the same HashLoginService, but inside a link:#deployable-descriptor-file[context xml file]:
 
 [source, xml, subs="{sub-order}"]
@@ -192,12 +193,12 @@ Here's how to define the same HashLoginService, but inside a link:#deployable-de
 
 ----
 
-Jetty provides a number of different LoginService types which can be seen in the next section.
+Jetty provides a number of different `LoginService` types which can be seen in the next section.
 
 [[configuring-login-service]]
 ==== Configuring a LoginService
 
-A link:{JDURL}/org/eclipse/jetty/security/LoginService.html[LoginService] instance is required by each context/webapp that has a authentication mechanism, which is used to check the validity of the username and credentials collected by the authentication mechanism. Jetty provides the following implementations of LoginService:
+A link:{JDURL}/org/eclipse/jetty/security/LoginService.html[`LoginService`] instance is required by each context/webapp that has a authentication mechanism, which is used to check the validity of the username and credentials collected by the authentication mechanism. Jetty provides the following implementations of `LoginService`:
 
 link:{JDURL}/org/eclipse/jetty/security/HashLoginService.html[HashLoginService]::
   A user realm that is backed by a hash map that is filled either programatically or from a Java properties file.
@@ -211,16 +212,16 @@ link:{JDURL}/org/eclipse/jetty/jaas/JAASLoginService.html[JAASLoginService]::
 link:{JDURL}/org/eclipse/jetty/security/SpnegoLoginService.html[SpnegoLoginService]::
   http://en.wikipedia.org/wiki/SPNEGO[SPNEGO] Authentication; see the section on link:#spnego-support[SPNEGO support] for more information.
 
-An instance of a LoginService can be matched to a context/webapp by:
+An instance of a `LoginService` can be matched to a context/webapp by:
 
-* A LoginService instance may be set directly on the SecurityHandler instance via embedded code or IoC XML
+* A `LoginService` instance may be set directly on the `SecurityHandler` instance via embedded code or IoC XML
-* Matching the realm-name defined in web.xml with the name of a LoginService instance that has been added to the Server instance as a dependent bean
+* Matching the realm-name defined in web.xml with the name of a `LoginService` instance that has been added to the Server instance as a dependent bean
-* If only a single LoginService instance has been set on the Server then it is used as the login service for the context
+* If only a single `LoginService` instance has been set on the Server then it is used as the login service for the context
 
 [[hash-login-service]]
 ===== HashLoginService
 
-The HashLoginService is a simple and efficient login service that loads usernames, credentials and roles from a Java properties file in the format:
+The `HashLoginService` is a simple and efficient login service that loads usernames, credentials and roles from a Java properties file in the format:
 
 [source,properties]
 ----
@@ -249,7 +250,7 @@ guest: guest,read-only
 
 ----
 
-You configure the HashLoginService with a name and a reference to the location of the properties file:
+You configure the `HashLoginService` with a name and a reference to the location of the properties file:
 
 [source, xml, subs="{sub-order}"]
 ----
@@ -378,12 +379,12 @@ When a user requests a resource that is access protected, the LoginService will
 
 Until Servlet 3.1, role-based authorization could define:
 
-* access granted to a set of named roles
+* Access granted to a set of named roles
-* access totally forbidden, regardless of role
+* Access totally forbidden, regardless of role
-* access granted to a user in any of the roles defined in the effective web.xml.
+* Access granted to a user in any of the roles defined in the effective web.xml.
-This is indicated by the special value of "*" for the `<role-name>` of a `<auth-constraint> `in the `<security-constraint>`
+This is indicated by the special value of `*` for the `<role-name>` of a `<auth-constraint>` in the `<security-constraint>`
 
 With the advent of Servlet 3.1, there is now another authorization:
 
-* access granted to any user who is authenticated, regardless of roles.
+* Access granted to any user who is authenticated, regardless of roles.
-This is indicated by the special value of "**" for the `<role-name>` of a `<auth-constraint>` in the `<security-constraint>`
+This is indicated by the special value of `**` for the `<role-name>` of a `<auth-constraint>` in the `<security-constraint>`

jetty-documentation/src/main/asciidoc/quick-start/getting-started/jetty-installing.adoc

@@ -30,6 +30,12 @@ When you download and unpack the binary, it is extracted into a directory called
 Put this directory in a convenient location.
 The rest of the instructions in this documentation refer to this location as either `JETTY_HOME` or as `$(jetty.home).`
 
+_____
+[IMPORTANT]
+It is important that only stable releases are used in production environments.
+Versions that have been deprecated or are released as Milestones (M) or Release Candidates (RC) are not suitable for production as they may contain security flaws or incomplete/non-functioning feature sets.
+_____
+
 [[distribution-content]]
 ==== Distribution Content
 

jetty-documentation/src/main/asciidoc/quick-start/introduction/jetty-coordinates.adoc

@@ -17,6 +17,12 @@
 [[quickstart-jetty-coordinates]]
 === Finding Jetty in Maven
 
+_____
+[IMPORTANT]
+It is important that only stable releases are used in production environments.
+Versions that have been deprecated or are released as Milestones (M) or Release Candidates (RC) are not suitable for production as they may contain security flaws or incomplete/non-functioning feature sets.
+_____
+
 ==== Maven Coordinates
 
 Jetty has existed in Maven Central almost since its inception, though the coordinates have changed over the years.
@@ -34,7 +40,7 @@ The top level Project Object Model (POM) for the Jetty project is located under
 </dependency>
 ----
 
-==== Changelogs in Central
+==== Changelogs in Maven Central
 
 The changes between versions of Jetty are tracked in a file called VERSIONS.txt, which is under source control and is generated on release.
 Those generated files are also uploaded into Maven Central during the release of the top level POM. You can find them as a classifier marked artifact.

jetty-documentation/src/main/asciidoc/quick-start/introduction/what-version.adoc

@@ -21,6 +21,12 @@ Jetty 9 is the most recent version of Jetty and has a great many improvements ov
 This documentation which focuses on Jetty 9.
 While many people continue to use older versions of Jetty, we generally recommend using Jetty 9 as it represents the version of Jetty that we will actively maintain and improve over the next few years.
 
+_____
+[IMPORTANT]
+It is important that only stable releases are used in production environments.
+Versions that have been deprecated or are released as Milestones (M) or Release Candidates (RC) are not suitable for production as they may contain security flaws or incomplete/non-functioning feature sets. 
+_____
+
 .Jetty Versions
 [width="100%",cols="12%,9%,15%,6%,21%,10%,6%,21%",options="header",]
 |=======================================================================

jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpSenderOverFCGI.java

@@ -125,4 +125,10 @@ public class HttpSenderOverFCGI extends HttpSender
             getHttpChannel().flush(result);
         }
     }
+
+    @Override
+    protected void sendTrailers(HttpExchange exchange, Callback callback)
+    {
+        callback.succeeded();
+    }
 }

jetty-http/src/main/java/org/eclipse/jetty/http/HttpGenerator.java

@@ -18,8 +18,6 @@
 
 package org.eclipse.jetty.http;
 
-import static org.eclipse.jetty.http.HttpStatus.INTERNAL_SERVER_ERROR_500;
-
 import java.io.IOException;
 import java.nio.BufferOverflowException;
 import java.nio.ByteBuffer;
@@ -34,6 +32,8 @@ import org.eclipse.jetty.util.Trie;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 
+import static org.eclipse.jetty.http.HttpStatus.INTERNAL_SERVER_ERROR_500;
+
 /**
  * HttpGenerator. Builds HTTP Messages.
  * <p>
@@ -764,7 +764,7 @@ public class HttpGenerator
         }
         // Else if we are HTTP/1.1 and the content length is unknown and we are either persistent
         // or it is a request with content (which cannot EOF) or the app has requested chunking
-        else if (http11 && content_length<0 && (_persistent || assumed_content_request || chunked_hint))
+        else if (http11 && (chunked_hint || content_length<0 && (_persistent || assumed_content_request)))
         {
             // we use chunking
             _endOfContent = EndOfContent.CHUNKED_CONTENT;

jetty-http2/http2-client/src/main/java/org/eclipse/jetty/http2/client/HTTP2Client.java

@@ -129,7 +129,6 @@ public class HTTP2Client extends ContainerLifeCycle
     private int initialSessionRecvWindow = FlowControlStrategy.DEFAULT_WINDOW_SIZE;
     private int initialStreamRecvWindow = FlowControlStrategy.DEFAULT_WINDOW_SIZE;
     private FlowControlStrategy.Factory flowControlStrategyFactory = () -> new BufferingFlowControlStrategy(0.5F);
-    private ExecutionStrategy.Factory executionStrategyFactory = new ProduceConsume.Factory();
 
     @Override
     protected void doStart() throws Exception

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Connection.java

@@ -31,12 +31,11 @@ import org.eclipse.jetty.io.ByteBufferPool;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.component.LifeCycle;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 import org.eclipse.jetty.util.thread.ExecutionStrategy;
-import org.eclipse.jetty.util.thread.Invocable;
+import org.eclipse.jetty.util.thread.strategy.EatWhatYouKill;
-import org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume;
-import org.eclipse.jetty.util.thread.strategy.ProduceExecuteConsume;
 
 public class HTTP2Connection extends AbstractConnection
 {
@@ -49,8 +48,7 @@ public class HTTP2Connection extends AbstractConnection
     private final Parser parser;
     private final ISession session;
     private final int bufferSize;
-    private final ExecutionStrategy blockingStrategy;
+    private final ExecutionStrategy strategy;
-    private final ExecutionStrategy nonBlockingStrategy;
 
     public HTTP2Connection(ByteBufferPool byteBufferPool, Executor executor, EndPoint endPoint, Parser parser, ISession session, int bufferSize)
     {
@@ -59,8 +57,9 @@ public class HTTP2Connection extends AbstractConnection
         this.parser = parser;
         this.session = session;
         this.bufferSize = bufferSize;
-        this.blockingStrategy = new ExecuteProduceConsume(producer, executor);
+        this.strategy = new EatWhatYouKill(producer, executor, 0);
-        this.nonBlockingStrategy = new ProduceExecuteConsume(producer, executor);
+        
+        LifeCycle.start(strategy);
     }
 
     @Override
@@ -96,7 +95,7 @@ public class HTTP2Connection extends AbstractConnection
         if (LOG.isDebugEnabled())
             LOG.debug("HTTP2 Open {} ", this);
         super.onOpen();
-        blockingStrategy.produce();
+        strategy.produce();
     }
 
     @Override
@@ -105,26 +104,16 @@ public class HTTP2Connection extends AbstractConnection
         if (LOG.isDebugEnabled())
             LOG.debug("HTTP2 Close {} ", this);
         super.onClose();
+
+        LifeCycle.stop(strategy);
     }
 
     @Override
     public void onFillable()
-    {
-        throw new UnsupportedOperationException();
-    }
-
-    private void onFillableBlocking()
     {
         if (LOG.isDebugEnabled())
-            LOG.debug("HTTP2 onFillableBlocking {} ", this);
+            LOG.debug("HTTP2 onFillable {} ", this);
-        blockingStrategy.produce();
+        strategy.produce();
-    }
-
-    private void onFillableNonBlocking()
-    {
-        if (LOG.isDebugEnabled())
-            LOG.debug("HTTP2 onFillableNonBlocking {} ", this);
-        nonBlockingStrategy.produce();
     }
 
     private int fill(EndPoint endPoint, ByteBuffer buffer)
@@ -158,16 +147,7 @@ public class HTTP2Connection extends AbstractConnection
     protected void offerTask(Runnable task, boolean dispatch)
     {
         offerTask(task);
-
+        strategy.dispatch();
-        // Because producing calls parse and parse can call offerTask, we have to make sure
-        // we use the same strategy otherwise produce can be reentrant and that messes with 
-        // the release mechanism.  TODO is this test sufficient to protect from this?
-        ExecutionStrategy s = Invocable.isNonBlockingInvocation() ? nonBlockingStrategy : blockingStrategy;
-        if (dispatch)
-            // TODO Why again is this necessary?
-            s.dispatch();
-        else
-            s.produce();
     }
 
     @Override
@@ -271,10 +251,7 @@ public class HTTP2Connection extends AbstractConnection
         @Override
         public void succeeded()
         {
-            if (Invocable.isNonBlockingInvocation())
+            onFillable();
-                onFillableNonBlocking();
-            else
-                onFillableBlocking();
         }
 
         @Override

jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpReceiverOverHTTP2.java

@@ -71,27 +71,37 @@ public class HttpReceiverOverHTTP2 extends HttpReceiver implements Stream.Listen
         if (exchange == null)
             return;
 
-        HttpResponse response = exchange.getResponse();
+        HttpResponse httpResponse = exchange.getResponse();
-        MetaData.Response metaData = (MetaData.Response)frame.getMetaData();
+        MetaData metaData = frame.getMetaData();
-        response.version(metaData.getHttpVersion()).status(metaData.getStatus()).reason(metaData.getReason());
+        if (metaData.isResponse())
-
-        if (responseBegin(exchange))
         {
-            HttpFields headers = metaData.getFields();
+            MetaData.Response response = (MetaData.Response)frame.getMetaData();
-            for (HttpField header : headers)
+            httpResponse.version(response.getHttpVersion()).status(response.getStatus()).reason(response.getReason());
-            {
-                if (!responseHeader(exchange, header))
-                    return;
-            }
 
-            if (responseHeaders(exchange))
+            if (responseBegin(exchange))
             {
-                int status = metaData.getStatus();
+                HttpFields headers = response.getFields();
-                boolean informational = HttpStatus.isInformational(status) && status != HttpStatus.SWITCHING_PROTOCOLS_101;
+                for (HttpField header : headers)
-                if (frame.isEndStream() || informational)
+                {
-                    responseSuccess(exchange);
+                    if (!responseHeader(exchange, header))
+                        return;
+                }
+
+                if (responseHeaders(exchange))
+                {
+                    int status = response.getStatus();
+                    boolean informational = HttpStatus.isInformational(status) && status != HttpStatus.SWITCHING_PROTOCOLS_101;
+                    if (frame.isEndStream() || informational)
+                        responseSuccess(exchange);
+                }
             }
         }
+        else
+        {
+            HttpFields trailers = metaData.getFields();
+            trailers.forEach(httpResponse::trailer);
+            responseSuccess(exchange);
+        }
     }
 
     @Override

jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpSenderOverHTTP2.java

@@ -19,11 +19,13 @@
 package org.eclipse.jetty.http2.client.http;
 
 import java.net.URI;
+import java.util.function.Supplier;
 
 import org.eclipse.jetty.client.HttpContent;
 import org.eclipse.jetty.client.HttpExchange;
+import org.eclipse.jetty.client.HttpRequest;
 import org.eclipse.jetty.client.HttpSender;
-import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpURI;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.http.MetaData;
@@ -49,11 +51,13 @@ public class HttpSenderOverHTTP2 extends HttpSender
     @Override
     protected void sendHeaders(HttpExchange exchange, final HttpContent content, final Callback callback)
     {
-        Request request = exchange.getRequest();
+        HttpRequest request = exchange.getRequest();
         String path = relativize(request.getPath());
         HttpURI uri = new HttpURI(request.getScheme(), request.getHost(), request.getPort(), path, null, request.getQuery(), null);
         MetaData.Request metaData = new MetaData.Request(request.getMethod(), uri, HttpVersion.HTTP_2, request.getHeaders());
-        HeadersFrame headersFrame = new HeadersFrame(metaData, null, !content.hasContent());
+        Supplier<HttpFields> trailers = request.getTrailers();
+        metaData.setTrailerSupplier(trailers);
+        HeadersFrame headersFrame = new HeadersFrame(metaData, null, trailers == null && !content.hasContent());
         HttpChannelOverHTTP2 channel = getHttpChannel();
         Promise<Stream> promise = new Promise<Stream>()
         {
@@ -66,7 +70,7 @@ public class HttpSenderOverHTTP2 extends HttpSender
                 if (content.hasContent() && !expects100Continue(request))
                 {
                     boolean advanced = content.advance();
-                    boolean lastContent = content.isLast();
+                    boolean lastContent = trailers == null && content.isLast();
                     if (advanced || lastContent)
                     {
                         DataFrame dataFrame = new DataFrame(stream.getId(), content.getByteBuffer(), lastContent);
@@ -115,8 +119,19 @@ public class HttpSenderOverHTTP2 extends HttpSender
         else
         {
             Stream stream = getHttpChannel().getStream();
-            DataFrame frame = new DataFrame(stream.getId(), content.getByteBuffer(), content.isLast());
+            Supplier<HttpFields> trailers = exchange.getRequest().getTrailers();
+            DataFrame frame = new DataFrame(stream.getId(), content.getByteBuffer(), trailers == null && content.isLast());
             stream.data(frame, callback);
         }
     }
+
+    @Override
+    protected void sendTrailers(HttpExchange exchange, Callback callback)
+    {
+        Supplier<HttpFields> trailers = exchange.getRequest().getTrailers();
+        MetaData metaData = new MetaData(HttpVersion.HTTP_2, trailers.get());
+        Stream stream = getHttpChannel().getStream();
+        HeadersFrame trailersFrame = new HeadersFrame(stream.getId(), metaData, null, true);
+        stream.headers(trailersFrame, callback);
+    }
 }

jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HTTP2ServerConnection.java

@@ -173,7 +173,11 @@ public class HTTP2ServerConnection extends HTTP2Connection implements Connection
             LOG.debug("Processing trailers {} on {}", frame, stream);
         HttpChannelOverHTTP2 channel = (HttpChannelOverHTTP2)stream.getAttribute(IStream.CHANNEL_ATTRIBUTE);
         if (channel != null)
-            channel.onRequestTrailers(frame);
+        {
+            Runnable task = channel.onRequestTrailers(frame);
+            if (task != null)
+                offerTask(task, false);
+        }
     }
 
     public boolean onStreamTimeout(IStream stream, Throwable failure)

jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpChannelOverHTTP2.java

@@ -282,11 +282,12 @@ public class HttpChannelOverHTTP2 extends HttpChannel
         return handle || wasDelayed ? this : null;
     }
 
-    public void onRequestTrailers(HeadersFrame frame)
+    public Runnable onRequestTrailers(HeadersFrame frame)
     {
         HttpFields trailers = frame.getMetaData().getFields();
-        onTrailers(trailers);
+        if (trailers.size() > 0)
-        onRequestComplete();
+            onTrailers(trailers);
+
         if (LOG.isDebugEnabled())
         {
             Stream stream = getStream();
@@ -294,6 +295,14 @@ public class HttpChannelOverHTTP2 extends HttpChannel
                     stream.getId(), Integer.toHexString(stream.getSession().hashCode()),
                     System.lineSeparator(), trailers);
         }
+
+        boolean handle = onRequestComplete();
+
+        boolean wasDelayed = _delayedUntilContent;
+        _delayedUntilContent = false;
+        if (wasDelayed)
+            _handled = true;
+        return handle || wasDelayed ? this : null;
     }
 
     public boolean isRequestHandled()

jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java

@@ -20,7 +20,9 @@ package org.eclipse.jetty.http2.server;
 
 import java.nio.ByteBuffer;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
 
+import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.http.MetaData;
@@ -48,6 +50,7 @@ public class HttpTransportOverHTTP2 implements HttpTransport
     private final Connector connector;
     private final HTTP2ServerConnection connection;
     private IStream stream;
+    private MetaData metaData;
 
     public HttpTransportOverHTTP2(Connector connector, HTTP2ServerConnection connection)
     {
@@ -85,48 +88,60 @@ public class HttpTransportOverHTTP2 implements HttpTransport
     public void send(MetaData.Response info, boolean isHeadRequest, ByteBuffer content, boolean lastContent, Callback callback)
     {
         boolean hasContent = BufferUtil.hasContent(content) && !isHeadRequest;
-
         if (info != null)
         {
+            metaData = info;
+
             int status = info.getStatus();
             boolean informational = HttpStatus.isInformational(status) && status != HttpStatus.SWITCHING_PROTOCOLS_101;
-            boolean committed = false;
+            if (informational)
-            if (!informational)
-                committed = commit.compareAndSet(false, true);
-
-            if (committed || informational)
             {
-                if (hasContent)
+                if (transportCallback.start(callback, false))
-                {
+                    sendHeaders(info, false, transportCallback);
-                    Callback commitCallback = new Callback.Nested(callback)
-                    {
-                        @Override
-                        public void succeeded()
-                        {
-                            if (transportCallback.start(callback, false))
-                                send(content, lastContent, transportCallback);
-                        }
-                    };
-                    if (transportCallback.start(commitCallback, true))
-                        commit(info, false, transportCallback);
-                }
-                else
-                {
-                    if (transportCallback.start(callback, false))
-                        commit(info, lastContent, transportCallback);
-                }
             }
             else
             {
-                callback.failed(new IllegalStateException("committed"));
+                boolean needsCommit = commit.compareAndSet(false, true);
+                if (needsCommit)
+                {
+                    Supplier<HttpFields> trailers = info.getTrailerSupplier();
+
+                    if (hasContent)
+                    {
+                        Callback nested = trailers == null || !lastContent ? callback : new SendTrailers(callback);
+                        Callback commitCallback = new Callback.Nested(nested)
+                        {
+                            @Override
+                            public void succeeded()
+                            {
+                                if (transportCallback.start(nested, false))
+                                    sendContent(content, lastContent, trailers == null && lastContent, transportCallback);
+                            }
+                        };
+                        if (transportCallback.start(commitCallback, true))
+                            sendHeaders(info, false, transportCallback);
+                    }
+                    else
+                    {
+                        Callback nested = trailers == null ? callback : new SendTrailers(callback);
+                        if (transportCallback.start(nested, true))
+                            sendHeaders(info, trailers == null && lastContent, transportCallback);
+                    }
+                }
+                else
+                {
+                    callback.failed(new IllegalStateException("committed"));
+                }
             }
         }
         else
         {
             if (hasContent || lastContent)
             {
-                if (transportCallback.start(callback, false))
+                Supplier<HttpFields> trailers = metaData.getTrailerSupplier();
-                    send(content, lastContent, transportCallback);
+                Callback nested = trailers == null ? callback : new SendTrailers(callback);
+                if (transportCallback.start(nested, false))
+                    sendContent(content, lastContent, trailers == null && lastContent, transportCallback);
             }
             else
             {
@@ -171,7 +186,7 @@ public class HttpTransportOverHTTP2 implements HttpTransport
         }, new Stream.Listener.Adapter()); // TODO: handle reset from the client ?
     }
 
-    private void commit(MetaData.Response info, boolean endStream, Callback callback)
+    private void sendHeaders(MetaData.Response info, boolean endStream, Callback callback)
     {
         if (LOG.isDebugEnabled())
         {
@@ -185,7 +200,7 @@ public class HttpTransportOverHTTP2 implements HttpTransport
         stream.headers(frame, callback);
     }
 
-    private void send(ByteBuffer content, boolean lastContent, Callback callback)
+    private void sendContent(ByteBuffer content, boolean lastContent, boolean endStream, Callback callback)
     {
         if (LOG.isDebugEnabled())
         {
@@ -193,10 +208,22 @@ public class HttpTransportOverHTTP2 implements HttpTransport
                     stream.getId(), Integer.toHexString(stream.getSession().hashCode()),
                     content.remaining(), lastContent ? " (last chunk)" : "");
         }
-        DataFrame frame = new DataFrame(stream.getId(), content, lastContent);
+        DataFrame frame = new DataFrame(stream.getId(), content, endStream);
         stream.data(frame, callback);
     }
 
+    private void sendTrailers(MetaData metaData, Callback callback)
+    {
+        if (LOG.isDebugEnabled())
+        {
+            LOG.debug("HTTP2 Response #{}/{}: trailers",
+                    stream.getId(), Integer.toHexString(stream.getSession().hashCode()));
+        }
+
+        HeadersFrame frame = new HeadersFrame(stream.getId(), metaData, null, true);
+        stream.headers(frame, callback);
+    }
+
     public void onStreamFailure(Throwable failure)
     {
         transportCallback.failed(failure);
@@ -277,7 +304,9 @@ public class HttpTransportOverHTTP2 implements HttpTransport
                 }
             }
             if (LOG.isDebugEnabled())
-                LOG.debug("HTTP2 Response #{} {}", stream.getId(), commit ? "committed" : "flushed content");
+                LOG.debug("HTTP2 Response #{}/{} {}",
+                        stream.getId(), Integer.toHexString(stream.getSession().hashCode()),
+                        commit ? "committed" : "flushed content");
             if (callback != null)
                 callback.succeeded();
         }
@@ -340,4 +369,19 @@ public class HttpTransportOverHTTP2 implements HttpTransport
     {
         IDLE, WRITING, FAILED, TIMEOUT
     }
+
+    private class SendTrailers extends Callback.Nested
+    {
+        private SendTrailers(Callback callback)
+        {
+            super(callback);
+        }
+
+        @Override
+        public void succeeded()
+        {
+            if (transportCallback.start(getCallback(), false))
+                sendTrailers(new MetaData(HttpVersion.HTTP_2, metaData.getTrailerSupplier().get()), transportCallback);
+        }
+    }
 }

jetty-io/src/main/java/org/eclipse/jetty/io/ManagedSelector.java

@@ -48,6 +48,7 @@ import org.eclipse.jetty.util.thread.Invocable;
 import org.eclipse.jetty.util.thread.Invocable.InvocationType;
 import org.eclipse.jetty.util.thread.Locker;
 import org.eclipse.jetty.util.thread.Scheduler;
+import org.eclipse.jetty.util.thread.strategy.EatWhatYouKill;
 import org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume;
 import org.eclipse.jetty.util.thread.strategy.ProduceExecuteConsume;
 
@@ -57,7 +58,7 @@ import org.eclipse.jetty.util.thread.strategy.ProduceExecuteConsume;
  * happen for registered channels. When events happen, it notifies the {@link EndPoint} associated
  * with the channel.</p>
  */
-public class ManagedSelector extends AbstractLifeCycle implements Dumpable
+public class ManagedSelector extends ContainerLifeCycle implements Dumpable
 {
     private static final Logger LOG = Log.getLogger(ManagedSelector.class);
 
@@ -67,7 +68,6 @@ public class ManagedSelector extends AbstractLifeCycle implements Dumpable
     private final SelectorManager _selectorManager;
     private final int _id;
     private final ExecutionStrategy _strategy;
-    private final ExecutionStrategy _lowPriorityStrategy;
     private Selector _selector;
 
     public ManagedSelector(SelectorManager selectorManager, int id)
@@ -76,8 +76,8 @@ public class ManagedSelector extends AbstractLifeCycle implements Dumpable
         _id = id;
         SelectorProducer producer = new SelectorProducer();
         Executor executor = selectorManager.getExecutor();
-        _strategy = new ExecuteProduceConsume(producer, executor, Invocable.InvocationType.BLOCKING);
+        _strategy = new EatWhatYouKill(producer,executor);            
-        _lowPriorityStrategy = new LowPriorityProduceExecuteConsume(producer, executor);
+        addBean(_strategy);
         setStopTimeout(5000);
     }
 
@@ -94,29 +94,6 @@ public class ManagedSelector extends AbstractLifeCycle implements Dumpable
         // The normal strategy obtains the produced task, schedules
         // a new thread to produce more, runs the task and then exits.
         _selectorManager.execute(_strategy::produce);
-
-        // The low priority strategy knows the producer will never
-        // be idle, that tasks are scheduled to run in different
-        // threads, therefore lowPriorityProduce() never exits.
-        _selectorManager.execute(this::lowPriorityProduce);
-    }
-
-    private void lowPriorityProduce()
-    {
-        Thread current = Thread.currentThread();
-        String name = current.getName();
-        int priority = current.getPriority();
-        current.setPriority(Thread.MIN_PRIORITY);
-        current.setName(name+"-lowPrioritySelector");
-        try
-        {
-            _lowPriorityStrategy.produce();
-        }
-        finally
-        {
-            current.setPriority(priority);
-            current.setName(name);
-        }
     }
 
     public int size()
@@ -135,11 +112,12 @@ public class ManagedSelector extends AbstractLifeCycle implements Dumpable
         CloseEndPoints close_endps = new CloseEndPoints();
         submit(close_endps);
         close_endps.await(getStopTimeout());
-        super.doStop();
         CloseSelector close_selector = new CloseSelector();
         submit(close_selector);
         close_selector.await(getStopTimeout());
 
+        super.doStop();
+        
         if (LOG.isDebugEnabled())
             LOG.debug("Stopped {}", this);
     }
@@ -185,42 +163,6 @@ public class ManagedSelector extends AbstractLifeCycle implements Dumpable
         void updateKey();
     }
 
-    private static class LowPriorityProduceExecuteConsume extends ProduceExecuteConsume
-    {
-        private LowPriorityProduceExecuteConsume(SelectorProducer producer, Executor executor)
-        {
-            super(producer, executor, InvocationType.BLOCKING);
-        }
-
-        @Override
-        protected boolean execute(Runnable task)
-        {
-            try
-            {
-                InvocationType invocation=Invocable.getInvocationType(task);
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Low Priority Selector executing {} {}",invocation,task);
-                switch (invocation)
-                {
-                    case NON_BLOCKING:
-                        task.run();
-                        return true;
-
-                    case EITHER:
-                        Invocable.invokeNonBlocking(task);
-                        return true;
-
-                    default:
-                        return super.execute(task);
-                }
-            }
-            finally
-            {
-                // Allow opportunity for main strategy to take over.
-                Thread.yield();
-            }
-        }
-    }
 
     private class SelectorProducer implements ExecutionStrategy.Producer
     {
@@ -230,30 +172,20 @@ public class ManagedSelector extends AbstractLifeCycle implements Dumpable
         @Override
         public Runnable produce()
         {
-            // This method is called from both the
+            while (true)
-            // normal and low priority strategies.
-            // Only one can produce at a time, so it's synchronized
-            // to enforce that only one strategy actually produces.
-            // When idle in select(), this method blocks;
-            // the other strategy's thread will be blocked
-            // waiting for this lock to be released.
-            synchronized (this)
             {
-                while (true)
+                Runnable task = processSelected();
-                {
+                if (task != null)
-                    Runnable task = processSelected();
+                    return task;
-                    if (task != null)
-                        return task;
 
-                    Runnable action = nextAction();
+                Runnable action = nextAction();
-                    if (action != null)
+                if (action != null)
-                        return action;
+                    return action;
 
-                    update();
+                update();
 
-                    if (!select())
+                if (!select())
-                        return null;
+                    return null;
-                }
             }
         }
 

jetty-io/src/main/java/org/eclipse/jetty/io/ssl/SslConnection.java

@@ -307,12 +307,13 @@ public class SslConnection extends AbstractConnection
         b = _decryptedInput;
         int di=b==null?-1:b.remaining();
 
+        Connection connection = _decryptedEndPoint.getConnection();
         return String.format("%s@%x{%s,eio=%d/%d,di=%d}=>%s",
                 getClass().getSimpleName(),
                 hashCode(),
                 _sslEngine.getHandshakeStatus(),
                 ei,eo,di,
-                ((AbstractConnection)_decryptedEndPoint.getConnection()).toConnectionString());
+                connection instanceof AbstractConnection ? ((AbstractConnection)connection).toConnectionString() : connection);
     }
 
     public class DecryptedEndPoint extends AbstractEndPoint

jetty-server/src/main/config/modules/requestlog.mod

@@ -27,7 +27,7 @@ logs/
 # jetty.requestlog.retainDays=90
 
 ## Whether to append to existing file
-# jetty.requestlog.append=true
+# jetty.requestlog.append=false
 
 ## Whether to use the extended log output
 # jetty.requestlog.extended=true

jetty-server/src/main/java/org/eclipse/jetty/server/AsyncContextEvent.java

@@ -106,6 +106,11 @@ public class AsyncContextEvent extends AsyncEvent implements Runnable
         _timeoutTask = task;
     }
 
+    public boolean hasTimeoutTask()
+    {
+        return _timeoutTask!=null;
+    }
+    
     public void cancelTimeoutTask()
     {
         Scheduler.Task task=_timeoutTask;

jetty-server/src/main/java/org/eclipse/jetty/server/HttpChannelState.java

@@ -78,7 +78,7 @@ public class HttpChannelState
         ERROR_DISPATCH,   // handle a normal error
         ASYNC_ERROR,      // handle an async error
         WRITE_CALLBACK,   // handle an IO write callback
-        READ_PRODUCE,    // Check is a read is possible by parsing/filling
+        READ_PRODUCE,     // Check is a read is possible by parsing/filling
         READ_CALLBACK,    // handle an IO read callback
         COMPLETE,         // Complete the response
         TERMINATED,       // No further actions
@@ -102,13 +102,12 @@ public class HttpChannelState
 
     private enum AsyncRead
     {
-        NONE,           // No isReady; No data
+        IDLE,           // No isReady; No data
-        AVAILABLE,      // No isReady; onDataAvailable
+        REGISTER,       // isReady()==false handling; No data
-        NEEDED,         // isReady()==false handling; No data
         REGISTERED,     // isReady()==false !handling; No data
-        POSSIBLE,       // isReady()==false async callback called (http/1 only)
+        POSSIBLE,       // isReady()==false async read callback called (http/1 only)
-        PRODUCING,      // isReady()==false handling content production  (http/1 only)
+        PRODUCING,      // isReady()==false READ_PRODUCE action is being handled (http/1 only)
-        READY           // isReady() was false, data now available
+        READY           // isReady() was false, onContentAdded has been called
     }
 
     private final Locker _locker=new Locker();
@@ -117,7 +116,7 @@ public class HttpChannelState
     private State _state;
     private Async _async;
     private boolean _initial;
-    private AsyncRead _asyncRead=AsyncRead.NONE;
+    private AsyncRead _asyncRead=AsyncRead.IDLE;
     private boolean _asyncWritePossible;
     private long _timeoutMs=DEFAULT_TIMEOUT;
     private AsyncContextEvent _event;
@@ -237,9 +236,13 @@ public class HttpChannelState
                             return Action.READ_PRODUCE;
                         case READY:
                             _state=State.ASYNC_IO;
-                            _asyncRead=AsyncRead.NONE;
+                            _asyncRead=AsyncRead.IDLE;
                             return Action.READ_CALLBACK;
-                        default: 
+                        case REGISTER:
+                        case PRODUCING:
+                            throw new IllegalStateException(toStringLocked());
+                        case IDLE:
+                        case REGISTERED:
                             break;
                     }
 
@@ -386,7 +389,6 @@ public class HttpChannelState
      */
     protected Action unhandle()
     {
-        Action action;
         boolean read_interested = false;
 
         try(Locker.Lock lock= _locker.lock())
@@ -414,41 +416,38 @@ public class HttpChannelState
             }
 
             _initial=false;
-            async: switch(_async)
+            switch(_async)
             {
                 case COMPLETE:
                     _state=State.COMPLETING;
                     _async=Async.NOT_ASYNC;
-                    action=Action.COMPLETE;
+                    return Action.COMPLETE;
-                    break;
 
                 case DISPATCH:
                     _state=State.DISPATCHED;
                     _async=Async.NOT_ASYNC;
-                    action=Action.ASYNC_DISPATCH;
+                    return Action.ASYNC_DISPATCH;
-                    break;
 
                 case STARTED:
                     switch(_asyncRead)
                     {
                         case READY:
                             _state=State.ASYNC_IO;
-                            _asyncRead=AsyncRead.NONE;
+                            _asyncRead=AsyncRead.IDLE;
-                            action=Action.READ_CALLBACK;
+                            return Action.READ_CALLBACK;
-                            break async;
 
                         case POSSIBLE:
                             _state=State.ASYNC_IO;
-                            action=Action.READ_PRODUCE;
+                            _asyncRead=AsyncRead.PRODUCING;
-                            break async;
+                            return Action.READ_PRODUCE;
                             
-                        case NEEDED:
+                        case REGISTER:
                         case PRODUCING:
                             _asyncRead=AsyncRead.REGISTERED;
                             read_interested=true;
-                                                        
+                            break;
-                        case NONE:
+                        
-                        case AVAILABLE:
+                        case IDLE:
                         case REGISTERED:
                             break;
                     }
@@ -457,54 +456,50 @@ public class HttpChannelState
                     {
                         _state=State.ASYNC_IO;
                         _asyncWritePossible=false;
-                        action=Action.WRITE_CALLBACK;
+                        return Action.WRITE_CALLBACK;
                     }
                     else
                     {
                         _state=State.ASYNC_WAIT;
-                        action=Action.WAIT; 
+                        
                         Scheduler scheduler=_channel.getScheduler();
-                        if (scheduler!=null && _timeoutMs>0)
+                        if (scheduler!=null && _timeoutMs>0 && !_event.hasTimeoutTask())
                             _event.setTimeoutTask(scheduler.schedule(_event,_timeoutMs,TimeUnit.MILLISECONDS));
+
+                        return Action.WAIT; 
                     }
-                    break;
 
                 case EXPIRING:
                     // onTimeout callbacks still being called, so just WAIT
                     _state=State.ASYNC_WAIT;
-                    action=Action.WAIT;
+                    return Action.WAIT;
-                    break;
 
                 case EXPIRED:
                     // onTimeout handling is complete, but did not dispatch as
                     // we were handling.  So do the error dispatch here
                     _state=State.DISPATCHED;
                     _async=Async.NOT_ASYNC;
-                    action=Action.ERROR_DISPATCH;
+                    return Action.ERROR_DISPATCH;
-                    break;
 
                 case ERRORED:
                     _state=State.DISPATCHED;
                     _async=Async.NOT_ASYNC;
-                    action=Action.ERROR_DISPATCH;
+                    return Action.ERROR_DISPATCH;
-                    break;
 
                 case NOT_ASYNC:
                     _state=State.COMPLETING;
-                    action=Action.COMPLETE;
+                    return Action.COMPLETE;
-                    break;
 
                 default:
                     _state=State.COMPLETING;
-                    action=Action.COMPLETE;
+                    return Action.COMPLETE;
-                    break;
             }
         }
-
+        finally
-        if (read_interested)
+        {
-            _channel.asyncReadFillInterested();
+            if (read_interested)
-
+                _channel.asyncReadFillInterested();
-        return action;
+        }
     }
 
     public void dispatch(ServletContext context, String path)
@@ -930,7 +925,7 @@ public class HttpChannelState
             _state=State.IDLE;
             _async=Async.NOT_ASYNC;
             _initial=true;
-            _asyncRead=AsyncRead.NONE;
+            _asyncRead=AsyncRead.IDLE;
             _asyncWritePossible=false;
             _timeoutMs=DEFAULT_TIMEOUT;
             _event=null;
@@ -957,7 +952,7 @@ public class HttpChannelState
             _state=State.UPGRADED;
             _async=Async.NOT_ASYNC;
             _initial=true;
-            _asyncRead=AsyncRead.NONE;
+            _asyncRead=AsyncRead.IDLE;
             _asyncWritePossible=false;
             _timeoutMs=DEFAULT_TIMEOUT;
             _event=null;
@@ -1148,10 +1143,8 @@ public class HttpChannelState
             
             switch(_asyncRead)
             {
-                case NONE:
+                case IDLE:
-                case AVAILABLE:
                 case READY:
-                case NEEDED:
                     if (_state==State.ASYNC_WAIT)
                     {
                         interested=true;
@@ -1159,17 +1152,15 @@ public class HttpChannelState
                     }
                     else
                     {
-                        _asyncRead=AsyncRead.NEEDED;
+                        _asyncRead=AsyncRead.REGISTER;
                     }
                     break;
 
+                case REGISTER:
                 case REGISTERED:
                 case POSSIBLE:
                 case PRODUCING:
                     break;
-                    
-                default:
-                    throw new IllegalStateException(toStringLocked()); 
             }
         }
 
@@ -1184,31 +1175,26 @@ public class HttpChannelState
      * is returned.
      * @return True IFF the channel was unready and in ASYNC_WAIT state
      */
-    public boolean onDataAvailable()
+    public boolean onContentAdded()
     {
         boolean woken=false;
         try(Locker.Lock lock= _locker.lock())
         {
             if (LOG.isDebugEnabled())
-                LOG.debug("onReadPossible {}",toStringLocked());
+                LOG.debug("onContentAdded {}",toStringLocked());
             
             switch(_asyncRead)
             {
-                case NONE:
+                case IDLE:
-                    _asyncRead=AsyncRead.AVAILABLE;
+                case READY:
-                    break;
-                    
-                case AVAILABLE:
                     break;
                     
                 case PRODUCING:
                     _asyncRead=AsyncRead.READY;
                     break;
                     
-                case NEEDED:
+                case REGISTER:
                 case REGISTERED:
-                case POSSIBLE:
-                case READY:
                     _asyncRead=AsyncRead.READY;
                     if (_state==State.ASYNC_WAIT)
                     {
@@ -1216,8 +1202,8 @@ public class HttpChannelState
                         _state=State.ASYNC_WOKEN;
                     }
                     break;
-                    
+
-                default:
+                case POSSIBLE:
                     throw new IllegalStateException(toStringLocked());
             }
         }
@@ -1227,7 +1213,7 @@ public class HttpChannelState
     /**
      * Called to signal that the channel is ready for a callback.
      * This is similar to calling {@link #onReadUnready()} followed by
-     * {@link #onDataAvailable()}, except that as content is already
+     * {@link #onContentAdded()}, except that as content is already
      * available, read interest is never set.
      * @return true if woken
      */
@@ -1241,8 +1227,7 @@ public class HttpChannelState
 
             switch(_asyncRead)
             {
-                case NONE:
+                case IDLE:
-                case AVAILABLE:
                     _asyncRead=AsyncRead.READY;
                     if (_state==State.ASYNC_WAIT)
                     {
@@ -1269,7 +1254,7 @@ public class HttpChannelState
         try(Locker.Lock lock= _locker.lock())
         {
             if (LOG.isDebugEnabled())
-                LOG.debug("onReadReady {}",toStringLocked());
+                LOG.debug("onReadPossible {}",toStringLocked());
 
             switch(_asyncRead)
             {
@@ -1288,7 +1273,7 @@ public class HttpChannelState
         }
         return woken;
     }
-    
+
     /**
      * Called to signal that a read has read -1.
      * Will wake if the read was called while in ASYNC_WAIT state

jetty-server/src/main/java/org/eclipse/jetty/server/HttpInput.java

@@ -260,9 +260,9 @@ public class HttpInput extends ServletInputStream implements Runnable
         int l;
         synchronized (_inputQ)
         {
-            // Setup blocking only if not async
             if (!isAsync())
             {
+                // Setup blocking only if not async
                 if (_blockUntil == 0)
                 {
                     long blockingTimeout = getBlockingTimeout();
@@ -306,8 +306,8 @@ public class HttpInput extends ServletInputStream implements Runnable
                     // Not blocking, so what should we return?
                     l = _state.noContent();
                     
-                    // If EOF do we need to wake for allDataRead callback?
                     if (l<0)
+                        // If EOF do we need to wake for allDataRead callback?
                         wake = _channelState.onReadEof();
                     break;
                 }
@@ -577,7 +577,7 @@ public class HttpInput extends ServletInputStream implements Runnable
             if (_listener == null)
                 _inputQ.notify();
             else
-                woken = _channelState.onDataAvailable();
+                woken = _channelState.onContentAdded();
         }
         return woken;
     }
@@ -612,7 +612,7 @@ public class HttpInput extends ServletInputStream implements Runnable
                 if (_listener == null)
                     _inputQ.notify();
                 else
-                    woken = _channelState.onDataAvailable();
+                    woken = _channelState.onContentAdded();
             }
         }
         return woken;
@@ -800,7 +800,7 @@ public class HttpInput extends ServletInputStream implements Runnable
             if (_listener == null)
                 _inputQ.notify();
             else
-                woken = _channelState.onDataAvailable();
+                woken = _channelState.onContentAdded();
         }
 
         return woken;

jetty-server/src/main/java/org/eclipse/jetty/server/Response.java

@@ -27,6 +27,7 @@ import java.util.EnumSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import javax.servlet.RequestDispatcher;
@@ -69,20 +70,11 @@ public class Response implements HttpServletResponse
 {
     private static final Logger LOG = Log.getLogger(Response.class);
     private static final String __COOKIE_DELIM="\",;\\ \t";
-    private final static String __01Jan1970_COOKIE = DateGenerator.formatCookieDate(0).trim();
+    private static final String __01Jan1970_COOKIE = DateGenerator.formatCookieDate(0).trim();
-    private final static int __MIN_BUFFER_SIZE = 1;
+    private static final int __MIN_BUFFER_SIZE = 1;
-    private final static HttpField __EXPIRES_01JAN1970 = new PreEncodedHttpField(HttpHeader.EXPIRES,DateGenerator.__01Jan1970);
+    private static final HttpField __EXPIRES_01JAN1970 = new PreEncodedHttpField(HttpHeader.EXPIRES,DateGenerator.__01Jan1970);
-
-
     // Cookie building buffer. Reduce garbage for cookie using applications
-    private static final ThreadLocal<StringBuilder> __cookieBuilder = new ThreadLocal<StringBuilder>()
+    private static final ThreadLocal<StringBuilder> __cookieBuilder = ThreadLocal.withInitial(() -> new StringBuilder(128));
-    {
-       @Override
-       protected StringBuilder initialValue()
-       {
-          return new StringBuilder(128);
-       }
-    };
 
     public enum OutputType
     {
@@ -116,11 +108,11 @@ public class Response implements HttpServletResponse
     private OutputType _outputType = OutputType.NONE;
     private ResponseWriter _writer;
     private long _contentLength = -1;
+    private Supplier<HttpFields> trailers;
 
-    private enum EncodingFrom { NOT_SET, INFERRED, SET_LOCALE, SET_CONTENT_TYPE, SET_CHARACTER_ENCODING };
+    private enum EncodingFrom { NOT_SET, INFERRED, SET_LOCALE, SET_CONTENT_TYPE, SET_CHARACTER_ENCODING }
     private static final EnumSet<EncodingFrom> __localeOverride = EnumSet.of(EncodingFrom.NOT_SET,EncodingFrom.INFERRED);
     private static final EnumSet<EncodingFrom> __explicitCharset = EnumSet.of(EncodingFrom.SET_LOCALE,EncodingFrom.SET_CHARACTER_ENCODING);
-    
 
     public Response(HttpChannel channel, HttpOutput out)
     {
@@ -686,7 +678,7 @@ public class Response implements HttpServletResponse
     /**
      * Sends a response with one of the 300 series redirection codes.
      * @param code the redirect status code
-     * @param location the location to send in <code>Location</code> headers
+     * @param location the location to send in {@code Location} headers
      * @throws IOException if unable to send the redirect
      */
     public void sendRedirect(int code, String location) throws IOException
@@ -765,7 +757,7 @@ public class Response implements HttpServletResponse
             if (HttpHeader.CONTENT_LENGTH == name)
             {
                 if (value == null)
-                    _contentLength = -1l;
+                    _contentLength = -1L;
                 else
                     _contentLength = Long.parseLong(value);
             }
@@ -790,7 +782,7 @@ public class Response implements HttpServletResponse
             if (HttpHeader.CONTENT_LENGTH.is(name))
             {
                 if (value == null)
-                    _contentLength = -1l;
+                    _contentLength = -1L;
                 else
                     _contentLength = Long.parseLong(value);
             }
@@ -800,8 +792,7 @@ public class Response implements HttpServletResponse
     @Override
     public Collection<String> getHeaderNames()
     {
-        final HttpFields fields = _fields;
+        return _fields.getFieldNamesCollection();
-        return fields.getFieldNamesCollection();
     }
 
     @Override
@@ -813,8 +804,7 @@ public class Response implements HttpServletResponse
     @Override
     public Collection<String> getHeaders(String name)
     {
-        final HttpFields fields = _fields;
+        Collection<String> i = _fields.getValuesList(name);
-        Collection<String> i = fields.getValuesList(name);
         if (i == null)
             return Collections.emptyList();
         return i;
@@ -1290,7 +1280,7 @@ public class Response implements HttpServletResponse
         }
 
         if (preserveCookies)
-            cookies.forEach(f->_fields.add(f));
+            cookies.forEach(_fields::add);
         else
         {
             Request request = getHttpChannel().getRequest();
@@ -1320,10 +1310,20 @@ public class Response implements HttpServletResponse
         _out.resetBuffer();
     }
 
+    public void setTrailers(Supplier<HttpFields> trailers)
+    {
+        this.trailers = trailers;
+    }
+
+    public Supplier<HttpFields> getTrailers()
+    {
+        return trailers;
+    }
+
     protected MetaData.Response newResponseMetaData()
     {
         MetaData.Response info = new MetaData.Response(_channel.getRequest().getHttpVersion(), getStatus(), getReason(), _fields, getLongContentLength());
-        // TODO info.setTrailerSupplier(trailers);
+        info.setTrailerSupplier(getTrailers());
         return info;
     }
 

jetty-server/src/test/java/org/eclipse/jetty/server/HttpInputAsyncStateTest.java

@@ -120,9 +120,9 @@ public class HttpInputAsyncStateTest
             }
 
             @Override
-            public boolean onDataAvailable()
+            public boolean onContentAdded()
             {
-                boolean wake = super.onDataAvailable();
+                boolean wake = super.onContentAdded();
                 __history.add("onReadPossible "+wake);
                 return wake;
             }

jetty-server/src/test/java/org/eclipse/jetty/server/HttpInputTest.java

@@ -111,10 +111,10 @@ public class HttpInputTest
             }
 
             @Override
-            public boolean onDataAvailable()
+            public boolean onContentAdded()
             {
                 _history.add("s.onDataAvailable");
-                return super.onDataAvailable();
+                return super.onContentAdded();
             }
 
             @Override

jetty-server/src/test/java/org/eclipse/jetty/server/HttpTrailersTest.java

@@ -1,170 +0,0 @@
-//
-//  ========================================================================
-//  Copyright (c) 1995-2017 Mort Bay Consulting Pty. Ltd.
-//  ------------------------------------------------------------------------
-//  All rights reserved. This program and the accompanying materials
-//  are made available under the terms of the Eclipse Public License v1.0
-//  and Apache License v2.0 which accompanies this distribution.
-//
-//      The Eclipse Public License is available at
-//      http://www.eclipse.org/legal/epl-v10.html
-//
-//      The Apache License v2.0 is available at
-//      http://www.opensource.org/licenses/apache2.0.php
-//
-//  You may elect to redistribute this code under either of these licenses.
-//  ========================================================================
-//
-
-package org.eclipse.jetty.server;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.Socket;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-
-import javax.servlet.ServletException;
-import javax.servlet.ServletInputStream;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.eclipse.jetty.http.HttpFields;
-import org.eclipse.jetty.http.HttpStatus;
-import org.eclipse.jetty.http.HttpTester;
-import org.eclipse.jetty.server.handler.AbstractHandler;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class HttpTrailersTest
-{
-    private Server server;
-    private ServerConnector connector;
-
-    private void start(Handler handler) throws Exception
-    {
-        server = new Server();
-        connector = new ServerConnector(server);
-        server.addConnector(connector);
-        server.setHandler(handler);
-        server.start();
-    }
-
-    @After
-    public void dispose() throws Exception
-    {
-        if (server != null)
-            server.stop();
-    }
-
-    @Test
-    public void testServletRequestTrailers() throws Exception
-    {
-        String trailerName = "Trailer";
-        String trailerValue = "value";
-        start(new AbstractHandler.ErrorDispatchHandler()
-        {
-            @Override
-            protected void doNonErrorHandle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
-            {
-                jettyRequest.setHandled(true);
-
-                // Read the content first.
-                ServletInputStream input = jettyRequest.getInputStream();
-                while (true)
-                {
-                    int read = input.read();
-                    if (read < 0)
-                        break;
-                }
-
-                // Now the trailers can be accessed.
-                HttpFields trailers = jettyRequest.getTrailers();
-                Assert.assertNotNull(trailers);
-                Assert.assertEquals(trailerValue, trailers.get(trailerName));
-            }
-        });
-
-        try (Socket client = new Socket("localhost", connector.getLocalPort()))
-        {
-            client.setSoTimeout(5000);
-
-            String request = "" +
-                    "GET / HTTP/1.1\r\n" +
-                    "Host: localhost\r\n" +
-                    "Transfer-Encoding: chunked\r\n" +
-                    "\r\n" +
-                    "0\r\n" +
-                    trailerName + ": " + trailerValue + "\r\n" +
-                    "\r\n";
-            OutputStream output = client.getOutputStream();
-            output.write(request.getBytes(StandardCharsets.UTF_8));
-            output.flush();
-
-            HttpTester.Response response = HttpTester.parseResponse(HttpTester.from(client.getInputStream()));
-            Assert.assertNotNull(response);
-            Assert.assertEquals(HttpStatus.OK_200, response.getStatus());
-        }
-    }
-
-    @Test
-    public void testHugeTrailer() throws Exception
-    {
-        start(new AbstractHandler.ErrorDispatchHandler()
-        {
-            @Override
-            protected void doNonErrorHandle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
-            {
-                jettyRequest.setHandled(true);
-
-                try
-                {
-                    // EOF will not be reached because of the huge trailer.
-                    ServletInputStream input = jettyRequest.getInputStream();
-                    while (true)
-                    {
-                        int read = input.read();
-                        if (read < 0)
-                            break;
-                    }
-                    Assert.fail();
-                }
-                catch (IOException x)
-                {
-                    // Expected.
-                }
-            }
-        });
-
-        char[] huge = new char[1024 * 1024];
-        Arrays.fill(huge, 'X');
-        try (Socket client = new Socket("localhost", connector.getLocalPort()))
-        {
-            client.setSoTimeout(5000);
-
-            try
-            {
-                String request = "" +
-                    "GET / HTTP/1.1\r\n" +
-                    "Host: localhost\r\n" +
-                    "Transfer-Encoding: chunked\r\n" +
-                    "\r\n" +
-                    "0\r\n" +
-                    "Trailer: " + new String(huge) + "\r\n" +
-                    "\r\n";
-                OutputStream output = client.getOutputStream();
-                output.write(request.getBytes(StandardCharsets.UTF_8));
-                output.flush();
-                
-                HttpTester.Response response = HttpTester.parseResponse(HttpTester.from(client.getInputStream()));
-                Assert.assertNotNull(response);
-                Assert.assertEquals(HttpStatus.OK_200, response.getStatus());
-            }
-            catch(Exception e)
-            {
-                // May be thrown if write fails and error handling is aborted
-            }
-        }
-    }
-}

jetty-util/src/main/config/modules/console-capture.mod

@@ -19,7 +19,7 @@ resources/
 # jetty.console-capture.dir=logs
 
 ## Whether to append to existing file
-# jetty.console-capture.append=false
+# jetty.console-capture.append=true
 
 ## How many days to retain old log files
 # jetty.console-capture.retainDays=90

jetty-util/src/main/java/org/eclipse/jetty/util/BufferUtil.java

@@ -420,11 +420,11 @@ public class BufferUtil
     /* ------------------------------------------------------------ */
     /**
      * Like append, but does not throw {@link BufferOverflowException}
-     * @param to Buffer is flush mode
+     * @param to Buffer The buffer to fill to. The buffer will be flipped to fill mode and then flipped back to flush mode.
-     * @param b bytes to fill
+     * @param b bytes The bytes to fill
-     * @param off offset into byte
+     * @param off offset into bytes
      * @param len length to fill
-     * @return The position of the valid data before the flipped position.
+     * @return the number of bytes taken from the buffer.
      */
     public static int fill(ByteBuffer to, byte[] b, int off, int len)
     {

jetty-util/src/main/java/org/eclipse/jetty/util/Callback.java

@@ -119,6 +119,11 @@ public interface Callback extends Invocable
             this.callback = nested.callback;
         }
 
+        public Callback getCallback()
+        {
+            return callback;
+        }
+
         @Override
         public void succeeded()
         {

jetty-util/src/main/java/org/eclipse/jetty/util/TypeUtil.java

@@ -34,8 +34,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-import javax.servlet.ServletContainerInitializer;
-
 import org.eclipse.jetty.util.annotation.Name;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
@@ -723,7 +721,7 @@ public class TypeUtil
         {
             try
             {
-                return Resource.newResource(URIUtil.getJarSource(url.toString()));
+                return Resource.newResource(URIUtil.getJarSource(url.toURI()));
             }
             catch(Exception e)
             {

jetty-util/src/main/java/org/eclipse/jetty/util/component/AbstractLifeCycle.java

@@ -225,7 +225,7 @@ public abstract class AbstractLifeCycle implements LifeCycle
     {
         this._stopTimeout = stopTimeout;
     }
-
+    
     public static abstract class AbstractLifeCycleListener implements LifeCycle.Listener
     {
         @Override public void lifeCycleFailure(LifeCycle event, Throwable cause) {}

jetty-util/src/main/java/org/eclipse/jetty/util/component/LifeCycle.java

@@ -122,4 +122,47 @@ public interface LifeCycle
         public void lifeCycleStopping(LifeCycle event);
         public void lifeCycleStopped(LifeCycle event);
     }
+    
+    
+    /**
+     * Utility to start an object if it is a LifeCycle and to convert
+     * any exception thrown to a {@link RuntimeException}
+     * @param object The instance to start.
+     * @throws RuntimeException if the call to start throws an exception.
+     */
+    public static void start(Object object)
+    {
+        if (object instanceof LifeCycle)
+        {
+            try
+            {
+                ((LifeCycle)object).start();
+            }
+            catch(Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+    
+    /**
+     * Utility to stop an object if it is a LifeCycle and to convert
+     * any exception thrown to a {@link RuntimeException}
+     * @param object The instance to stop.
+     * @throws RuntimeException if the call to stop throws an exception.
+     */
+    public static void stop(Object object)
+    {
+        if (object instanceof LifeCycle)
+        {
+            try
+            {
+                ((LifeCycle)object).stop();
+            }
+            catch(Exception e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+    }
 }

jetty-util/src/main/java/org/eclipse/jetty/util/thread/ExecutionStrategy.java

@@ -25,6 +25,7 @@ import java.util.concurrent.RejectedExecutionException;
 import org.eclipse.jetty.util.Loader;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
+import org.eclipse.jetty.util.thread.strategy.EatWhatYouKill;
 import org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume;
 
 /**
@@ -73,55 +74,4 @@ public interface ExecutionStrategy
         Runnable produce();
     }
     
-    
-    /**
-     * <p>A factory for {@link ExecutionStrategy}.</p>
-     */
-    public static interface Factory
-    {
-        /**
-         * <p>Creates a new {@link ExecutionStrategy}.</p>
-         *
-         * @param producer the execution strategy producer
-         * @param executor the execution strategy executor
-         * @return a new {@link ExecutionStrategy}
-         */
-        public ExecutionStrategy newExecutionStrategy(Producer producer, Executor executor);
-
-        /**
-         * @return the default {@link ExecutionStrategy}
-         */
-        public static Factory getDefault()
-        {
-            return DefaultExecutionStrategyFactory.INSTANCE;
-        }
-    }
-
-    public static class DefaultExecutionStrategyFactory implements Factory
-    {
-        private static final Logger LOG = Log.getLogger(Factory.class);
-        private static final Factory INSTANCE = new DefaultExecutionStrategyFactory();
-
-        @Override
-        public ExecutionStrategy newExecutionStrategy(Producer producer, Executor executor)
-        {
-            String strategy = System.getProperty(producer.getClass().getName() + ".ExecutionStrategy");
-            if (strategy != null)
-            {
-                try
-                {
-                    Class<? extends ExecutionStrategy> c = Loader.loadClass(strategy);
-                    Constructor<? extends ExecutionStrategy> m = c.getConstructor(Producer.class, Executor.class);
-                    LOG.info("Use {} for {}", c.getSimpleName(), producer.getClass().getName());
-                    return m.newInstance(producer, executor);
-                }
-                catch (Exception e)
-                {
-                    LOG.warn(e);
-                }
-            }
-
-            return new ExecuteProduceConsume(producer, executor);
-        }
-    }
 }

jetty-util/src/main/java/org/eclipse/jetty/util/thread/Invocable.java

@@ -18,7 +18,13 @@
 
 package org.eclipse.jetty.util.thread;
 
+import java.io.Closeable;
 import java.util.concurrent.Callable;
+import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
+
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
 
 /**
  * <p>A task (typically either a {@link Runnable} or {@link Callable}
@@ -151,7 +157,7 @@ public interface Invocable
                 break;
 
             case EITHER:
-                if (getInvocationType(task) == InvocationType.EITHER && preferredInvocationType == InvocationType.NON_BLOCKING)
+                if (preferredInvocationType == InvocationType.NON_BLOCKING)
                     return () -> invokeNonBlocking(task);
                 break;
         }
@@ -179,4 +185,85 @@ public interface Invocable
     {
         return InvocationType.BLOCKING;
     }
+    
+    /**
+     * An Executor wrapper that knows about Invocable
+     *
+     */
+    public static class InvocableExecutor implements Executor
+    {
+        private static final Logger LOG = Log.getLogger(InvocableExecutor.class);
+
+        private final Executor _executor;
+        private final InvocationType _preferredInvocationForExecute;
+        private final InvocationType _preferredInvocationForInvoke;
+
+        public InvocableExecutor(Executor executor,InvocationType preferred)
+        {
+            this(executor,preferred,preferred);
+        }
+        
+        public InvocableExecutor(Executor executor,InvocationType preferredInvocationForExecute,InvocationType preferredInvocationForIvoke)
+        {
+            _executor=executor;
+            _preferredInvocationForExecute=preferredInvocationForExecute;
+            _preferredInvocationForInvoke=preferredInvocationForIvoke;
+        }
+
+        public Invocable.InvocationType getPreferredInvocationType()
+        {
+            return _preferredInvocationForInvoke;
+        }
+
+        public void invoke(Runnable task)
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("{} invoke  {}", this, task);
+            Invocable.invokePreferred(task,_preferredInvocationForInvoke);
+            if (LOG.isDebugEnabled())
+                LOG.debug("{} invoked {}", this, task);
+        }
+        
+        public void execute(Runnable task)
+        {
+            tryExecute(task,_preferredInvocationForExecute);
+        }
+
+        public void execute(Runnable task, InvocationType preferred)
+        {
+            tryExecute(task,preferred);
+        }
+        
+        public boolean tryExecute(Runnable task)
+        {
+            return tryExecute(task,_preferredInvocationForExecute);
+        }
+        
+        public boolean tryExecute(Runnable task, InvocationType preferred)
+        {
+            try
+            {
+                _executor.execute(Invocable.asPreferred(task,preferred));
+                return true;
+            }
+            catch(RejectedExecutionException e)
+            {
+                // If we cannot execute, then close the task
+                LOG.debug(e);
+                LOG.warn("Rejected execution of {}",task);
+                try
+                {
+                    if (task instanceof Closeable)
+                        ((Closeable)task).close();
+                }
+                catch (Exception x)
+                {
+                    e.addSuppressed(x);
+                    LOG.warn(e);
+                }
+            }
+            return false;
+        }
+
+    }
 }

jetty-util/src/main/java/org/eclipse/jetty/util/thread/strategy/EatWhatYouKill.java

@@ -0,0 +1,374 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2017 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.util.thread.strategy;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.locks.Condition;
+
+import org.eclipse.jetty.util.component.AbstractLifeCycle;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.Logger;
+import org.eclipse.jetty.util.thread.ExecutionStrategy;
+import org.eclipse.jetty.util.thread.Invocable;
+import org.eclipse.jetty.util.thread.Invocable.InvocableExecutor;
+import org.eclipse.jetty.util.thread.Invocable.InvocationType;
+import org.eclipse.jetty.util.thread.Locker;
+import org.eclipse.jetty.util.thread.Locker.Lock;
+
+/**
+ * <p>A strategy where the thread that produces will run the resulting task if it 
+ * is possible to do so without thread starvation.</p>
+ * 
+ * <p>This strategy preemptively dispatches a thread as a pending producer, so that
+ * when a thread produces a task it can immediately run the task and let the pending
+ * producer thread take over producing.  If necessary another thread will be dispatched
+ * to replace the pending producing thread.   When operating in this pattern, the 
+ * sub-strategy is called Execute Produce Consume (EPC)
+ * </p>
+ * <p>However, if the task produced uses the {@link Invocable} API to indicate that 
+ * it will not block, then the strategy will run it directly, regardless of the 
+ * presence of a pending producing thread and then resume producing after the 
+ * task has completed. This sub-strategy is also used if the strategy has been
+ * configured with a maximum of 0 pending threads and the thread currently producing
+ * does not use the {@link Invocable} API to indicate that it will not block.
+ * When operating in this pattern, the sub-strategy is called
+ * ProduceConsume (PC).
+ * </p>
+ * <p>If there is no pending producer thread available and if the task has not 
+ * indicated it is non-blocking, then this strategy will dispatch the execution of
+ * the task and immediately continue producing.  When operating in this pattern, the
+ * sub-strategy is called ProduceExecuteConsume (PEC).
+ * </p>
+ * 
+ */
+public class EatWhatYouKill extends AbstractLifeCycle implements ExecutionStrategy, Runnable
+{
+    private static final Logger LOG = Log.getLogger(EatWhatYouKill.class);
+
+    enum State { IDLE, PRODUCING, REPRODUCING };
+    
+    private final Locker _locker = new Locker();
+    private State _state = State.IDLE;
+    private final Runnable _runProduce = new RunProduce();
+    private final Producer _producer;
+    private final InvocableExecutor _executor;
+    private int _pendingProducersMax;
+    private int _pendingProducers;
+    private int _pendingProducersDispatched;
+    private int _pendingProducersSignalled;
+    private Condition _produce = _locker.newCondition();
+
+    public EatWhatYouKill(Producer producer, Executor executor)
+    {
+        this(producer,executor,InvocationType.NON_BLOCKING,InvocationType.BLOCKING);
+    }
+
+    public EatWhatYouKill(Producer producer, Executor executor, int maxProducersPending )
+    {
+        this(producer,executor,InvocationType.NON_BLOCKING,InvocationType.BLOCKING);
+    }
+    
+    public EatWhatYouKill(Producer producer, Executor executor, InvocationType preferredInvocationPEC, InvocationType preferredInvocationEPC)
+    {
+        this(producer,executor,preferredInvocationPEC,preferredInvocationEPC,Integer.getInteger("org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.maxProducersPending",1));
+    }
+    
+    public EatWhatYouKill(Producer producer, Executor executor, InvocationType preferredInvocationPEC, InvocationType preferredInvocationEPC, int maxProducersPending )
+    {
+        _producer = producer;
+        _pendingProducersMax = maxProducersPending;
+        _executor = new InvocableExecutor(executor,preferredInvocationPEC,preferredInvocationEPC);
+    }
+
+    @Override
+    public void produce()
+    {
+        boolean produce;
+        try (Lock locked = _locker.lock())
+        {
+            switch(_state)
+            {
+                case IDLE:
+                    _state = State.PRODUCING;
+                    produce = true;
+                    break;
+                    
+                case PRODUCING:
+                    _state = State.REPRODUCING;
+                    produce = false;
+                    break;
+                    
+                default:     
+                    produce = false;   
+            }
+        }
+
+        if (LOG.isDebugEnabled())
+            LOG.debug("{} execute {}", this, produce);
+
+        if (produce)
+            doProduce();
+    }
+
+    @Override
+    public void dispatch()
+    {
+        boolean dispatch = false;
+        try (Lock locked = _locker.lock())
+        {
+            switch(_state)
+            {
+                case IDLE:
+                    dispatch = true;
+                    break;
+                    
+                case PRODUCING:
+                    _state = State.REPRODUCING;
+                    dispatch = false;
+                    break;
+                    
+                default:     
+                    dispatch = false;   
+            }
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("{} dispatch {}", this, dispatch);
+        if (dispatch)
+            _executor.execute(_runProduce,InvocationType.BLOCKING);
+    }
+
+    @Override
+    public void run()
+    {
+        if (LOG.isDebugEnabled())
+            LOG.debug("{} run", this);
+        if (!isRunning())
+            return;
+        boolean producing = false;
+        try (Lock locked = _locker.lock())
+        {
+            _pendingProducersDispatched--;
+            _pendingProducers++;
+
+            loop: while (isRunning())
+            {
+                try
+                {
+                    _produce.await();
+
+                    if (_pendingProducersSignalled==0)
+                    {
+                        // spurious wakeup!
+                        continue loop;
+                    } 
+
+                    _pendingProducersSignalled--;
+                    if (_state == State.IDLE)                    
+                    {
+                        _state = State.PRODUCING;
+                        producing = true;
+                    } 
+                }
+                catch (InterruptedException e)
+                {
+                    LOG.debug(e);
+                    _pendingProducers--;
+                }
+               
+                break loop;
+            }     
+        }
+
+        if (producing)
+            doProduce();
+    }
+
+    private void doProduce()
+    {
+        boolean may_block_caller = !Invocable.isNonBlockingInvocation();
+        if (LOG.isDebugEnabled())
+            LOG.debug("{} produce {}", this,may_block_caller?"non-blocking":"blocking");
+
+        producing: while (isRunning())
+        {
+            // If we got here, then we are the thread that is producing.
+            Runnable task = _producer.produce();
+
+            boolean produce;
+            boolean consume;
+            boolean execute_producer;
+            
+            StringBuilder state = null;
+            
+            try (Lock locked = _locker.lock())
+            {
+                if (LOG.isDebugEnabled())
+                {
+                    state = new StringBuilder();
+                    getString(state);
+                    getState(state);
+                    state.append("->");
+                }
+                
+                // Did we produced a task?
+                if (task == null)
+                {
+                    // There is no task.
+                    // Could another one just have been queued with a produce call?
+                    if (_state==State.REPRODUCING)
+                    {
+                        _state = State.PRODUCING;
+                        continue producing;
+                    }
+
+                    // ... and no additional calls to execute, so we are idle
+                    _state = State.IDLE;
+                    break producing;
+                }
+                
+                // Will we eat our own kill - ie consume the task we just produced?
+                if (Invocable.getInvocationType(task)==InvocationType.NON_BLOCKING)
+                {
+                    // ProduceConsume
+                    produce = true;
+                    consume = true;
+                    execute_producer = false;
+                }
+                else if (may_block_caller && (_pendingProducers>0 || _pendingProducersMax==0))
+                {
+                    // ExecuteProduceConsume (eat what we kill!)
+                    produce = false;
+                    consume = true;
+                    execute_producer = true;
+                    _pendingProducersDispatched++;
+                    _state = State.IDLE;
+                    _pendingProducers--;
+                    _pendingProducersSignalled++;
+                    _produce.signal();
+                }
+                else
+                {
+                    // ProduceExecuteConsume
+                    produce = true;
+                    consume = false;
+                    execute_producer = (_pendingProducersDispatched + _pendingProducers)<_pendingProducersMax;
+                    if (execute_producer)
+                        _pendingProducersDispatched++;
+                }
+                
+                if (LOG.isDebugEnabled())
+                    getState(state);
+                
+            }
+            
+            if (LOG.isDebugEnabled())
+            {
+                LOG.debug("{} {} {}",
+                    state,
+                    consume?(execute_producer?"EPC!":"PC"):"PEC",
+                    task);
+            }
+
+            if (execute_producer)
+                // Spawn a new thread to continue production by running the produce loop.
+                _executor.execute(this);
+            
+            // Run or execute the task.
+            if (consume)
+                _executor.invoke(task);
+            else
+                _executor.execute(task);
+           
+            // Once we have run the task, we can try producing again.
+            if (produce)
+                continue producing;
+
+            try (Lock locked = _locker.lock())
+            {
+                if (_state==State.IDLE)
+                {
+                    _state = State.PRODUCING;
+                    continue producing;
+                }
+            }
+
+            break producing;
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("{} produce exit",this);
+    }
+
+    public Boolean isIdle()
+    {
+        try (Lock locked = _locker.lock())
+        {
+            return _state==State.IDLE;
+        }
+    }
+
+    @Override
+    protected void doStop() throws Exception
+    {
+        try (Lock locked = _locker.lock())
+        {
+            _pendingProducersSignalled=_pendingProducers+_pendingProducersDispatched;
+            _pendingProducers=0;
+            _produce.signalAll();
+        }
+    }
+
+    public String toString()
+    {
+        StringBuilder builder = new StringBuilder();
+        getString(builder);
+        try (Lock locked = _locker.lock())
+        {
+            getState(builder);
+        }
+        return builder.toString();
+    }
+    
+    private void getString(StringBuilder builder)
+    {
+        builder.append(getClass().getSimpleName());
+        builder.append('@');
+        builder.append(Integer.toHexString(hashCode()));
+        builder.append('/');
+        builder.append(_producer);
+        builder.append('/');
+    }
+
+    private void getState(StringBuilder builder)
+    {
+        builder.append(_state);
+        builder.append('/');
+        builder.append(_pendingProducers);
+        builder.append('/');
+        builder.append(_pendingProducersMax);
+    }
+
+    private class RunProduce implements Runnable
+    {
+        @Override
+        public void run()
+        {
+            produce();
+        }
+    }
+}

jetty-util/src/main/java/org/eclipse/jetty/util/thread/strategy/ExecuteProduceConsume.java

@@ -24,6 +24,7 @@ import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 import org.eclipse.jetty.util.thread.ExecutionStrategy;
 import org.eclipse.jetty.util.thread.Invocable;
+import org.eclipse.jetty.util.thread.Invocable.InvocableExecutor;
 import org.eclipse.jetty.util.thread.Invocable.InvocationType;
 import org.eclipse.jetty.util.thread.Locker;
 import org.eclipse.jetty.util.thread.Locker.Lock;
@@ -41,13 +42,14 @@ import org.eclipse.jetty.util.thread.Locker.Lock;
  * does not yet have capacity to consume, which can save memory and exert back
  * pressure on producers.</p>
  */
-public class ExecuteProduceConsume extends ExecutingExecutionStrategy implements ExecutionStrategy, Runnable
+public class ExecuteProduceConsume implements ExecutionStrategy, Runnable
 {
     private static final Logger LOG = Log.getLogger(ExecuteProduceConsume.class);
 
     private final Locker _locker = new Locker();
     private final Runnable _runProduce = new RunProduce();
     private final Producer _producer;
+    private final InvocableExecutor _executor;
     private boolean _idle = true;
     private boolean _execute;
     private boolean _producing;
@@ -61,10 +63,10 @@ public class ExecuteProduceConsume extends ExecutingExecutionStrategy implements
     
     public ExecuteProduceConsume(Producer producer, Executor executor, InvocationType preferred )
     {
-        super(executor,preferred);
         this._producer = producer;
+        _executor = new InvocableExecutor(executor,preferred);
     }
-
+    
     @Override
     public void produce()
     {
@@ -111,7 +113,7 @@ public class ExecuteProduceConsume extends ExecutingExecutionStrategy implements
                 _execute = true;
         }
         if (dispatch)
-            execute(_runProduce);
+            _executor.execute(_runProduce);
     }
 
     @Override
@@ -190,7 +192,7 @@ public class ExecuteProduceConsume extends ExecutingExecutionStrategy implements
                 // Spawn a new thread to continue production by running the produce loop.
                 if (LOG.isDebugEnabled())
                     LOG.debug("{} dispatch", this);
-                if (!execute(this))
+                if (!_executor.tryExecute(this))
                     task = null;
             }
 
@@ -198,7 +200,7 @@ public class ExecuteProduceConsume extends ExecutingExecutionStrategy implements
             if (LOG.isDebugEnabled())
                 LOG.debug("{} run {}", this, task);
             if (task != null)
-                invoke(task);
+                _executor.invoke(task);
             if (LOG.isDebugEnabled())
                 LOG.debug("{} ran {}", this, task);
 
@@ -247,13 +249,4 @@ public class ExecuteProduceConsume extends ExecutingExecutionStrategy implements
             produce();
         }
     }
-
-    public static class Factory implements ExecutionStrategy.Factory
-    {
-        @Override
-        public ExecutionStrategy newExecutionStrategy(Producer producer, Executor executor)
-        {
-            return new ExecuteProduceConsume(producer, executor);
-        }
-    }
 }

jetty-util/src/main/java/org/eclipse/jetty/util/thread/strategy/ExecutingExecutionStrategy.java

@@ -1,88 +0,0 @@
-//
-//  ========================================================================
-//  Copyright (c) 1995-2017 Mort Bay Consulting Pty. Ltd.
-//  ------------------------------------------------------------------------
-//  All rights reserved. This program and the accompanying materials
-//  are made available under the terms of the Eclipse Public License v1.0
-//  and Apache License v2.0 which accompanies this distribution.
-//
-//      The Eclipse Public License is available at
-//      http://www.eclipse.org/legal/epl-v10.html
-//
-//      The Apache License v2.0 is available at
-//      http://www.opensource.org/licenses/apache2.0.php
-//
-//  You may elect to redistribute this code under either of these licenses.
-//  ========================================================================
-//
-
-package org.eclipse.jetty.util.thread.strategy;
-
-import java.io.Closeable;
-import java.util.concurrent.Executor;
-import java.util.concurrent.RejectedExecutionException;
-
-import org.eclipse.jetty.util.log.Log;
-import org.eclipse.jetty.util.log.Logger;
-import org.eclipse.jetty.util.thread.ExecutionStrategy;
-import org.eclipse.jetty.util.thread.Invocable;
-
-/**
- * <p>Base class for strategies that need to execute a task by submitting it to an {@link Executor}.</p>
- * <p>If the submission to the {@code Executor} is rejected (via a {@link RejectedExecutionException}),
- * the task is tested whether it implements {@link Closeable}; if it does, then {@link Closeable#close()}
- * is called on the task object.</p>
- */
-public abstract class ExecutingExecutionStrategy implements ExecutionStrategy
-{
-    private static final Logger LOG = Log.getLogger(ExecutingExecutionStrategy.class);
-
-    private final Executor _executor;
-    private final Invocable.InvocationType _preferredInvocationType;
-
-    protected ExecutingExecutionStrategy(Executor executor,Invocable.InvocationType preferred)
-    {
-        _executor=executor;
-        _preferredInvocationType=preferred;
-    }
-
-    public Invocable.InvocationType getPreferredInvocationType()
-    {
-        return _preferredInvocationType;
-    }
-
-    public void invoke(Runnable task)
-    {
-        if (LOG.isDebugEnabled())
-            LOG.debug("{} invoke  {}", this, task);
-        Invocable.invokePreferred(task,_preferredInvocationType);
-        if (LOG.isDebugEnabled())
-            LOG.debug("{} invoked {}", this, task);
-    }
-    
-    protected boolean execute(Runnable task)
-    {
-        try
-        {
-            _executor.execute(Invocable.asPreferred(task,_preferredInvocationType));
-            return true;
-        }
-        catch(RejectedExecutionException e)
-        {
-            // If we cannot execute, then close the task and keep producing.
-            LOG.debug(e);
-            LOG.warn("Rejected execution of {}",task);
-            try
-            {
-                if (task instanceof Closeable)
-                    ((Closeable)task).close();
-            }
-            catch (Exception x)
-            {
-                e.addSuppressed(x);
-                LOG.warn(e);
-            }
-        }
-        return false;
-    }
-}

jetty-util/src/main/java/org/eclipse/jetty/util/thread/strategy/ProduceConsume.java

@@ -106,15 +106,6 @@ public class ProduceConsume implements ExecutionStrategy, Runnable
         produce();
     }
 
-    public static class Factory implements ExecutionStrategy.Factory
-    {
-        @Override
-        public ExecutionStrategy newExecutionStrategy(Producer producer, Executor executor)
-        {
-            return new ProduceConsume(producer, executor);
-        }
-    }
-
     private enum State
     {
         IDLE, PRODUCE, EXECUTE

jetty-util/src/main/java/org/eclipse/jetty/util/thread/strategy/ProduceExecuteConsume.java

@@ -23,7 +23,8 @@ import java.util.concurrent.Executor;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 import org.eclipse.jetty.util.thread.ExecutionStrategy;
-import org.eclipse.jetty.util.thread.Invocable;
+import org.eclipse.jetty.util.thread.Invocable.InvocationType;
+import org.eclipse.jetty.util.thread.Invocable.InvocableExecutor;
 import org.eclipse.jetty.util.thread.Locker;
 import org.eclipse.jetty.util.thread.Locker.Lock;
 
@@ -31,23 +32,24 @@ import org.eclipse.jetty.util.thread.Locker.Lock;
  * <p>A strategy where the caller thread iterates over task production, submitting each
  * task to an {@link Executor} for execution.</p>
  */
-public class ProduceExecuteConsume extends ExecutingExecutionStrategy implements ExecutionStrategy
+public class ProduceExecuteConsume implements ExecutionStrategy
 {
     private static final Logger LOG = Log.getLogger(ProduceExecuteConsume.class);
 
     private final Locker _locker = new Locker();
     private final Producer _producer;
+    private final InvocableExecutor _executor;
     private State _state = State.IDLE;
 
     public ProduceExecuteConsume(Producer producer, Executor executor)
     {
-        this(producer,executor,Invocable.InvocationType.NON_BLOCKING);
+        this(producer,executor,InvocationType.NON_BLOCKING);
     }
     
-    public ProduceExecuteConsume(Producer producer, Executor executor, Invocable.InvocationType preferred)
+    public ProduceExecuteConsume(Producer producer, Executor executor, InvocationType preferred)
     {
-        super(executor,preferred);
+        _producer = producer;
-        this._producer = producer;
+        _executor = new InvocableExecutor(executor,preferred);
     }
 
     @Override
@@ -95,7 +97,7 @@ public class ProduceExecuteConsume extends ExecutingExecutionStrategy implements
             }
 
             // Execute the task.
-            execute(task);
+            _executor.execute(task);
         }        
     }
 
@@ -105,15 +107,6 @@ public class ProduceExecuteConsume extends ExecutingExecutionStrategy implements
         produce();
     }
 
-    public static class Factory implements ExecutionStrategy.Factory
-    {
-        @Override
-        public ExecutionStrategy newExecutionStrategy(Producer producer, Executor executor)
-        {
-            return new ProduceExecuteConsume(producer, executor);
-        }
-    }
-
     private enum State
     {
         IDLE, PRODUCE, EXECUTE

jetty-util/src/test/java/org/eclipse/jetty/util/thread/strategy/ExecutionStrategyTest.java

@@ -0,0 +1,218 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2017 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.util.thread.strategy;
+
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.eclipse.jetty.util.component.LifeCycle;
+import org.eclipse.jetty.util.thread.ExecutionStrategy;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.eclipse.jetty.util.thread.ExecutionStrategy.Producer;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class ExecutionStrategyTest
+{
+    @Parameterized.Parameters(name = "{0}")
+    public static Iterable<Object[]> data()
+    {
+        return Arrays.asList(new Object[][]{
+            {ProduceExecuteConsume.class},
+            {ExecuteProduceConsume.class},
+            {EatWhatYouKill.class}
+        });
+    }
+
+    QueuedThreadPool _threads = new QueuedThreadPool(20);
+    Class<? extends ExecutionStrategy> _strategyClass;
+    ExecutionStrategy _strategy;
+
+    public ExecutionStrategyTest(Class<? extends ExecutionStrategy> strategy)
+    {
+        _strategyClass = strategy;
+    }
+    
+    void newExecutionStrategy(Producer producer, Executor executor) throws Exception
+    {
+        _strategy = _strategyClass.getConstructor(Producer.class,Executor.class).newInstance(producer,executor);
+        LifeCycle.start(_strategy);
+    }
+    
+    @Before
+    public void before() throws Exception
+    {
+        _threads.start();
+    }
+    
+    @After
+    public void after() throws Exception
+    {
+        LifeCycle.stop(_strategy);
+        _threads.stop();
+    }
+    
+    public static abstract class TestProducer implements Producer
+    {
+        @Override
+        public String toString()
+        {
+            return "TestProducer";
+        }
+    }
+    
+    @Test
+    public void idleTest() throws Exception
+    {
+        AtomicInteger count = new AtomicInteger(0);
+        Producer producer = new TestProducer()
+        {
+            @Override
+            public Runnable produce()
+            {
+                count.incrementAndGet();
+                return null;
+            }
+        };
+        
+        newExecutionStrategy(producer,_threads);
+        _strategy.produce();
+        assertThat(count.get(),greaterThan(0));
+    }
+    
+    @Test
+    public void simpleTest() throws Exception
+    {
+        final int TASKS = 3*_threads.getMaxThreads();
+        final CountDownLatch latch = new CountDownLatch(TASKS);
+        Producer producer = new TestProducer()
+        {
+            int tasks = TASKS;
+            @Override
+            public Runnable produce()
+            {
+                if (tasks-->0)
+                {
+                    return new Runnable()
+                    {
+                        @Override
+                        public void run()
+                        {
+                            latch.countDown();
+                        }
+                    };
+                }
+
+                return null;
+            }
+        };
+        
+        newExecutionStrategy(producer,_threads);
+
+        for (int p=0; latch.getCount()>0 && p<TASKS; p++)
+            _strategy.produce();
+
+        assertTrue(latch.await(10,TimeUnit.SECONDS));
+    }
+    
+
+    @Test
+    public void blockingProducerTest() throws Exception
+    {
+        final int TASKS = 3*_threads.getMaxThreads();
+        final BlockingQueue<CountDownLatch> q = new ArrayBlockingQueue<>(500);
+        
+        Producer producer = new TestProducer()
+        {
+            int tasks=TASKS;
+            @Override
+            public Runnable produce()
+            {
+                if (tasks-->0)
+                {
+                    try
+                    {
+                        final CountDownLatch latch = q.poll(10,TimeUnit.SECONDS);
+
+                        if (latch!=null)
+                        {
+                            return new Runnable()
+                            {
+                                @Override
+                                public void run()
+                                {
+                                    latch.countDown();
+                                }
+                            };
+                        }
+                    }
+                    catch(InterruptedException e)
+                    {
+                        e.printStackTrace();
+                    }
+                }
+
+                return null;
+            }
+        };
+        
+        newExecutionStrategy(producer,_threads);
+        _threads.execute(()->_strategy.produce());
+        
+        
+        
+        final CountDownLatch latch = new CountDownLatch(TASKS);
+        _threads.execute(new Runnable()
+        {
+            @Override
+            public void run()
+            {
+                try
+                {
+                    for (int t=TASKS;t-->0;)
+                    {
+                        Thread.sleep(20);
+                        q.offer(latch);
+                        _strategy.produce();
+                    }
+                }
+                catch(Exception e)
+                {
+                    e.printStackTrace();
+                }
+            }
+        });
+
+        assertTrue(latch.await(10,TimeUnit.SECONDS));
+    }
+}

jetty-util/src/test/resources/jetty-logging.properties

@@ -3,3 +3,5 @@ org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog
 #org.eclipse.jetty.util.LEVEL=DEBUG
 
 #org.eclipse.jetty.util.PathWatcher.Noisy.LEVEL=OFF
+
+org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.LEVEL=DEBUG

jetty-webapp/src/main/java/org/eclipse/jetty/webapp/WebAppContext.java

@@ -84,8 +84,8 @@ import org.eclipse.jetty.util.resource.ResourceCollection;
  * The handlers are configured by pluggable configuration classes, with
  * the default being  {@link org.eclipse.jetty.webapp.WebXmlConfiguration} and
  * {@link org.eclipse.jetty.webapp.JettyWebXmlConfiguration}.
- * 
+ *
- * 
+ *
  * <p>
  * The Start/Configuration of a WebAppContext is rather complex so as to allow
  * pluggable behaviour to be added in almost arbitrary ordering.  The
@@ -102,7 +102,7 @@ import org.eclipse.jetty.util.resource.ResourceCollection;
  *         <li>Add all Server class exclusions from this webapps {@link Configurations}</li>
  *         <li>Add all System classes inclusions and exclusions for this webapps {@link Configurations}</li>
  *         <li>Instantiate the WebAppClassLoader (if one not already explicitly set)</li>
- *         <li>{@link Configuration#preConfigure(WebAppContext)} which calls  
+ *         <li>{@link Configuration#preConfigure(WebAppContext)} which calls
  *         {@link Configuration#preConfigure(WebAppContext)} for this webapps {@link Configurations}</li>
  *       </ul>
  *     </li>
@@ -150,7 +150,7 @@ import org.eclipse.jetty.util.resource.ResourceCollection;
  *     </li>
  *     <li>{@link #postConfigure()}</li>
  *   </ul>
- *   
+ *
  * </blockquote>
  *
  */
@@ -179,23 +179,6 @@ public class WebAppContext extends ServletContextHandler implements WebAppClassL
         "org.w3c."                          // javax.xml
     ) ;
 
-    // Find the location of the JVM lib directory
-    public final static String __jvmlib;
-    static
-    {
-        String lib=null;
-        try
-        { 
-            lib=TypeUtil.getLoadedFrom(System.class).getFile().getParentFile().toURI().toString();
-        }
-        catch(Exception e)
-        {
-            LOG.warn(e);
-            lib=null;
-        }
-        __jvmlib=lib;
-    }
-    
     // Server classes are classes that are hidden from being
     // loaded by the web application using system classloader,
     // so if web application needs to load any of such classes,
@@ -309,7 +292,7 @@ public class WebAppContext extends ServletContextHandler implements WebAppClassL
      */
     public WebAppContext(HandlerContainer parent, String contextPath, SessionHandler sessionHandler, SecurityHandler securityHandler, ServletHandler servletHandler, ErrorHandler errorHandler,int options) 
     {
-        // always pass parent as null and then set below, so that any resulting setServer call 
+        // always pass parent as null and then set below, so that any resulting setServer call
         // is done after this instance is constructed.
         super(null,contextPath,sessionHandler, securityHandler, servletHandler, errorHandler,options);
         _scontext = new Context();
@@ -492,7 +475,7 @@ public class WebAppContext extends ServletContextHandler implements WebAppClassL
      * @throws Exception if unable to pre configure
      */
     public void preConfigure() throws Exception
-    {        
+    {
         // Add the known server class inclusions for all known configurations
         for (Configuration configuration : Configurations.getKnown())
             _serverClasses.include(configuration.getServerClasses().getInclusions());
@@ -597,7 +580,7 @@ public class WebAppContext extends ServletContextHandler implements WebAppClassL
             catch(Exception e)
             {
                 mx.add(e);
-            }   
+            }
         }
         _configurations.clear();
         super.destroy();
@@ -708,7 +691,7 @@ public class WebAppContext extends ServletContextHandler implements WebAppClassL
      * from the context.  If the context needs to load these classes, it must have its
      * own copy of them in WEB-INF/lib or WEB-INF/classes.
      * @param serverClasses the server classes pattern
-     * 
+     *
      */
     public void setServerClasspathPattern(ClasspathPattern serverClasses)
     {
@@ -1396,7 +1379,7 @@ public class WebAppContext extends ServletContextHandler implements WebAppClassL
     {
         super.stopContext();
         try
-        {                
+        {
             for (int i=_configurations.size();i-->0;)
                 _configurations.get(i).deconfigure(this);
 

jetty-webapp/src/test/java/org/eclipse/jetty/webapp/URLStreamHandlerUtil.java

@@ -0,0 +1,78 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2017 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.webapp;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.net.URL;
+import java.net.URLStreamHandlerFactory;
+import java.util.Arrays;
+import java.util.Optional;
+
+public final class URLStreamHandlerUtil
+{
+    public static void setFactory(URLStreamHandlerFactory factory)
+    {
+        try
+        {
+            // First, reset the factory field
+            Field factoryField = getURLStreamHandlerFactoryField();
+            factoryField.setAccessible(true);
+            factoryField.set(null, null);
+            
+            if(factory != null)
+            {
+                // Next, set the factory
+                URL.setURLStreamHandlerFactory(factory);
+            }
+        }
+        catch(Throwable ignore)
+        {
+            ignore.printStackTrace(System.err);
+        }
+    }
+    
+    public static URLStreamHandlerFactory getFactory()
+    {
+        try
+        {
+            // First, reset the factory field
+            Field factoryField = getURLStreamHandlerFactoryField();
+            factoryField.setAccessible(true);
+            return (URLStreamHandlerFactory) factoryField.get(null);
+        }
+        catch(Throwable ignore)
+        {
+            return null;
+        }
+    }
+    
+    private static Field getURLStreamHandlerFactoryField()
+    {
+        Optional<Field> optFactoryField = Arrays.stream(URL.class.getDeclaredFields())
+                .filter((f) -> Modifier.isStatic(f.getModifiers()) &&
+                               f.getType().equals(URLStreamHandlerFactory.class))
+                .findFirst();
+        
+        if(optFactoryField.isPresent())
+            return optFactoryField.get();
+        
+        throw new RuntimeException( "Cannot find URLStreamHandlerFactory field in " + URL.class.getName() );
+    }
+}

jetty-webapp/src/test/java/org/eclipse/jetty/webapp/WebAppClassLoaderTest.java

@@ -24,6 +24,7 @@ import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeThat;
 
 import java.lang.instrument.ClassFileTransformer;
 import java.lang.instrument.IllegalClassFormatException;
@@ -51,7 +52,7 @@ public class WebAppClassLoaderTest
 
     private Path testWebappDir;
     private WebAppContext _context;
-    private WebAppClassLoader _loader;
+    protected WebAppClassLoader _loader;
 
     @Before
     public void init() throws Exception
@@ -278,6 +279,9 @@ public class WebAppClassLoaderTest
     @Test
     public void testResources() throws Exception
     {
+        // The existence of a URLStreamHandler changes the behavior
+        assumeThat("No URLStreamHandler in place", URLStreamHandlerUtil.getFactory(), nullValue());
+        
         List<URL> expected = new ArrayList<>();
         List<URL> resources;
         

jetty-webapp/src/test/java/org/eclipse/jetty/webapp/WebAppClassLoaderUrlStreamTest.java

@@ -0,0 +1,113 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2017 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.webapp;
+
+import java.net.URLStreamHandler;
+import java.net.URLStreamHandlerFactory;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.After;
+import org.junit.Before;
+
+public class WebAppClassLoaderUrlStreamTest extends WebAppClassLoaderTest
+{
+    public static class URLHandlers implements URLStreamHandlerFactory
+    {
+        private static final String[] STREAM_HANDLER_PREFIXES;
+        
+        static
+        {
+            STREAM_HANDLER_PREFIXES = new String[]{
+                    "sun.net.www.protocol",
+                    "org.apache.harmony.luni.internal.net.www.protocol",
+                    "javax.net.ssl"
+            };
+        }
+        
+        private Map<String, URLStreamHandler> handlers = new HashMap<>();
+        private ClassLoader loader;
+        
+        public URLHandlers(ClassLoader loader)
+        {
+            this.loader = loader;
+        }
+        
+        private URLStreamHandler getBuiltInHandler(String protocol, ClassLoader classLoader)
+        {
+            URLStreamHandler handler = handlers.get(protocol);
+            
+            if (handler == null)
+            {
+                for (String prefix : STREAM_HANDLER_PREFIXES)
+                {
+                    String className = prefix + '.' + protocol + ".Handler";
+                    try
+                    {
+                        Class<?> clazz = Class.forName(className, false, classLoader);
+                        handler = (URLStreamHandler) clazz.newInstance();
+                        break;
+                    }
+                    catch (Exception ignore)
+                    {
+                        ignore.printStackTrace(System.err);
+                    }
+                }
+                
+                if (handler != null)
+                {
+                    handlers.put(protocol, handler);
+                }
+            }
+            
+            if (handler == null)
+            {
+                throw new RuntimeException("Unable to find handler for protocol [" + protocol + "]");
+            }
+            
+            return handler;
+        }
+        
+        @Override
+        public URLStreamHandler createURLStreamHandler(String protocol)
+        {
+            try
+            {
+                return getBuiltInHandler(protocol, loader);
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException("Unable to create URLStreamHandler for protocol [" + protocol + "]");
+            }
+        }
+    }
+    
+    @After
+    public void cleanupURLStreamHandlerFactory()
+    {
+        URLStreamHandlerUtil.setFactory(null);
+    }
+    
+    @Before
+    public void init() throws Exception
+    {
+        super.init();
+        URLStreamHandlerUtil.setFactory(new URLHandlers(_loader));
+    }
+}

tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/HttpTrailersTest.java

@@ -0,0 +1,227 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2017 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.http.client;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.servlet.ServletException;
+import javax.servlet.ServletInputStream;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.eclipse.jetty.client.HttpRequest;
+import org.eclipse.jetty.client.HttpResponse;
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.client.util.BytesContentProvider;
+import org.eclipse.jetty.http.HttpFields;
+import org.eclipse.jetty.http.HttpMethod;
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Response;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class HttpTrailersTest extends AbstractTest
+{
+    public HttpTrailersTest(Transport transport)
+    {
+        super(transport == Transport.FCGI ? null : transport);
+    }
+
+    @Test
+    public void testRequestTrailersNoContent() throws Exception
+    {
+        testRequestTrailers(null);
+    }
+
+    @Test
+    public void testRequestTrailersWithContent() throws Exception
+    {
+        testRequestTrailers("abcdefghijklmnopqrstuvwxyz".getBytes(StandardCharsets.UTF_8));
+    }
+
+    private void testRequestTrailers(byte[] content) throws Exception
+    {
+        String trailerName = "Trailer";
+        String trailerValue = "value";
+        start(new AbstractHandler.ErrorDispatchHandler()
+        {
+            @Override
+            protected void doNonErrorHandle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            {
+                jettyRequest.setHandled(true);
+
+                // Read the content first.
+                ServletInputStream input = jettyRequest.getInputStream();
+                while (true)
+                {
+                    int read = input.read();
+                    if (read < 0)
+                        break;
+                }
+
+                // Now the trailers can be accessed.
+                HttpFields trailers = jettyRequest.getTrailers();
+                Assert.assertNotNull(trailers);
+                Assert.assertEquals(trailerValue, trailers.get(trailerName));
+            }
+        });
+
+        HttpFields trailers = new HttpFields();
+        trailers.put(trailerName, trailerValue);
+
+        HttpRequest request = (HttpRequest)client.newRequest(newURI());
+        request = request.trailers(() -> trailers);
+        if (content != null)
+            request.method(HttpMethod.POST).content(new BytesContentProvider(content));
+        ContentResponse response = request.timeout(5, TimeUnit.SECONDS).send();
+        Assert.assertEquals(HttpStatus.OK_200, response.getStatus());
+    }
+
+    @Test
+    public void testEmptyRequestTrailers() throws Exception
+    {
+        start(new AbstractHandler.ErrorDispatchHandler()
+        {
+            @Override
+            protected void doNonErrorHandle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            {
+                jettyRequest.setHandled(true);
+
+                // Read the content first.
+                ServletInputStream input = jettyRequest.getInputStream();
+                while (true)
+                {
+                    int read = input.read();
+                    if (read < 0)
+                        break;
+                }
+
+                // Now the trailers can be accessed.
+                HttpFields trailers = jettyRequest.getTrailers();
+                Assert.assertNull(trailers);
+            }
+        });
+
+        HttpFields trailers = new HttpFields();
+        HttpRequest request = (HttpRequest)client.newRequest(newURI());
+        request = request.trailers(() -> trailers);
+        ContentResponse response = request.timeout(5, TimeUnit.SECONDS).send();
+        Assert.assertEquals(HttpStatus.OK_200, response.getStatus());
+    }
+
+    @Test
+    public void testResponseTrailersNoContent() throws Exception
+    {
+        testResponseTrailers(null);
+    }
+
+    @Test
+    public void testResponseTrailersWithContent() throws Exception
+    {
+        testResponseTrailers("abcdefghijklmnopqrstuvwxyz".getBytes(StandardCharsets.UTF_8));
+    }
+
+    private void testResponseTrailers(byte[] content) throws Exception
+    {
+        String trailerName = "Trailer";
+        String trailerValue = "value";
+        start(new AbstractHandler.ErrorDispatchHandler()
+        {
+            @Override
+            protected void doNonErrorHandle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            {
+                jettyRequest.setHandled(true);
+
+                HttpFields trailers = new HttpFields();
+                trailers.put(trailerName, trailerValue);
+
+                Response jettyResponse = (Response)response;
+                jettyResponse.setTrailers(() -> trailers);
+                if (content != null)
+                    response.getOutputStream().write(content);
+            }
+        });
+
+        AtomicReference<Throwable> failure = new AtomicReference<>(new Throwable("no_success"));
+        ContentResponse response = client.newRequest(newURI())
+                .onResponseSuccess(r ->
+                {
+                    try
+                    {
+                        HttpResponse httpResponse = (HttpResponse)r;
+                        HttpFields trailers = httpResponse.getTrailers();
+                        Assert.assertNotNull(trailers);
+                        Assert.assertEquals(trailerValue, trailers.get(trailerName));
+                        failure.set(null);
+                    }
+                    catch (Throwable x)
+                    {
+                        failure.set(x);
+                    }
+                })
+                .timeout(5, TimeUnit.SECONDS)
+                .send();
+        Assert.assertEquals(HttpStatus.OK_200, response.getStatus());
+        Assert.assertNull(failure.get());
+    }
+
+    @Test
+    public void testEmptyResponseTrailers() throws Exception
+    {
+        start(new AbstractHandler.ErrorDispatchHandler()
+        {
+            @Override
+            protected void doNonErrorHandle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            {
+                jettyRequest.setHandled(true);
+
+                HttpFields trailers = new HttpFields();
+
+                Response jettyResponse = (Response)response;
+                jettyResponse.setTrailers(() -> trailers);
+            }
+        });
+
+        AtomicReference<Throwable> failure = new AtomicReference<>(new Throwable("no_success"));
+        ContentResponse response = client.newRequest(newURI())
+                .onResponseSuccess(r ->
+                {
+                    try
+                    {
+                        HttpResponse httpResponse = (HttpResponse)r;
+                        HttpFields trailers = httpResponse.getTrailers();
+                        Assert.assertNull(trailers);
+                        failure.set(null);
+                    }
+                    catch (Throwable x)
+                    {
+                        failure.set(x);
+                    }
+                })
+                .timeout(5, TimeUnit.SECONDS)
+                .send();
+        Assert.assertEquals(HttpStatus.OK_200, response.getStatus());
+        Assert.assertNull(failure.get());
+    }
+}