Merged branch 'jetty-10.0.x' into 'jetty-10.0.x-3537-bootstrap_websocket_http2'.

.github/stale.yml

@@ -0,0 +1,19 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 365
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 30
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - Pinned
+  - Security
+  - Specification
+# Label to use when marking an issue as stale
+staleLabel: Stale
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has been a
+  full year without activit. It will be closed if no further activity occurs.
+  Thank you for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: >
+  This issue has been closed due to it having no activity.

Jenkinsfile

@@ -40,11 +40,11 @@ pipeline {
           }
         }
 
-        stage("Build / Test - JDK12") {
+        stage("Build / Test - JDK13") {
           agent { node { label 'linux' } }
           steps {
             timeout(time: 120, unit: 'MINUTES') {
-              mavenBuild("jdk12", "-Pmongodb install", "maven3", true)
+              mavenBuild("jdk13", "-Pmongodb install", "maven3", true)
               warnings consoleParsers: [[parserName: 'Maven'], [parserName: 'Java']]
               junit testResults: '**/target/surefire-reports/*.xml,**/target/invoker-reports/TEST*.xml'
             }

README.md

@@ -23,7 +23,7 @@ Documentation
 
 Project documentation is available on the Jetty Eclipse website.
 
-- [http://www.eclipse.org/jetty/documentation](http://www.eclipse.org/jetty/documentation)
+- [https://www.eclipse.org/jetty/documentation](https://www.eclipse.org/jetty/documentation)
 
 Building
 ========
@@ -40,9 +40,9 @@ The first build may take a longer than expected as Maven downloads all the depen
 
 The build tests do a lot of stress testing, and on some machines it is necessary to set the file descriptor limit to greater than 2048 for the tests to all pass successfully.
 
-It is possible to bypass tests by building with `mvn -Dmaven.test.skip=true install` but note that this will not produce some of the test jars that are leveraged in other places in the build.
+It is possible to bypass tests by building with `mvn clean install -DskipTests`.
 
 Professional Services
 ---------------------
 
-Expert advice and production support are available through [Webtide.com](http://webtide.com).
+Expert advice and production support are available through [Webtide.com](https://webtide.com).

VERSION.txt

@@ -1,13 +1,14 @@
 jetty-10.0.0-SNAPSHOT
 
-jetty-10.0.0-alpha0 - 11 July 2019
+jetty-10.0.0-alpha0 - 11 July
+   2019
  + 113 Add support for NCSA Extended Log File Format
  + 114 Bring back overlay deployer
  + 132 ClientConnector abstraction
  + 207 Support javax.websocket version 1.1
  + 215 Add Conscrypt for native ALPN/TLS/SSL
  + 300 Implement Deflater / Inflater Object Pool
- + 482 [jetty-osgi] The CCL while parsing the xml files should be set to a
+ + 482 jetty-osgi] The CCL while parsing the xml files should be set to a
    combination of Jetty and Bundle-Classloader
  + 592 Support no-value Host header in HttpParser
  + 632 JMX tests rely on fixed port
@@ -79,7 +80,7 @@ jetty-10.0.0-alpha0 - 11 July 2019
  + 2095 Remove FastCGI multiplexing
  + 2103 Server should open connectors early in start sequence
  + 2108 Update licence headers and plugin for 2018
- + 2140 Infinispan and hazelcast changes to scavenge zombie expired sessions.
+ + 2140 Infinispan and hazelcast changes to scavenge zombie expired sessions
  + 2172 Support javax.websocket 1.1
  + 2175 Refactor WebSocket close handling
  + 2191 JPMS Support
@@ -92,13 +93,12 @@ jetty-10.0.0-alpha0 - 11 July 2019
  + 2978 Add module-info.java to relevant Jetty modules
  + 2983 Jetty 10 Configuration abstraction
  + 2985 Jetty 10 Configuration replacement algorithm incorrect
- + 2996 ContextHandler.setDefaultContextPath() not implemented for quickstart.
+ + 2996 ContextHandler.setDefaultContextPath() not implemented for quickstart
  + 3009 Update Jetty 10 to use non-LEGACY Compliance Modes
  + 3010 Move old MultiPart parsing implementation to jetty-http
  + 3011 Move HttpCompliance to HttpConfiguration
  + 3012 Refactor HttpCompliance and HttpComplianceSection to be friendlier to
    customization
- + 3106 Websocket connection stats and request stats
  + 3129 javax-websocket-common pom.xml is wrong
  + 3139 NPE on
    WebSocketServerContainerInitializer.configureContext(ServletContextHandler)
@@ -115,7 +115,7 @@ jetty-10.0.0-alpha0 - 11 July 2019
  + 3197 Use jetty specific websocket API jar
  + 3213 MetaInfConfigurationTest tests disabled in jetty-10.0.x
  + 3216 Autobahn WebSocketServer failures in jetty 10
- + 3225 Response.sendError should not set reason.
+ + 3225 Response.sendError should not set reason
  + 3246 javax-websocket-tests exception stacktraces
  + 3249 Update to apache jasper 9.0.14 for jetty-10
  + 3274 OSGi versions of java.base classes in
@@ -123,7 +123,7 @@ jetty-10.0.0-alpha0 - 11 July 2019
  + 3279 WebSocket write may hang forever
  + 3288 Correct websocket artifactIds on jetty-10.0.x
  + 3290 async websocket onOpen, onError and onClose in 10.0.x
- + 3298 Review jetty-10 websocket CompletableFuture usage.
+ + 3298 Review jetty-10 websocket CompletableFuture usage
  + 3303 Update to jakarta ee javax artifacts for jetty-10
  + 3308 Remove deprecated methods from sessions
  + 3320 Review Jetty 10 module-info.java
@@ -157,7 +157,7 @@ jetty-10.0.0-alpha0 - 11 July 2019
  + 3648 javax.websocket client container incorrectly creates Server
    SslContextFactory
  + 3661 JettyWebSocketServerContainer exposes websocket common classes
- + 3666 WebSocket - Handling sent 1009 close frame.
+ + 3666 WebSocket - Handling sent 1009 close frame
  + 3696 Unwrap JavaxWebSocketClientContainer.connectToServer() exceptions
  + 3698 Missing WebSocket ServerContainer after server restart
  + 3700 stackoverflow in WebAppClassLoaderUrlStreamTest
@@ -193,6 +193,98 @@ jetty-10.0.0-alpha0 - 11 July 2019
  + 3849 ClosedChannelException from jetty-test-webapp javax websocket chat
    example
 
+jetty-9.4.22.v20191022 - 22 October 2019
+ + 2429 HttpClient backpressure improved
+ + 3558 Error notifications can be received after a successful websocket
+ + 3787 Jetty client sometimes returns EOFException instead of
+   SSLHandshakeException on certificate errors.
+ + 3913 Clustered HttpSession IllegalStateException: Invalid for read
+ + 3989 Inform custom ManagedSelector of dead selector via optional
+   onFailedSelect()
+ + 4096 Thread in ReservedThreadExecutor does not exit when stopped
+ + 4104 Frames are sent through ExtensionStack even if WebSocket Session is
+   closed
+ + 4105 QueuedThreadPool increased thread usage and no idle thread decay
+ + 4115 Drop HTTP/2 pseudo headers
+ + 4121 QueuedThreadPool should support ThreadFactory behaviors
+ + 4122 QueuedThreadPool should reset thread interrupted on failed run
+ + 4128 OpenIdCredetials can't decode JWT ID token
+ + 4132 Should be possible to use OIDC without metadata
+ + 4141 ClassCastException with non-async Servlet + async Filter +
+   HttpServletRequestWrapper
+ + 4142 Configurable HTTP/2 RateControl
+ + 4144 Naked cast to Request should be avoided
+ + 4156 IllegalStateException when forwarding to jsp with new session
+ + 4158 Behaviour change in session handling in 9.4.21.v20190926
+ + 4170 Client-side alias selection based on SSLEngine
+ + 4174 ConcurrentModificationException when stopping jetty:run-war
+ + 4176 Should not set header if sendError has been called
+ + 4177 Configure HTTP proxy with SslContextFactory
+ + 4179 Improve HttpChannel$SendCallback references for GC
+ + 4183 Jetty considers bootstrap injected class to be a "server class"
+ + 4188 Spin in HttpOutput.close
+ + 4190 Jetty hangs after thread blocked in SharedBlockingCallback.block()
+   called by HttpOutput.close
+ + 4191 Increase GzipHandler minGzipSize default size
+ + 4193 InetAccessHandler - new includeConnectors/excludeConnectors not quite
+   correct anymore
+ + 4201 Throw SSLHandshakeException in case of TLS handshake failures
+ + 4203 Some Transfer-Encoding and Content-Length combinations do not result in
+   expected 400 Bad Request
+ + 4204 Transfer-Encoding behavior does not follow RFC7230
+ + 4208 Regression in Jetty 9.4.21: 304 response with Content-Length fails
+ + 4209 Unused TLS connection is not closed in Java 11
+ + 4217 SslConnection.DecryptedEnpoint.flush eternal busy loop
+ + 4227 First authorization request produced by OIDC module fails due to
+   inclusion of sessionid
+
+jetty-9.4.21.v20190926 - 26 September 2019
+ + 97 Permanent UnavailableException thrown during servlet request handling
+   should cause servlet destroy
+ + 137 Support OAuth
+ + 155 No way to set keystore for JSR 356 websocket clients, needed for SSL
+   client authentication
+ + 1036 Allow easy configuration of Scheduler-Threads and name them more
+   appropriate
+ + 2815 HPack fields are opaque octets
+ + 3040 Allow RFC6265 Cookies to include optional SameSite attribute
+ + 3106 WebSocket connection stats and request stats
+ + 3734 WebSocket suspend when input closed
+ + 3747 Make Jetty Demo work with JPMS
+ + 3806 Error Page handling Async race with ProxyServlet
+ + 3913 Clustered HttpSession IllegalStateException: Invalid for read
+ + 3936 Race condition when modifying session + sendRedirect()
+ + 3956 Remove and warn on use of illegal HTTP/2 response headers
+ + 3964 Improve efficiency of listeners
+ + 3968 WebSocket sporadic ReadPendingException using suspend/resume
+ + 3978 HTTP/2 fixes for robustly handling abnormal traffic and resource
+   exhaustion
+ + 3983 JarFileResource incorrectly lists the contents of directories with
+   spaces
+ + 3985 Improve lenient Cookie parsing
+ + 3989 Inform custom ManagedSelector of dead selector via optional
+   onFailedSelect()
+ + 4000 Add SameFileAliasChecker to help with FileSystem static file access
+   normalization on Mac and Windows
+ + 4007 NullPointerException while trying to run jetty start.run on Windows
+ + 4009 ServletContextHandler setSecurityHandler broke handler chain
+ + 4020 Revert WebSocket ExtensionFactory change to interface
+ + 4022 Servlet which is added by ServletRegistration can't be started
+ + 4025 Provide more write-through behaviours for DefaultSessionCache
+ + 4027 Ensure AbstractSessionDataStore cannot be used unless it is started
+ + 4033 Ignore bad percent encodings in paths during
+   URIUtil.equalsIgnoreEncodings()
+ + 4047 Gracefully stopped Jetty not flushing all response data
+ + 4048 Multiple values in X-Forwarded-Port throw NumberFormatException
+ + 4057 NullPointerException in o.e.j.h.HttpFields
+ + 4064 NullPointerException initializing embedded servlet
+ + 4075 Do not fail on servlet-mapping with url-pattern /On*
+ + 4082 NullPointerExceptoin while Debug logging in client
+ + 4084 Use of HttpConfiguration.setBlockingTimeout(long) in jetty.xml produces
+   warning on jetty-home startup
+ + 4105 Cleanup of Idle thread count in QueuedThreadPool
+ + 4113 HttpClient fails with JDK 13 and TLS 1.3
+
 jetty-9.4.20.v20190813 - 13 August 2019
  + 300 Implement Deflater / Inflater Object Pool
  + 2061 WebSocket hangs in blockingWrite
@@ -290,7 +382,7 @@ jetty-9.4.18.v20190429 - 29 April 2019
  + 3609 Fix infinispan start module dependencies
 
 jetty-9.4.17.v20190418 - 18 April 2019
- + 2140 Infinispan and hazelcast changes to scavenge zombie expired sessions.
+ + 2140 Infinispan and hazelcast changes to scavenge zombie expired sessions
  + 3464 Split SslContextFactory into Client and Server
  + 3549 Directory Listing on Windows reveals Resource Base path
  + 3555 DefaultHandler Reveals Base Resource Path of each Context
@@ -368,6 +460,11 @@ jetty-9.4.15.v20190215 - 15 February 2019
  + 3350 Do not expect to be able to connect to https URLs with the HttpClient
    created from a parameterless constructor
 
+jetty-9.3.28.v20191105 - 05 November 2019
+ + 3989 Inform custom ManagedSelector of dead selector via optional
+   onFailedSelect()
+ + 4217 SslConnection.DecryptedEnpoint.flush eternal busy loop
+
 jetty-9.3.27.v20190418 - 18 April 2019
  + 3549 Directory Listing on Windows reveals Resource Base path
  + 3555 DefaultHandler Reveals Base Resource Path of each Context
@@ -380,6 +477,9 @@ jetty-9.3.26.v20190403 - 03 April 2019
    ForwardedRequestCustomizer
  + 3319 Allow reverse sort for directory listed files
 
+jetty-9.2.29.v20191105 - 05 November 2019
+ + 4217 SslConnection.DecryptedEnpoint.flush eternal busy loop
+
 jetty-9.2.28.v20190418 - 18 April 2019
  + 3549 Directory Listing on Windows reveals Resource Base path
  + 3555 DefaultHandler Reveals Base Resource Path of each Context

examples/embedded/src/test/java/org/eclipse/jetty/embedded/ProxyServerTest.java

@@ -27,6 +27,7 @@ import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.server.Server;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -53,6 +54,7 @@ public class ProxyServerTest extends AbstractEmbeddedTest
         server.stop();
     }
 
+    @Tag("external")
     @Test
     public void testGetProxiedRFC() throws Exception
     {

examples/embedded/src/test/resources/jetty-logging.properties

@@ -1,5 +1,5 @@
 org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog
-org.eclipse.jetty.LEVEL=WARN
+org.eclipse.jetty.LEVEL=INFO
 org.eclipse.jetty.embedded.JettyDistribution.LEVEL=DEBUG
 #org.eclipse.jetty.STACKS=true
 #org.eclipse.jetty.STACKS=false

jetty-alpn/jetty-alpn-java-client/src/main/java/org/eclipse/jetty/alpn/java/client/JDK9ClientALPNProcessor.java

@@ -55,13 +55,13 @@ public class JDK9ClientALPNProcessor implements ALPNProcessor.Client
         ALPNClientConnection alpn = (ALPNClientConnection)connection;
         SSLParameters sslParameters = sslEngine.getSSLParameters();
         List<String> protocols = alpn.getProtocols();
-        sslParameters.setApplicationProtocols(protocols.toArray(new String[protocols.size()]));
+        sslParameters.setApplicationProtocols(protocols.toArray(new String[0]));
         sslEngine.setSSLParameters(sslParameters);
         ((DecryptedEndPoint)connection.getEndPoint()).getSslConnection()
             .addHandshakeListener(new ALPNListener(alpn));
     }
 
-    private final class ALPNListener implements SslHandshakeListener
+    private static final class ALPNListener implements SslHandshakeListener
     {
         private final ALPNClientConnection alpnConnection;
 

jetty-alpn/jetty-alpn-java-server/src/main/java/org/eclipse/jetty/alpn/java/server/JDK9ServerALPNProcessor.java

@@ -55,7 +55,7 @@ public class JDK9ServerALPNProcessor implements ALPNProcessor.Server, SslHandsha
         sslEngine.setHandshakeApplicationProtocolSelector(new ALPNCallback((ALPNServerConnection)connection));
     }
 
-    private final class ALPNCallback implements BiFunction<SSLEngine, List<String>, String>, SslHandshakeListener
+    private static final class ALPNCallback implements BiFunction<SSLEngine, List<String>, String>, SslHandshakeListener
     {
         private final ALPNServerConnection alpnConnection;
 
@@ -67,12 +67,21 @@ public class JDK9ServerALPNProcessor implements ALPNProcessor.Server, SslHandsha
 
         @Override
         public String apply(SSLEngine engine, List<String> protocols)
+        {
+            try
             {
                 if (LOG.isDebugEnabled())
                     LOG.debug("apply {} {}", alpnConnection, protocols);
                 alpnConnection.select(protocols);
                 return alpnConnection.getProtocol();
             }
+            catch (Throwable x)
+            {
+                // Cannot negotiate the protocol, return null to have
+                // JSSE send Alert.NO_APPLICATION_PROTOCOL to the client.
+                return null;
+            }
+        }
 
         @Override
         public void handshakeSucceeded(Event event)

jetty-alpn/jetty-alpn-java-server/src/test/java/org/eclipse/jetty/alpn/java/server/JDK9ALPNTest.java

@@ -22,8 +22,13 @@ import java.io.BufferedReader;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.SocketChannel;
 import java.nio.charset.StandardCharsets;
 import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLEngineResult;
 import javax.net.ssl.SSLParameters;
 import javax.net.ssl.SSLSocket;
 import javax.servlet.http.HttpServletRequest;
@@ -38,12 +43,16 @@ import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
 
 public class JDK9ALPNTest
 {
@@ -85,10 +94,10 @@ public class JDK9ALPNTest
     @Test
     public void testClientNotSupportingALPNServerSpeaksDefaultProtocol() throws Exception
     {
-        startServer(new AbstractHandler.ErrorDispatchHandler()
+        startServer(new AbstractHandler()
         {
             @Override
-            protected void doNonErrorHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
             {
                 baseRequest.setHandled(true);
             }
@@ -127,10 +136,10 @@ public class JDK9ALPNTest
     @Test
     public void testClientSupportingALPNServerSpeaksNegotiatedProtocol() throws Exception
     {
-        startServer(new AbstractHandler.ErrorDispatchHandler()
+        startServer(new AbstractHandler()
         {
             @Override
-            protected void doNonErrorHandle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
             {
                 baseRequest.setHandled(true);
             }
@@ -168,4 +177,57 @@ public class JDK9ALPNTest
             }
         }
     }
+
+    @Test
+    public void testClientSupportingALPNCannotNegotiateProtocol() throws Exception
+    {
+        startServer(new AbstractHandler() {
+            @Override
+            public void handle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response)
+            {
+                jettyRequest.setHandled(true);
+            }
+        });
+
+        SslContextFactory sslContextFactory = new SslContextFactory.Client(true);
+        sslContextFactory.start();
+        String host = "localhost";
+        int port = connector.getLocalPort();
+        try (SocketChannel client = SocketChannel.open(new InetSocketAddress(host, port)))
+        {
+            client.socket().setSoTimeout(5000);
+
+            SSLEngine sslEngine = sslContextFactory.newSSLEngine(host, port);
+            sslEngine.setUseClientMode(true);
+            SSLParameters sslParameters = sslEngine.getSSLParameters();
+            sslParameters.setApplicationProtocols(new String[]{"unknown/1.0"});
+            sslEngine.setSSLParameters(sslParameters);
+            sslEngine.beginHandshake();
+            assertSame(SSLEngineResult.HandshakeStatus.NEED_WRAP, sslEngine.getHandshakeStatus());
+
+            ByteBuffer sslBuffer = ByteBuffer.allocate(sslEngine.getSession().getPacketBufferSize());
+
+            SSLEngineResult result = sslEngine.wrap(BufferUtil.EMPTY_BUFFER, sslBuffer);
+            assertSame(SSLEngineResult.Status.OK, result.getStatus());
+
+            sslBuffer.flip();
+            client.write(sslBuffer);
+
+            assertSame(SSLEngineResult.HandshakeStatus.NEED_UNWRAP, sslEngine.getHandshakeStatus());
+
+            sslBuffer.clear();
+            int read = client.read(sslBuffer);
+            assertThat(read, greaterThan(0));
+
+            sslBuffer.flip();
+            // TLS frame layout: record_type, major_version, minor_version, hi_length, lo_length
+            int recordTypeAlert = 21;
+            assertEquals(recordTypeAlert, sslBuffer.get(0) & 0xFF);
+            // Alert record layout: alert_level, alert_code
+            int alertLevelFatal = 2;
+            assertEquals(alertLevelFatal, sslBuffer.get(5) & 0xFF);
+            int alertCodeNoApplicationProtocol = 120;
+            assertEquals(alertCodeNoApplicationProtocol, sslBuffer.get(6) & 0xFF);
+        }
+    }
 }

jetty-alpn/jetty-alpn-server/src/main/config/modules/alpn-impl/alpn-13.mod

@@ -1,4 +1,4 @@
 DO NOT EDIT - See: https://www.eclipse.org/jetty/documentation/current/startup-modules.html
 
 [depend]
-alpn-impl/alpn-9
+alpn-impl/alpn-11

jetty-alpn/jetty-alpn-server/src/main/config/modules/alpn-impl/alpn-14.mod

@@ -1,4 +1,4 @@
 DO NOT EDIT - See: https://www.eclipse.org/jetty/documentation/current/startup-modules.html
 
 [depend]
-alpn-impl/alpn-9
+alpn-impl/alpn-11

jetty-ant/src/main/java/org/eclipse/jetty/ant/AntWebAppContext.java

@@ -27,7 +27,6 @@ import java.security.CodeSource;
 import java.security.PermissionCollection;
 import java.util.ArrayList;
 import java.util.Enumeration;
-import java.util.EventListener;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -615,9 +614,9 @@ public class AntWebAppContext extends WebAppContext
             TaskLog.logWithTimestamp("Stopping web application " + this);
             Thread.currentThread().sleep(500L);
             super.doStop();
-            //remove all filters, servlets and listeners. They will be recreated
-            //either via application of a context xml file or web.xml or annotation or servlet api
-            setEventListeners(new EventListener[0]);
+            // remove all filters and servlets. They will be recreated
+            // either via application of a context xml file or web.xml or annotation or servlet api.
+            // Event listeners are reset in ContextHandler.doStop()
             getServletHandler().setFilters(new FilterHolder[0]);
             getServletHandler().setFilterMappings(new FilterMapping[0]);
             getServletHandler().setServlets(new ServletHolder[0]);

jetty-bom/pom.xml

@@ -284,6 +284,11 @@
         <artifactId>jetty-security</artifactId>
         <version>10.0.0-SNAPSHOT</version>
       </dependency>
+      <dependency>
+        <groupId>org.eclipse.jetty</groupId>
+        <artifactId>jetty-openid</artifactId>
+        <version>10.0.0-SNAPSHOT</version>
+      </dependency>
       <dependency>
         <groupId>org.eclipse.jetty</groupId>
         <artifactId>jetty-server</artifactId>

jetty-client/pom.xml

@@ -111,7 +111,6 @@
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-alpn-client</artifactId>
       <version>${project.version}</version>
-      <optional>true</optional>
     </dependency>
     <dependency>
       <groupId>org.eclipse.jetty</groupId>

jetty-client/src/main/config/modules/client.mod

@@ -5,3 +5,4 @@ Adds the Jetty HTTP client to the server classpath.
 
 [lib]
 lib/jetty-client-${jetty.version}.jar
+lib/jetty-alpn-client-${jetty.version}.jar

jetty-client/src/main/java/module-info.java

@@ -26,14 +26,13 @@ module org.eclipse.jetty.client
     exports org.eclipse.jetty.client.proxy;
     exports org.eclipse.jetty.client.util;
 
+    requires org.eclipse.jetty.alpn.client;
     requires org.eclipse.jetty.http;
     requires org.eclipse.jetty.io;
     requires org.eclipse.jetty.util;
 
     // Only required if using SPNEGO.
     requires static java.security.jgss;
-    // Only required if using the dynamic transport.
-    requires static org.eclipse.jetty.alpn.client;
     // Only required if using JMX.
     requires static org.eclipse.jetty.jmx;
 }

jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java

@@ -497,27 +497,6 @@ public class HttpClient extends ContainerLifeCycle
         return uri;
     }
 
-    /**
-     * Returns a {@link Destination} for the given scheme, host and port.
-     * Applications may use {@link Destination}s to create {@link Connection}s
-     * that will be outside HttpClient's pooling mechanism, to explicitly
-     * control the connection lifecycle (in particular their termination with
-     * {@link Connection#close()}).
-     *
-     * @param scheme the destination scheme
-     * @param host the destination host
-     * @param port the destination port
-     * @return the destination
-     * @see #getDestinations()
-     * @deprecated use {@link #resolveDestination(Request)} instead
-     */
-    @Deprecated
-    public Destination getDestination(String scheme, String host, int port)
-    {
-        Origin origin = createOrigin(scheme, host, port);
-        return resolveDestination(new HttpDestination.Key(origin, null));
-    }
-
     public Destination resolveDestination(Request request)
     {
         Origin origin = createOrigin(request.getScheme(), request.getHost(), request.getPort());
@@ -544,25 +523,15 @@ public class HttpClient extends ContainerLifeCycle
 
     HttpDestination resolveDestination(HttpDestination.Key key)
     {
-        HttpDestination destination = destinations.get(key);
-        if (destination == null)
+        return destinations.computeIfAbsent(key, k ->
         {
-            destination = getTransport().newHttpDestination(key);
+            HttpDestination destination = getTransport().newHttpDestination(k);
             // Start the destination before it's published to other threads.
             addManaged(destination);
-            HttpDestination existing = destinations.putIfAbsent(key, destination);
-            if (existing != null)
-            {
-                removeBean(destination);
-                destination = existing;
-            }
-            else
-            {
             if (LOG.isDebugEnabled())
                 LOG.debug("Created {}", destination);
-            }
-        }
             return destination;
+        });
     }
 
     protected boolean removeDestination(HttpDestination destination)
@@ -1137,9 +1106,11 @@ public class HttpClient extends ContainerLifeCycle
         return HttpScheme.HTTPS.is(scheme) || HttpScheme.WSS.is(scheme);
     }
 
-    protected ClientConnectionFactory newSslClientConnectionFactory(ClientConnectionFactory connectionFactory)
+    protected ClientConnectionFactory newSslClientConnectionFactory(SslContextFactory.Client sslContextFactory, ClientConnectionFactory connectionFactory)
     {
-        return new SslClientConnectionFactory(getSslContextFactory(), getByteBufferPool(), getExecutor(), connectionFactory);
+        if (sslContextFactory == null)
+            sslContextFactory = getSslContextFactory();
+        return new SslClientConnectionFactory(sslContextFactory, getByteBufferPool(), getExecutor(), connectionFactory);
     }
 
     private class ContentDecoderFactorySet implements Set<ContentDecoder.Factory>

jetty-client/src/main/java/org/eclipse/jetty/client/HttpDestination.java

@@ -53,6 +53,7 @@ import org.eclipse.jetty.util.component.Dumpable;
 import org.eclipse.jetty.util.component.DumpableCollection;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.Scheduler;
 import org.eclipse.jetty.util.thread.Sweeper;
 
@@ -108,12 +109,12 @@ public class HttpDestination extends ContainerLifeCycle implements Destination,
         {
             connectionFactory = proxy.newClientConnectionFactory(connectionFactory);
             if (proxy.isSecure())
-                connectionFactory = newSslClientConnectionFactory(connectionFactory);
+                connectionFactory = newSslClientConnectionFactory(proxy.getSslContextFactory(), connectionFactory);
         }
         else
         {
             if (isSecure())
-                connectionFactory = newSslClientConnectionFactory(connectionFactory);
+                connectionFactory = newSslClientConnectionFactory(null, connectionFactory);
         }
         return connectionFactory;
     }
@@ -149,9 +150,9 @@ public class HttpDestination extends ContainerLifeCycle implements Destination,
         return new BlockingArrayQueue<>(client.getMaxRequestsQueuedPerDestination());
     }
 
-    protected ClientConnectionFactory newSslClientConnectionFactory(ClientConnectionFactory connectionFactory)
+    protected ClientConnectionFactory newSslClientConnectionFactory(SslContextFactory.Client sslContextFactory, ClientConnectionFactory connectionFactory)
     {
-        return client.newSslClientConnectionFactory(connectionFactory);
+        return client.newSslClientConnectionFactory(sslContextFactory, connectionFactory);
     }
 
     public boolean isSecure()

jetty-client/src/main/java/org/eclipse/jetty/client/HttpProxy.java

@@ -19,6 +19,7 @@
 package org.eclipse.jetty.client;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.net.URI;
 import java.util.List;
 import java.util.Map;
@@ -34,10 +35,12 @@ import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpScheme;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.io.ClientConnectionFactory;
+import org.eclipse.jetty.io.ClientConnector;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.util.Promise;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
 
 public class HttpProxy extends ProxyConfiguration.Proxy
 {
@@ -50,12 +53,27 @@ public class HttpProxy extends ProxyConfiguration.Proxy
 
     public HttpProxy(Origin.Address address, boolean secure)
     {
-        this(address, secure, new HttpDestination.Protocol(List.of("http/1.1"), false));
+        this(address, secure, null, new HttpDestination.Protocol(List.of("http/1.1"), false));
     }
 
     public HttpProxy(Origin.Address address, boolean secure, HttpDestination.Protocol protocol)
     {
-        super(address, secure, Objects.requireNonNull(protocol));
+        this(address, secure, null, Objects.requireNonNull(protocol));
+    }
+
+    public HttpProxy(Origin.Address address, SslContextFactory.Client sslContextFactory)
+    {
+        this(address, true, sslContextFactory, new HttpDestination.Protocol(List.of("http/1.1"), false));
+    }
+
+    public HttpProxy(Origin.Address address, SslContextFactory.Client sslContextFactory, HttpDestination.Protocol protocol)
+    {
+        this(address, true, sslContextFactory, Objects.requireNonNull(protocol));
+    }
+
+    private HttpProxy(Origin.Address address, boolean secure, SslContextFactory.Client sslContextFactory, HttpDestination.Protocol protocol)
+    {
+        super(address, secure, sslContextFactory, Objects.requireNonNull(protocol));
     }
 
     @Override
@@ -206,7 +224,12 @@ public class HttpProxy extends ProxyConfiguration.Proxy
                 HttpDestination destination = (HttpDestination)context.get(HttpClientTransport.HTTP_DESTINATION_CONTEXT_KEY);
                 ClientConnectionFactory connectionFactory = this.connectionFactory;
                 if (destination.isSecure())
-                    connectionFactory = destination.newSslClientConnectionFactory(connectionFactory);
+                {
+                    // Don't want to do DNS resolution here.
+                    InetSocketAddress address = InetSocketAddress.createUnresolved(destination.getHost(), destination.getPort());
+                    context.put(ClientConnector.REMOTE_SOCKET_ADDRESS_CONTEXT_KEY, address);
+                    connectionFactory = destination.newSslClientConnectionFactory(null, connectionFactory);
+                }
                 var oldConnection = endPoint.getConnection();
                 var newConnection = connectionFactory.newConnection(endPoint, context);
                 endPoint.upgrade(newConnection);

jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java

@@ -23,9 +23,14 @@ import java.net.URI;
 import java.nio.ByteBuffer;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.LongConsumer;
+import java.util.function.LongUnaryOperator;
 import java.util.stream.Collectors;
 
 import org.eclipse.jetty.client.api.Response;
@@ -35,7 +40,7 @@ import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.Callback;
-import org.eclipse.jetty.util.IteratingNestedCallback;
+import org.eclipse.jetty.util.MathUtils;
 import org.eclipse.jetty.util.component.Destroyable;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
@@ -71,9 +76,11 @@ public abstract class HttpReceiver
 
     private final AtomicReference<ResponseState> responseState = new AtomicReference<>(ResponseState.IDLE);
     private final HttpChannel channel;
-    private List<Response.AsyncContentListener> contentListeners;
-    private ContentDecoder decoder;
+    private ContentListeners contentListeners;
+    private Decoder decoder;
     private Throwable failure;
+    private long demand;
+    private boolean stalled;
 
     protected HttpReceiver(HttpChannel channel)
     {
@@ -85,6 +92,55 @@ public abstract class HttpReceiver
         return channel;
     }
 
+    void demand(long n)
+    {
+        if (n <= 0)
+            throw new IllegalArgumentException("Invalid demand " + n);
+
+        boolean resume = false;
+        synchronized (this)
+        {
+            demand = MathUtils.cappedAdd(demand, n);
+            if (stalled)
+            {
+                stalled = false;
+                resume = true;
+            }
+            if (LOG.isDebugEnabled())
+                LOG.debug("Response demand={}/{}, resume={}", n, demand, resume);
+        }
+
+        if (resume)
+        {
+            if (decoder != null)
+                decoder.resume();
+            else
+                receive();
+        }
+    }
+
+    protected long demand()
+    {
+        return demand(LongUnaryOperator.identity());
+    }
+
+    private long demand(LongUnaryOperator operator)
+    {
+        synchronized (this)
+        {
+            return demand = operator.applyAsLong(demand);
+        }
+    }
+
+    protected boolean hasDemandOrStall()
+    {
+        synchronized (this)
+        {
+            stalled = demand <= 0;
+            return !stalled;
+        }
+    }
+
     protected HttpExchange getHttpExchange()
     {
         return channel.getHttpExchange();
@@ -100,6 +156,10 @@ public abstract class HttpReceiver
         return responseState.get() == ResponseState.FAILURE;
     }
 
+    protected void receive()
+    {
+    }
+
     /**
      * Method to be invoked when the response status code is available.
      * <p>
@@ -116,13 +176,12 @@ public abstract class HttpReceiver
         if (!updateResponseState(ResponseState.IDLE, ResponseState.TRANSIENT))
             return false;
 
-        final HttpConversation conversation = exchange.getConversation();
-        final HttpResponse response = exchange.getResponse();
+        HttpConversation conversation = exchange.getConversation();
+        HttpResponse response = exchange.getResponse();
         // Probe the protocol handlers
-        final HttpDestination destination = getHttpDestination();
-        final HttpClient client = destination.getHttpClient();
-        final ProtocolHandler protocolHandler = client.findProtocolHandler(exchange.getRequest(), response);
-
+        HttpDestination destination = getHttpDestination();
+        HttpClient client = destination.getHttpClient();
+        ProtocolHandler protocolHandler = client.findProtocolHandler(exchange.getRequest(), response);
         Response.Listener handlerListener = null;
         if (protocolHandler != null)
         {
@@ -241,25 +300,19 @@ public abstract class HttpReceiver
      */
     protected boolean responseHeaders(HttpExchange exchange)
     {
-        out:
         while (true)
         {
             ResponseState current = responseState.get();
-            switch (current)
-            {
-                case BEGIN:
-                case HEADER:
+            if (current == ResponseState.BEGIN || current == ResponseState.HEADER)
             {
                 if (updateResponseState(current, ResponseState.TRANSIENT))
-                        break out;
                     break;
             }
-                default:
+            else
             {
                 return false;
             }
         }
-        }
 
         HttpResponse response = exchange.getResponse();
         if (LOG.isDebugEnabled())
@@ -267,11 +320,11 @@ public abstract class HttpReceiver
         ResponseNotifier notifier = getHttpDestination().getResponseNotifier();
         List<Response.ResponseListener> responseListeners = exchange.getConversation().getResponseListeners();
         notifier.notifyHeaders(responseListeners, response);
-        contentListeners = responseListeners.stream()
-            .filter(Response.AsyncContentListener.class::isInstance)
-            .map(Response.AsyncContentListener.class::cast)
-            .collect(Collectors.toList());
+        contentListeners = new ContentListeners(responseListeners);
+        contentListeners.notifyBeforeContent(response);
 
+        if (!contentListeners.isEmpty())
+        {
             List<String> contentEncodings = response.getHeaders().getCSV(HttpHeader.CONTENT_ENCODING.asString(), false);
             if (contentEncodings != null && !contentEncodings.isEmpty())
             {
@@ -281,15 +334,21 @@ public abstract class HttpReceiver
                     {
                         if (factory.getEncoding().equalsIgnoreCase(encoding))
                         {
-                        this.decoder = factory.newContentDecoder();
+                            decoder = new Decoder(response, factory.newContentDecoder());
                             break;
                         }
                     }
                 }
             }
+        }
 
         if (updateResponseState(ResponseState.TRANSIENT, ResponseState.HEADERS))
-            return true;
+        {
+            boolean hasDemand = hasDemandOrStall();
+            if (LOG.isDebugEnabled())
+                LOG.debug("Response headers {}, hasDemand={}", response, hasDemand);
+            return hasDemand;
+        }
 
         terminateResponse(exchange);
         return false;
@@ -307,45 +366,83 @@ public abstract class HttpReceiver
      */
     protected boolean responseContent(HttpExchange exchange, ByteBuffer buffer, Callback callback)
     {
-        out:
         while (true)
         {
             ResponseState current = responseState.get();
-            switch (current)
-            {
-                case HEADERS:
-                case CONTENT:
+            if (current == ResponseState.HEADERS || current == ResponseState.CONTENT)
             {
                 if (updateResponseState(current, ResponseState.TRANSIENT))
-                        break out;
                     break;
             }
-                default:
+            else
             {
                 callback.failed(new IllegalStateException("Invalid response state " + current));
                 return false;
             }
         }
+
+        boolean proceed = true;
+        if (demand() <= 0)
+        {
+            callback.failed(new IllegalStateException("No demand for response content"));
+            proceed = false;
         }
 
         HttpResponse response = exchange.getResponse();
+        if (proceed)
+        {
             if (LOG.isDebugEnabled())
                 LOG.debug("Response content {}{}{}", response, System.lineSeparator(), BufferUtil.toDetailString(buffer));
 
-        ResponseNotifier notifier = getHttpDestination().getResponseNotifier();
-
-        ContentDecoder decoder = this.decoder;
-        if (decoder == null)
+            ContentListeners listeners = this.contentListeners;
+            if (listeners != null)
             {
-            notifier.notifyContent(response, buffer, callback, contentListeners);
+                if (listeners.isEmpty())
+                {
+                    callback.succeeded();
                 }
                 else
                 {
-            new Decoder(notifier, response, decoder, buffer, callback).iterate();
+                    Decoder decoder = this.decoder;
+                    if (decoder == null)
+                    {
+                        listeners.notifyContent(response, buffer, callback);
+                    }
+                    else
+                    {
+                        try
+                        {
+                            proceed = decoder.decode(buffer, callback);
+                        }
+                        catch (Throwable x)
+                        {
+                            callback.failed(x);
+                            proceed = false;
+                        }
+                    }
+                }
+            }
+            else
+            {
+                // May happen in case of concurrent abort.
+                proceed = false;
+            }
         }
 
         if (updateResponseState(ResponseState.TRANSIENT, ResponseState.CONTENT))
-            return true;
+        {
+            if (proceed)
+            {
+                boolean hasDemand = hasDemandOrStall();
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Response content {}, hasDemand={}", response, hasDemand);
+                return hasDemand;
+            }
+            else
+            {
+                return false;
+            }
+        }
 
         terminateResponse(exchange);
         return false;
@@ -386,8 +483,7 @@ public abstract class HttpReceiver
 
         // Mark atomically the response as terminated, with
         // respect to concurrency between request and response.
-        Result result = exchange.terminateResponse();
-        terminateResponse(exchange, result);
+        terminateResponse(exchange);
 
         return true;
     }
@@ -410,6 +506,9 @@ public abstract class HttpReceiver
         if (exchange == null)
             return false;
 
+        if (LOG.isDebugEnabled())
+            LOG.debug("Response failure " + exchange.getResponse(), failure);
+
         // Mark atomically the response as completed, with respect
         // to concurrency between response success and response failure.
         if (exchange.responseComplete(failure))
@@ -448,7 +547,7 @@ public abstract class HttpReceiver
     }
 
     /**
-     * Resets this {@link HttpReceiver} state.
+     * Resets the state of this HttpReceiver.
      * <p>
      * Subclasses should override (but remember to call {@code super}) to reset their own state.
      * <p>
@@ -456,13 +555,11 @@ public abstract class HttpReceiver
      */
     protected void reset()
     {
-        contentListeners = null;
-        destroyDecoder(decoder);
-        decoder = null;
+        cleanup();
     }
 
     /**
-     * Disposes this {@link HttpReceiver} state.
+     * Disposes the state of this HttpReceiver.
      * <p>
      * Subclasses should override (but remember to call {@code super}) to dispose their own state.
      * <p>
@@ -470,43 +567,34 @@ public abstract class HttpReceiver
      */
     protected void dispose()
     {
-        destroyDecoder(decoder);
-        decoder = null;
+        cleanup();
     }
 
-    private static void destroyDecoder(ContentDecoder decoder)
+    private void cleanup()
     {
-        if (decoder instanceof Destroyable)
-        {
-            ((Destroyable)decoder).destroy();
-        }
+        contentListeners = null;
+        if (decoder != null)
+            decoder.destroy();
+        decoder = null;
+        demand = 0;
+        stalled = false;
     }
 
     public boolean abort(HttpExchange exchange, Throwable failure)
     {
         // Update the state to avoid more response processing.
         boolean terminate;
-        out:
         while (true)
         {
             ResponseState current = responseState.get();
-            switch (current)
-            {
-                case FAILURE:
-                {
+            if (current == ResponseState.FAILURE)
                 return false;
-                }
-                default:
-                {
             if (updateResponseState(current, ResponseState.FAILURE))
             {
                 terminate = current != ResponseState.TRANSIENT;
-                        break out;
-                    }
                 break;
             }
         }
-        }
 
         this.failure = failure;
 
@@ -514,17 +602,19 @@ public abstract class HttpReceiver
 
         HttpResponse response = exchange.getResponse();
         if (LOG.isDebugEnabled())
-            LOG.debug("Response failure {} {} on {}: {}", response, exchange, getHttpChannel(), failure);
+            LOG.debug("Response abort {} {} on {}: {}", response, exchange, getHttpChannel(), failure);
         List<Response.ResponseListener> listeners = exchange.getConversation().getResponseListeners();
         ResponseNotifier notifier = getHttpDestination().getResponseNotifier();
         notifier.notifyFailure(listeners, response, failure);
 
+        // We want to deliver the "complete" event as last,
+        // so we emit it here only if no event handlers are
+        // executing, otherwise they will emit it.
         if (terminate)
         {
             // Mark atomically the response as terminated, with
             // respect to concurrency between request and response.
-            Result result = exchange.terminateResponse();
-            terminateResponse(exchange, result);
+            terminateResponse(exchange);
             return true;
         }
         else
@@ -591,46 +681,163 @@ public abstract class HttpReceiver
         FAILURE
     }
 
-    private class Decoder extends IteratingNestedCallback
+    /**
+     * <p>Wraps a list of content listeners, notifies them about content events and
+     * tracks individual listener demand to produce a global demand for content.</p>
+     */
+    private class ContentListeners
     {
-        private final ResponseNotifier notifier;
-        private final HttpResponse response;
-        private final ContentDecoder decoder;
-        private final ByteBuffer buffer;
-        private ByteBuffer decoded;
+        private final Map<Object, Long> demands = new ConcurrentHashMap<>();
+        private final LongConsumer demand = HttpReceiver.this::demand;
+        private final List<Response.DemandedContentListener> listeners;
 
-        public Decoder(ResponseNotifier notifier, HttpResponse response, ContentDecoder decoder, ByteBuffer buffer, Callback callback)
+        private ContentListeners(List<Response.ResponseListener> responseListeners)
         {
-            super(callback);
-            this.notifier = notifier;
-            this.response = response;
-            this.decoder = decoder;
-            this.buffer = buffer;
+            listeners = responseListeners.stream()
+                .filter(Response.DemandedContentListener.class::isInstance)
+                .map(Response.DemandedContentListener.class::cast)
+                .collect(Collectors.toList());
         }
 
-        @Override
-        protected Action process()
+        private boolean isEmpty()
+        {
+            return listeners.isEmpty();
+        }
+
+        private void notifyBeforeContent(HttpResponse response)
+        {
+            if (isEmpty())
+            {
+                // If no listeners, we want to proceed and consume any content.
+                demand.accept(1);
+            }
+            else
+            {
+                ResponseNotifier notifier = getHttpDestination().getResponseNotifier();
+                notifier.notifyBeforeContent(response, this::demand, listeners);
+            }
+        }
+
+        private void notifyContent(HttpResponse response, ByteBuffer buffer, Callback callback)
+        {
+            HttpReceiver.this.demand(d -> d - 1);
+            ResponseNotifier notifier = getHttpDestination().getResponseNotifier();
+            notifier.notifyContent(response, this::demand, buffer, callback, listeners);
+        }
+
+        private void demand(Object context, long value)
+        {
+            if (listeners.size() > 1)
+                accept(context, value);
+            else
+                demand.accept(value);
+        }
+
+        private void accept(Object context, long value)
+        {
+            // Increment the demand for the given listener.
+            demands.merge(context, value, MathUtils::cappedAdd);
+
+            // Check if we have demand from all listeners.
+            if (demands.size() == listeners.size())
+            {
+                long minDemand = Long.MAX_VALUE;
+                for (Long demand : demands.values())
+                {
+                    if (demand < minDemand)
+                        minDemand = demand;
+                }
+                if (minDemand > 0)
+                {
+                    // We are going to demand for minDemand content
+                    // chunks, so decrement the listener's demand by
+                    // minDemand and remove those that have no demand left.
+                    Iterator<Map.Entry<Object, Long>> iterator = demands.entrySet().iterator();
+                    while (iterator.hasNext())
+                    {
+                        Map.Entry<Object, Long> entry = iterator.next();
+                        long newValue = entry.getValue() - minDemand;
+                        if (newValue == 0)
+                            iterator.remove();
+                        else
+                            entry.setValue(newValue);
+                    }
+
+                    // Demand more content chunks for all the listeners.
+                    demand.accept(minDemand);
+                }
+            }
+        }
+    }
+
+    /**
+     * <p>Implements the decoding of content, producing decoded buffers only if there is demand for content.</p>
+     */
+    private class Decoder implements Destroyable
+    {
+        private final HttpResponse response;
+        private final ContentDecoder decoder;
+        private ByteBuffer encoded;
+        private Callback callback;
+
+        private Decoder(HttpResponse response, ContentDecoder decoder)
+        {
+            this.response = response;
+            this.decoder = Objects.requireNonNull(decoder);
+        }
+
+        private boolean decode(ByteBuffer encoded, Callback callback)
+        {
+            this.encoded = encoded;
+            this.callback = callback;
+            return decode();
+        }
+
+        private boolean decode()
         {
             while (true)
             {
-                decoded = decoder.decode(buffer);
-                if (decoded.hasRemaining())
+                ByteBuffer buffer;
+                while (true)
+                {
+                    buffer = decoder.decode(encoded);
+                    if (buffer.hasRemaining())
                         break;
-                if (!buffer.hasRemaining())
-                    return Action.SUCCEEDED;
+                    if (!encoded.hasRemaining())
+                    {
+                        callback.succeeded();
+                        encoded = null;
+                        callback = null;
+                        return true;
                     }
+                }
+                ByteBuffer decoded = buffer;
                 if (LOG.isDebugEnabled())
                     LOG.debug("Response content decoded ({}) {}{}{}", decoder, response, System.lineSeparator(), BufferUtil.toDetailString(decoded));
 
-            notifier.notifyContent(response, decoded, this, contentListeners);
-            return Action.SCHEDULED;
+                contentListeners.notifyContent(response, decoded, Callback.from(() -> decoder.release(decoded), callback::failed));
+
+                boolean hasDemand = hasDemandOrStall();
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Response content decoded {}, hasDemand={}", response, hasDemand);
+                if (!hasDemand)
+                    return false;
+            }
+        }
+
+        private void resume()
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Response content resuming decoding {}", response);
+            if (decode())
+                receive();
         }
 
         @Override
-        public void succeeded()
+        public void destroy()
         {
-            decoder.release(decoded);
-            super.succeeded();
+            if (decoder instanceof Destroyable)
+                ((Destroyable)decoder).destroy();
         }
     }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/HttpRequest.java

@@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiFunction;
+import java.util.function.LongConsumer;
 import java.util.function.Supplier;
 
 import org.eclipse.jetty.client.api.ContentProvider;
@@ -308,7 +309,7 @@ public class HttpRequest implements Request
     @Override
     public List<HttpCookie> getCookies()
     {
-        return cookies != null ? cookies : Collections.<HttpCookie>emptyList();
+        return cookies != null ? cookies : Collections.emptyList();
     }
 
     @Override
@@ -332,7 +333,7 @@ public class HttpRequest implements Request
     @Override
     public Map<String, Object> getAttributes()
     {
-        return attributes != null ? attributes : Collections.<String, Object>emptyMap();
+        return attributes != null ? attributes : Collections.emptyMap();
     }
 
     @Override
@@ -348,7 +349,7 @@ public class HttpRequest implements Request
         // This method is invoked often in a request/response conversation,
         // so we avoid allocation if there is no need to filter.
         if (type == null || requestListeners == null)
-            return requestListeners != null ? (List<T>)requestListeners : Collections.<T>emptyList();
+            return requestListeners != null ? (List<T>)requestListeners : Collections.emptyList();
 
         ArrayList<T> result = new ArrayList<>();
         for (RequestListener listener : requestListeners)
@@ -509,15 +510,16 @@ public class HttpRequest implements Request
     @Override
     public Request onResponseContent(final Response.ContentListener listener)
     {
-        this.responseListeners.add(new Response.AsyncContentListener()
+        this.responseListeners.add(new Response.DemandedContentListener()
         {
             @Override
-            public void onContent(Response response, ByteBuffer content, Callback callback)
+            public void onContent(Response response, LongConsumer demand, ByteBuffer content, Callback callback)
             {
                 try
                 {
                     listener.onContent(response, content);
                     callback.succeeded();
+                    demand.accept(1);
                 }
                 catch (Throwable x)
                 {
@@ -531,12 +533,36 @@ public class HttpRequest implements Request
     @Override
     public Request onResponseContentAsync(final Response.AsyncContentListener listener)
     {
-        this.responseListeners.add(new Response.AsyncContentListener()
+        this.responseListeners.add(new Response.DemandedContentListener()
         {
             @Override
-            public void onContent(Response response, ByteBuffer content, Callback callback)
+            public void onContent(Response response, LongConsumer demand, ByteBuffer content, Callback callback)
             {
-                listener.onContent(response, content, callback);
+                listener.onContent(response, content, Callback.from(() ->
+                {
+                    callback.succeeded();
+                    demand.accept(1);
+                }, callback::failed));
+            }
+        });
+        return this;
+    }
+
+    @Override
+    public Request onResponseContentDemanded(Response.DemandedContentListener listener)
+    {
+        this.responseListeners.add(new Response.DemandedContentListener()
+        {
+            @Override
+            public void onBeforeContent(Response response, LongConsumer demand)
+            {
+                listener.onBeforeContent(response, demand);
+            }
+
+            @Override
+            public void onContent(Response response, LongConsumer demand, ByteBuffer content, Callback callback)
+            {
+                listener.onContent(response, demand, content, callback);
             }
         });
         return this;
@@ -897,6 +923,6 @@ public class HttpRequest implements Request
     @Override
     public String toString()
     {
-        return String.format("%s[%s %s %s]@%x", this.getClass().getSimpleName(), getMethod(), getPath(), getVersion(), hashCode());
+        return String.format("%s[%s %s %s]@%x", getClass().getSimpleName(), getMethod(), getPath(), getVersion(), hashCode());
     }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/HttpSender.java

@@ -344,6 +344,9 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         if (exchange == null)
             return;
 
+        if (LOG.isDebugEnabled())
+            LOG.debug("Request failure " + exchange.getRequest(), failure);
+
         // Mark atomically the request as completed, with respect
         // to concurrency between request success and request failure.
         if (exchange.requestComplete(failure))
@@ -559,7 +562,7 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
 
         Request request = exchange.getRequest();
         if (LOG.isDebugEnabled())
-            LOG.debug("Request failure {} {} on {}: {}", request, exchange, getHttpChannel(), failure);
+            LOG.debug("Request abort {} {} on {}: {}", request, exchange, getHttpChannel(), failure);
         HttpDestination destination = getHttpChannel().getHttpDestination();
         destination.getRequestNotifier().notifyFailure(request, failure);
 

jetty-client/src/main/java/org/eclipse/jetty/client/ProxyConfiguration.java

@@ -26,6 +26,7 @@ import java.util.Set;
 
 import org.eclipse.jetty.io.ClientConnectionFactory;
 import org.eclipse.jetty.util.HostPort;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
 
 /**
  * The configuration of the forward proxy to use with {@link org.eclipse.jetty.client.HttpClient}.
@@ -64,12 +65,14 @@ public class ProxyConfiguration
         private final Set<String> excluded = new HashSet<>();
         private final Origin.Address address;
         private final boolean secure;
+        private final SslContextFactory.Client sslContextFactory;
         private final HttpDestination.Protocol protocol;
 
-        protected Proxy(Origin.Address address, boolean secure, HttpDestination.Protocol protocol)
+        protected Proxy(Origin.Address address, boolean secure, SslContextFactory.Client sslContextFactory, HttpDestination.Protocol protocol)
         {
             this.address = address;
             this.secure = secure;
+            this.sslContextFactory = sslContextFactory;
             this.protocol = protocol;
         }
 
@@ -89,6 +92,17 @@ public class ProxyConfiguration
             return secure;
         }
 
+        /**
+         * @return the optional SslContextFactory to use when connecting to proxies
+         */
+        public SslContextFactory.Client getSslContextFactory()
+        {
+            return sslContextFactory;
+        }
+
+        /**
+         * @return the protocol spoken by this proxy
+         */
         public HttpDestination.Protocol getProtocol()
         {
             return protocol;

jetty-client/src/main/java/org/eclipse/jetty/client/ResponseNotifier.java

@@ -21,6 +21,8 @@ package org.eclipse.jetty.client;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
 import java.util.List;
+import java.util.function.LongConsumer;
+import java.util.function.ObjLongConsumer;
 import java.util.stream.Collectors;
 
 import org.eclipse.jetty.client.api.ContentResponse;
@@ -103,36 +105,54 @@ public class ResponseNotifier
         }
     }
 
-    public void notifyContent(List<Response.ResponseListener> listeners, Response response, ByteBuffer buffer, Callback callback)
+    public void notifyBeforeContent(Response response, ObjLongConsumer<Object> demand, List<Response.DemandedContentListener> contentListeners)
     {
-        List<Response.AsyncContentListener> contentListeners = listeners.stream()
-            .filter(Response.AsyncContentListener.class::isInstance)
-            .map(Response.AsyncContentListener.class::cast)
-            .collect(Collectors.toList());
-        notifyContent(response, buffer, callback, contentListeners);
-    }
-
-    public void notifyContent(Response response, ByteBuffer buffer, Callback callback, List<Response.AsyncContentListener> contentListeners)
+        for (Response.DemandedContentListener listener : contentListeners)
         {
-        if (contentListeners.isEmpty())
-        {
-            callback.succeeded();
-        }
-        else
-        {
-            CountingCallback counter = new CountingCallback(callback, contentListeners.size());
-            for (Response.AsyncContentListener listener : contentListeners)
-            {
-                notifyContent(listener, response, buffer.slice(), counter);
-            }
+            notifyBeforeContent(listener, response, d -> demand.accept(listener, d));
         }
     }
 
-    private void notifyContent(Response.AsyncContentListener listener, Response response, ByteBuffer buffer, Callback callback)
+    private void notifyBeforeContent(Response.DemandedContentListener listener, Response response, LongConsumer demand)
     {
         try
         {
-            listener.onContent(response, buffer, callback);
+            listener.onBeforeContent(response, demand);
+        }
+        catch (Throwable x)
+        {
+            LOG.info("Exception while notifying listener " + listener, x);
+        }
+    }
+
+    public void notifyContent(Response response, ObjLongConsumer<Object> demand, ByteBuffer buffer, Callback callback, List<Response.DemandedContentListener> contentListeners)
+    {
+        int count = contentListeners.size();
+        if (count == 0)
+        {
+            callback.succeeded();
+            demand.accept(null, 1);
+        }
+        else if (count == 1)
+        {
+            Response.DemandedContentListener listener = contentListeners.get(0);
+            notifyContent(listener, response, d -> demand.accept(listener, d), buffer.slice(), callback);
+        }
+        else
+        {
+            callback = new CountingCallback(callback, count);
+            for (Response.DemandedContentListener listener : contentListeners)
+            {
+                notifyContent(listener, response, d -> demand.accept(listener, d), buffer.slice(), callback);
+            }
+        }
+    }
+
+    private void notifyContent(Response.DemandedContentListener listener, Response response, LongConsumer demand, ByteBuffer buffer, Callback callback)
+    {
+        try
+        {
+            listener.onContent(response, demand, buffer, callback);
         }
         catch (Throwable x)
         {
@@ -236,7 +256,15 @@ public class ResponseNotifier
         {
             byte[] content = ((ContentResponse)response).getContent();
             if (content != null && content.length > 0)
-                notifyContent(listeners, response, ByteBuffer.wrap(content), Callback.NOOP);
+            {
+                List<Response.DemandedContentListener> contentListeners = listeners.stream()
+                    .filter(Response.DemandedContentListener.class::isInstance)
+                    .map(Response.DemandedContentListener.class::cast)
+                    .collect(Collectors.toList());
+                ObjLongConsumer<Object> demand = (context, value) -> {};
+                notifyBeforeContent(response, demand, contentListeners);
+                notifyContent(response, demand, ByteBuffer.wrap(content), Callback.NOOP, contentListeners);
+            }
         }
     }
 

jetty-client/src/main/java/org/eclipse/jetty/client/Socks4Proxy.java

@@ -45,7 +45,7 @@ public class Socks4Proxy extends ProxyConfiguration.Proxy
 
     public Socks4Proxy(Origin.Address address, boolean secure)
     {
-        super(address, secure, null);
+        super(address, secure, null, null);
     }
 
     @Override
@@ -197,7 +197,7 @@ public class Socks4Proxy extends ProxyConfiguration.Proxy
                 HttpDestination destination = (HttpDestination)context.get(HttpClientTransport.HTTP_DESTINATION_CONTEXT_KEY);
                 ClientConnectionFactory connectionFactory = this.connectionFactory;
                 if (destination.isSecure())
-                    connectionFactory = destination.newSslClientConnectionFactory(connectionFactory);
+                    connectionFactory = destination.newSslClientConnectionFactory(null, connectionFactory);
                 org.eclipse.jetty.io.Connection newConnection = connectionFactory.newConnection(getEndPoint(), context);
                 getEndPoint().upgrade(newConnection);
                 if (LOG.isDebugEnabled())

jetty-client/src/main/java/org/eclipse/jetty/client/api/Request.java

@@ -370,6 +370,12 @@ public interface Request
      */
     Request onResponseContentAsync(Response.AsyncContentListener listener);
 
+    /**
+     * @param listener an asynchronous listener for response content events
+     * @return this request object
+     */
+    Request onResponseContentDemanded(Response.DemandedContentListener listener);
+
     /**
      * @param listener a listener for response success event
      * @return this request object

jetty-client/src/main/java/org/eclipse/jetty/client/api/Response.java

@@ -21,6 +21,8 @@ package org.eclipse.jetty.client.api;
 import java.nio.ByteBuffer;
 import java.util.EventListener;
 import java.util.List;
+import java.util.concurrent.Flow;
+import java.util.function.LongConsumer;
 
 import org.eclipse.jetty.client.util.BufferingResponseListener;
 import org.eclipse.jetty.http.HttpField;
@@ -109,7 +111,7 @@ public interface Response
     public interface HeaderListener extends ResponseListener
     {
         /**
-         * Callback method invoked when a response header has been received,
+         * Callback method invoked when a response header has been received and parsed,
          * returning whether the header should be processed or not.
          *
          * @param response the response containing the response line data and the headers so far
@@ -125,7 +127,7 @@ public interface Response
     public interface HeadersListener extends ResponseListener
     {
         /**
-         * Callback method invoked when the response headers have been received and parsed.
+         * Callback method invoked when all the response headers have been received and parsed.
          *
          * @param response the response containing the response line data and the headers
          */
@@ -133,14 +135,16 @@ public interface Response
     }
 
     /**
-     * Listener for the response content events.
+     * Synchronous listener for the response content events.
+     *
+     * @see AsyncContentListener
      */
     public interface ContentListener extends ResponseListener
     {
         /**
-         * Callback method invoked when the response content has been received.
-         * This method may be invoked multiple times, and the {@code content} buffer must be consumed
-         * before returning from this method.
+         * Callback method invoked when the response content has been received, parsed and there is demand.
+         * This method may be invoked multiple times, and the {@code content} buffer
+         * must be consumed (or copied) before returning from this method.
          *
          * @param response the response containing the response line data and the headers
          * @param content the content bytes received
@@ -148,18 +152,60 @@ public interface Response
         public void onContent(Response response, ByteBuffer content);
     }
 
+    /**
+     * Asynchronous listener for the response content events.
+     *
+     * @see DemandedContentListener
+     */
     public interface AsyncContentListener extends ResponseListener
     {
         /**
-         * Callback method invoked asynchronously when the response content has been received.
+         * Callback method invoked when the response content has been received, parsed and there is demand.
+         * The {@code callback} object should be succeeded to signal that the
+         * {@code content} buffer has been consumed and to demand more content.
          *
          * @param response the response containing the response line data and the headers
          * @param content the content bytes received
-         * @param callback the callback to call when the content is consumed.
+         * @param callback the callback to call when the content is consumed and to demand more content
          */
         public void onContent(Response response, ByteBuffer content, Callback callback);
     }
 
+    /**
+     * Asynchronous listener for the response content events.
+     */
+    public interface DemandedContentListener extends ResponseListener
+    {
+        /**
+         * Callback method invoked before response content events.
+         * The {@code demand} object should be used to demand content, otherwise
+         * the demand remains at zero (no demand) and
+         * {@link #onContent(Response, LongConsumer, ByteBuffer, Callback)} will
+         * not be invoked even if content has been received and parsed.
+         *
+         * @param response the response containing the response line data and the headers
+         * @param demand the object that allows to demand content buffers
+         */
+        public default void onBeforeContent(Response response, LongConsumer demand)
+        {
+            demand.accept(1);
+        }
+
+        /**
+         * Callback method invoked when the response content has been received.
+         * The {@code callback} object should be succeeded to signal that the
+         * {@code content} buffer has been consumed.
+         * The {@code demand} object should be used to demand more content,
+         * similarly to {@link Flow.Subscription#request(long)}.
+         *
+         * @param response the response containing the response line data and the headers
+         * @param demand the object that allows to demand content buffers
+         * @param content the content bytes received
+         * @param callback the callback to call when the content is consumed
+         */
+        public void onContent(Response response, LongConsumer demand, ByteBuffer content, Callback callback);
+    }
+
     /**
      * Listener for the response succeeded event.
      */
@@ -212,7 +258,7 @@ public interface Response
     /**
      * Listener for all response events.
      */
-    public interface Listener extends BeginListener, HeaderListener, HeadersListener, ContentListener, AsyncContentListener, SuccessListener, FailureListener, CompleteListener
+    public interface Listener extends BeginListener, HeaderListener, HeadersListener, ContentListener, AsyncContentListener, DemandedContentListener, SuccessListener, FailureListener, CompleteListener
     {
         /**
          * An empty implementation of {@link Listener}
@@ -254,6 +300,16 @@ public interface Response
                 }
             }
 
+            @Override
+            public void onContent(Response response, LongConsumer demand, ByteBuffer content, Callback callback)
+            {
+                onContent(response, content, Callback.from(() ->
+                {
+                    callback.succeeded();
+                    demand.accept(1);
+                }, callback::failed));
+            }
+
             @Override
             public void onSuccess(Response response)
             {

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpClientTransportOverHTTP.java

@@ -33,6 +33,7 @@ import org.eclipse.jetty.io.ClientConnector;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.util.ProcessorUtils;
 import org.eclipse.jetty.util.Promise;
+import org.eclipse.jetty.util.annotation.ManagedAttribute;
 import org.eclipse.jetty.util.annotation.ManagedObject;
 
 @ManagedObject("The HTTP/1.1 client transport")
@@ -40,6 +41,9 @@ public class HttpClientTransportOverHTTP extends AbstractConnectorHttpClientTran
 {
     public static final HttpDestination.Protocol HTTP11 = new HttpDestination.Protocol(List.of("http/1.1"), false);
 
+    private int headerCacheSize = 1024;
+    private boolean headerCacheCaseSensitive;
+
     public HttpClientTransportOverHTTP()
     {
         this(Math.max(1, ProcessorUtils.availableProcessors() / 2));
@@ -75,7 +79,7 @@ public class HttpClientTransportOverHTTP extends AbstractConnectorHttpClientTran
         HttpDestination destination = (HttpDestination)context.get(HTTP_DESTINATION_CONTEXT_KEY);
         @SuppressWarnings("unchecked")
         Promise<Connection> promise = (Promise<Connection>)context.get(HTTP_CONNECTION_PROMISE_CONTEXT_KEY);
-        org.eclipse.jetty.io.Connection connection = newHttpConnection(endPoint, destination, promise);
+        var connection = newHttpConnection(endPoint, destination, promise);
         if (LOG.isDebugEnabled())
             LOG.debug("Created {}", connection);
         return customize(connection, context);
@@ -85,4 +89,26 @@ public class HttpClientTransportOverHTTP extends AbstractConnectorHttpClientTran
     {
         return new HttpConnectionOverHTTP(endPoint, destination, promise);
     }
+
+    @ManagedAttribute("The maximum allowed size in bytes for an HTTP header field cache")
+    public int getHeaderCacheSize()
+    {
+        return headerCacheSize;
+    }
+
+    public void setHeaderCacheSize(int headerCacheSize)
+    {
+        this.headerCacheSize = headerCacheSize;
+    }
+
+    @ManagedAttribute("Whether the header field cache is case sensitive")
+    public boolean isHeaderCacheCaseSensitive()
+    {
+        return headerCacheCaseSensitive;
+    }
+
+    public void setHeaderCacheCaseSensitive(boolean headerCacheCaseSensitive)
+    {
+        this.headerCacheCaseSensitive = headerCacheCaseSensitive;
+    }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpConnectionOverHTTP.java

@@ -154,19 +154,9 @@ public class HttpConnectionOverHTTP extends AbstractConnection implements IConne
 
     @Override
     public void onFillable()
-    {
-        HttpExchange exchange = channel.getHttpExchange();
-        if (exchange != null)
     {
         channel.receive();
     }
-        else
-        {
-            // If there is no exchange, then could be either a remote close,
-            // or garbage bytes; in both cases we close the connection
-            close();
-        }
-    }
 
     @Override
     public ByteBuffer onUpgradeFrom()

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpReceiverOverHTTP.java

@@ -22,6 +22,7 @@ import java.io.EOFException;
 import java.nio.ByteBuffer;
 
 import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.HttpClientTransport;
 import org.eclipse.jetty.client.HttpExchange;
 import org.eclipse.jetty.client.HttpReceiver;
 import org.eclipse.jetty.client.HttpResponse;
@@ -34,20 +35,29 @@ import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.io.ByteBufferPool;
 import org.eclipse.jetty.io.EndPoint;
+import org.eclipse.jetty.io.RetainableByteBuffer;
 import org.eclipse.jetty.util.BufferUtil;
-import org.eclipse.jetty.util.CompletableCallback;
+import org.eclipse.jetty.util.Callback;
 
 public class HttpReceiverOverHTTP extends HttpReceiver implements HttpParser.ResponseHandler
 {
     private final HttpParser parser;
-    private ByteBuffer buffer;
+    private RetainableByteBuffer networkBuffer;
     private boolean shutdown;
     private boolean complete;
 
     public HttpReceiverOverHTTP(HttpChannelOverHTTP channel)
     {
         super(channel);
-        parser = new HttpParser(this, -1, channel.getHttpDestination().getHttpClient().getHttpCompliance());
+        HttpClient httpClient = channel.getHttpDestination().getHttpClient();
+        parser = new HttpParser(this, -1, httpClient.getHttpCompliance());
+        HttpClientTransport transport = httpClient.getTransport();
+        if (transport instanceof HttpClientTransportOverHTTP)
+        {
+            HttpClientTransportOverHTTP httpTransport = (HttpClientTransportOverHTTP)transport;
+            parser.setHeaderCacheSize(httpTransport.getHeaderCacheSize());
+            parser.setHeaderCacheCaseSensitive(httpTransport.isHeaderCacheCaseSensitive());
+        }
     }
 
     @Override
@@ -63,41 +73,66 @@ public class HttpReceiverOverHTTP extends HttpReceiver implements HttpParser.Res
 
     protected ByteBuffer getResponseBuffer()
     {
-        return buffer;
+        return networkBuffer == null ? null : networkBuffer.getBuffer();
     }
 
+    @Override
     public void receive()
     {
-        if (buffer == null)
-            acquireBuffer();
+        if (networkBuffer == null)
+            acquireNetworkBuffer();
         process();
     }
 
-    private void acquireBuffer()
+    private void acquireNetworkBuffer()
     {
-        HttpClient client = getHttpDestination().getHttpClient();
-        ByteBufferPool bufferPool = client.getByteBufferPool();
-        buffer = bufferPool.acquire(client.getResponseBufferSize(), true);
+        networkBuffer = newNetworkBuffer();
+        if (LOG.isDebugEnabled())
+            LOG.debug("Acquired {}", networkBuffer);
     }
 
-    private void releaseBuffer()
+    private void reacquireNetworkBuffer()
     {
-        if (buffer == null)
+        RetainableByteBuffer currentBuffer = networkBuffer;
+        if (currentBuffer == null)
             throw new IllegalStateException();
-        if (BufferUtil.hasContent(buffer))
+
+        if (currentBuffer.hasRemaining())
             throw new IllegalStateException();
+
+        currentBuffer.release();
+        networkBuffer = newNetworkBuffer();
+        if (LOG.isDebugEnabled())
+            LOG.debug("Reacquired {} <- {}", currentBuffer, networkBuffer);
+    }
+
+    private RetainableByteBuffer newNetworkBuffer()
+    {
         HttpClient client = getHttpDestination().getHttpClient();
         ByteBufferPool bufferPool = client.getByteBufferPool();
-        bufferPool.release(buffer);
-        buffer = null;
+        return new RetainableByteBuffer(bufferPool, client.getResponseBufferSize(), true);
+    }
+
+    private void releaseNetworkBuffer()
+    {
+        if (networkBuffer == null)
+            throw new IllegalStateException();
+        if (networkBuffer.hasRemaining())
+            throw new IllegalStateException();
+        networkBuffer.release();
+        if (LOG.isDebugEnabled())
+            LOG.debug("Released {}", networkBuffer);
+        networkBuffer = null;
     }
 
     protected ByteBuffer onUpgradeFrom()
     {
-        if (BufferUtil.hasContent(buffer))
+        if (networkBuffer.hasRemaining())
         {
-            ByteBuffer upgradeBuffer = ByteBuffer.allocate(buffer.remaining());
-            upgradeBuffer.put(buffer).flip();
+            ByteBuffer upgradeBuffer = BufferUtil.allocate(networkBuffer.remaining());
+            BufferUtil.clearToFill(upgradeBuffer);
+            BufferUtil.put(networkBuffer.getBuffer(), upgradeBuffer);
+            BufferUtil.flipToFlush(upgradeBuffer, 0);
             return upgradeBuffer;
         }
         return null;
@@ -111,39 +146,42 @@ public class HttpReceiverOverHTTP extends HttpReceiver implements HttpParser.Res
             EndPoint endPoint = connection.getEndPoint();
             while (true)
             {
-                boolean upgraded = connection != endPoint.getConnection();
+                // Always parse even empty buffers to advance the parser.
+                boolean stopProcessing = parse();
 
                 // Connection may be closed or upgraded in a parser callback.
+                boolean upgraded = connection != endPoint.getConnection();
                 if (connection.isClosed() || upgraded)
                 {
                     if (LOG.isDebugEnabled())
                         LOG.debug("{} {}", connection, upgraded ? "upgraded" : "closed");
-                    releaseBuffer();
+                    releaseNetworkBuffer();
                     return;
                 }
 
-                if (parse())
+                if (stopProcessing)
                     return;
 
-                int read = endPoint.fill(buffer);
+                if (networkBuffer.getReferences() > 1)
+                    reacquireNetworkBuffer();
+
+                int read = endPoint.fill(networkBuffer.getBuffer());
                 if (LOG.isDebugEnabled())
-                    LOG.debug("Read {} bytes {} from {}", read, BufferUtil.toDetailString(buffer), endPoint);
+                    LOG.debug("Read {} bytes in {} from {}", read, networkBuffer, endPoint);
 
                 if (read > 0)
                 {
                     connection.addBytesIn(read);
-                    if (parse())
-                        return;
                 }
                 else if (read == 0)
                 {
-                    releaseBuffer();
+                    releaseNetworkBuffer();
                     fillInterested();
                     return;
                 }
                 else
                 {
-                    releaseBuffer();
+                    releaseNetworkBuffer();
                     shutdown();
                     return;
                 }
@@ -153,9 +191,8 @@ public class HttpReceiverOverHTTP extends HttpReceiver implements HttpParser.Res
         {
             if (LOG.isDebugEnabled())
                 LOG.debug(x);
-            BufferUtil.clear(buffer);
-            if (buffer != null)
-                releaseBuffer();
+            networkBuffer.clear();
+            releaseNetworkBuffer();
             failAndClose(x);
         }
     }
@@ -169,20 +206,20 @@ public class HttpReceiverOverHTTP extends HttpReceiver implements HttpParser.Res
     {
         while (true)
         {
-            boolean handle = parser.parseNext(buffer);
+            boolean handle = parser.parseNext(networkBuffer.getBuffer());
             boolean complete = this.complete;
             this.complete = false;
             if (LOG.isDebugEnabled())
-                LOG.debug("Parsed {}, remaining {} {}", handle, buffer.remaining(), parser);
+                LOG.debug("Parsed {}, remaining {} {}", handle, networkBuffer.remaining(), parser);
             if (handle)
                 return true;
-            if (!buffer.hasRemaining())
+            if (networkBuffer.isEmpty())
                 return false;
             if (complete)
             {
                 if (LOG.isDebugEnabled())
-                    LOG.debug("Discarding unexpected content after response: {}", BufferUtil.toDetailString(buffer));
-                BufferUtil.clear(buffer);
+                    LOG.debug("Discarding unexpected content after response: {}", networkBuffer);
+                networkBuffer.clear();
                 return false;
             }
         }
@@ -215,32 +252,18 @@ public class HttpReceiverOverHTTP extends HttpReceiver implements HttpParser.Res
     }
 
     @Override
-    public int getHeaderCacheSize()
-    {
-        // TODO get from configuration
-        return 4096;
-    }
-
-    @Override
-    public boolean isHeaderCacheCaseSensitive()
-    {
-        // TODO get from configuration
-        return false;
-    }
-
-    @Override
-    public boolean startResponse(HttpVersion version, int status, String reason)
+    public void startResponse(HttpVersion version, int status, String reason)
     {
         HttpExchange exchange = getHttpExchange();
         if (exchange == null)
-            return false;
+            return;
 
         String method = exchange.getRequest().getMethod();
         parser.setHeadResponse(HttpMethod.HEAD.is(method) ||
             (HttpMethod.CONNECT.is(method) && status == HttpStatus.OK_200));
         exchange.getResponse().version(version).status(status).reason(reason);
 
-        return !responseBegin(exchange);
+        responseBegin(exchange);
     }
 
     @Override
@@ -276,26 +299,8 @@ public class HttpReceiverOverHTTP extends HttpReceiver implements HttpParser.Res
         if (exchange == null)
             return false;
 
-        CompletableCallback callback = new CompletableCallback()
-        {
-            @Override
-            public void resume()
-            {
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Content consumed asynchronously, resuming processing");
-                process();
-            }
-
-            @Override
-            public void abort(Throwable x)
-            {
-                failAndClose(x);
-            }
-        };
-        // Do not short circuit these calls.
-        boolean proceed = responseContent(exchange, buffer, callback);
-        boolean async = callback.tryComplete();
-        return !proceed || async;
+        networkBuffer.retain();
+        return !responseContent(exchange, buffer, Callback.from(networkBuffer::release, this::failAndClose));
     }
 
     @Override

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpSenderOverHTTP.java

@@ -288,7 +288,6 @@ public class HttpSenderOverHTTP extends HttpSender
         public void failed(Throwable x)
         {
             release();
-            callback.failed(x);
             super.failed(x);
         }
 
@@ -299,6 +298,13 @@ public class HttpSenderOverHTTP extends HttpSender
             callback.succeeded();
         }
 
+        @Override
+        protected void onCompleteFailure(Throwable cause)
+        {
+            super.onCompleteFailure(cause);
+            callback.failed(cause);
+        }
+
         private void release()
         {
             ByteBufferPool bufferPool = httpClient.getByteBufferPool();

jetty-client/src/test/java/org/eclipse/jetty/client/EmptyServerHandler.java

@@ -26,10 +26,10 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.handler.AbstractHandler;
 
-public class EmptyServerHandler extends AbstractHandler.ErrorDispatchHandler
+public class EmptyServerHandler extends AbstractHandler
 {
     @Override
-    protected final void doNonErrorHandle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+    public final void handle(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
     {
         jettyRequest.setHandled(true);
         service(target, jettyRequest, request, response);

jetty-client/src/test/java/org/eclipse/jetty/client/HostnameVerificationTest.java

@@ -37,7 +37,6 @@ import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.hamcrest.Matchers;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -48,7 +47,6 @@ import static org.junit.jupiter.api.Assertions.fail;
  * This test class runs tests to make sure that hostname verification (http://www.ietf.org/rfc/rfc2818.txt
  * section 3.1) is configurable in SslContextFactory and works as expected.
  */
-@Disabled
 public class HostnameVerificationTest
 {
     private SslContextFactory.Client clientSslContextFactory = new SslContextFactory.Client();
@@ -100,18 +98,15 @@ public class HostnameVerificationTest
     {
         client.stop();
         server.stop();
-        server.join();
     }
 
     /**
      * This test is supposed to verify that hostname verification works as described in:
      * http://www.ietf.org/rfc/rfc2818.txt section 3.1. It uses a certificate with a common name different to localhost
      * and sends a request to localhost. This should fail with an SSLHandshakeException.
-     *
-     * @throws Exception on test failure
      */
     @Test
-    public void simpleGetWithHostnameVerificationEnabledTest() throws Exception
+    public void simpleGetWithHostnameVerificationEnabledTest()
     {
         clientSslContextFactory.setEndpointIdentificationAlgorithm("HTTPS");
         String uri = "https://localhost:" + connector.getLocalPort() + "/";
@@ -119,8 +114,16 @@ public class HostnameVerificationTest
         ExecutionException x = assertThrows(ExecutionException.class, () -> client.GET(uri));
         Throwable cause = x.getCause();
         assertThat(cause, Matchers.instanceOf(SSLHandshakeException.class));
-        Throwable root = cause.getCause().getCause();
-        assertThat(root, Matchers.instanceOf(CertificateException.class));
+
+        // Search for the CertificateException.
+        Throwable certificateException = cause.getCause();
+        while (certificateException != null)
+        {
+            if (certificateException instanceof CertificateException)
+                break;
+            certificateException = certificateException.getCause();
+        }
+        assertThat(certificateException, Matchers.instanceOf(CertificateException.class));
     }
 
     /**

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientCustomProxyTest.java

@@ -116,7 +116,7 @@ public class HttpClientCustomProxyTest
     {
         private CAFEBABEProxy(Origin.Address address, boolean secure)
         {
-            super(address, secure, null);
+            super(address, secure, null, null);
         }
 
         @Override

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientGZIPTest.java

@@ -24,7 +24,7 @@ import java.io.InterruptedIOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Random;
-import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.zip.GZIPOutputStream;
 import javax.servlet.ServletOutputStream;
@@ -44,7 +44,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.lessThan;
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 public class HttpClientGZIPTest extends AbstractHttpClientServerTest
@@ -217,16 +217,11 @@ public class HttpClientGZIPTest extends AbstractHttpClientServerTest
             }
         });
 
-        final CountDownLatch latch = new CountDownLatch(1);
+        assertThrows(ExecutionException.class, () ->
             client.newRequest("localhost", connector.getLocalPort())
                 .scheme(scenario.getScheme())
-            .send(result ->
-            {
-                if (result.isFailed())
-                    latch.countDown();
-            });
-
-        assertTrue(latch.await(5, TimeUnit.SECONDS));
+                .timeout(5, TimeUnit.SECONDS)
+                .send());
     }
 
     @ParameterizedTest

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTLSTest.java

@@ -19,18 +19,25 @@
 package org.eclipse.jetty.client;
 
 import java.io.BufferedReader;
+import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.ServerSocket;
 import java.net.Socket;
 import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLEngineResult;
 import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.SSLPeerUnverifiedException;
 import javax.net.ssl.SSLSocket;
 
@@ -39,17 +46,26 @@ import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpScheme;
 import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.io.ByteBufferPool;
 import org.eclipse.jetty.io.ClientConnectionFactory;
 import org.eclipse.jetty.io.ClientConnector;
+import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.io.ssl.SslClientConnectionFactory;
+import org.eclipse.jetty.io.ssl.SslConnection;
 import org.eclipse.jetty.io.ssl.SslHandshakeListener;
+import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SslConnectionFactory;
 import org.eclipse.jetty.util.StringUtil;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.ExecutorThreadPool;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.hamcrest.Matchers;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.EnabledOnJre;
@@ -68,7 +84,6 @@ public class HttpClientTLSTest
     private Server server;
     private ServerConnector connector;
     private HttpClient client;
-    private SSLSocket sslSocket;
 
     private void startServer(SslContextFactory.Server sslContextFactory, Handler handler) throws Exception
     {
@@ -424,16 +439,16 @@ public class HttpClientTLSTest
 
         String host = "localhost";
         int port = connector.getLocalPort();
-        Socket socket = new Socket(host, port);
-        sslSocket = (SSLSocket)clientTLSFactory.getSslContext().getSocketFactory().createSocket(socket, host, port, true);
+        Socket socket1 = new Socket(host, port);
+        SSLSocket sslSocket1 = (SSLSocket)clientTLSFactory.getSslContext().getSocketFactory().createSocket(socket1, host, port, true);
         CountDownLatch handshakeLatch1 = new CountDownLatch(1);
         AtomicReference<byte[]> session1 = new AtomicReference<>();
-        sslSocket.addHandshakeCompletedListener(event ->
+        sslSocket1.addHandshakeCompletedListener(event ->
         {
             session1.set(event.getSession().getId());
             handshakeLatch1.countDown();
         });
-        sslSocket.startHandshake();
+        sslSocket1.startHandshake();
         assertTrue(handshakeLatch1.await(5, TimeUnit.SECONDS));
 
         // In TLS 1.3 the server sends a NewSessionTicket post-handshake message
@@ -441,29 +456,29 @@ public class HttpClientTLSTest
 
         assertThrows(SocketTimeoutException.class, () ->
         {
-            sslSocket.setSoTimeout(1000);
-            sslSocket.getInputStream().read();
+            sslSocket1.setSoTimeout(1000);
+            sslSocket1.getInputStream().read();
         });
 
         // The client closes abruptly.
-        socket.close();
+        socket1.close();
 
         // Try again and compare the session ids.
-        socket = new Socket(host, port);
-        sslSocket = (SSLSocket)clientTLSFactory.getSslContext().getSocketFactory().createSocket(socket, host, port, true);
+        Socket socket2 = new Socket(host, port);
+        SSLSocket sslSocket2 = (SSLSocket)clientTLSFactory.getSslContext().getSocketFactory().createSocket(socket2, host, port, true);
         CountDownLatch handshakeLatch2 = new CountDownLatch(1);
         AtomicReference<byte[]> session2 = new AtomicReference<>();
-        sslSocket.addHandshakeCompletedListener(event ->
+        sslSocket2.addHandshakeCompletedListener(event ->
         {
             session2.set(event.getSession().getId());
             handshakeLatch2.countDown();
         });
-        sslSocket.startHandshake();
+        sslSocket2.startHandshake();
         assertTrue(handshakeLatch2.await(5, TimeUnit.SECONDS));
 
         assertArrayEquals(session1.get(), session2.get());
 
-        sslSocket.close();
+        sslSocket2.close();
     }
 
     @Test
@@ -482,10 +497,10 @@ public class HttpClientTLSTest
             client = new HttpClient(new HttpClientTransportOverHTTP(clientConnector))
             {
                 @Override
-                protected ClientConnectionFactory newSslClientConnectionFactory(ClientConnectionFactory connectionFactory)
+                protected ClientConnectionFactory newSslClientConnectionFactory(SslContextFactory.Client sslContextFactory, ClientConnectionFactory connectionFactory)
                 {
-                    SslClientConnectionFactory ssl = (SslClientConnectionFactory)super.newSslClientConnectionFactory(connectionFactory);
-                    ssl.setAllowMissingCloseMessage(false);
+                    SslClientConnectionFactory ssl = (SslClientConnectionFactory)super.newSslClientConnectionFactory(sslContextFactory, connectionFactory);
+                    ssl.setRequireCloseMessage(true);
                     return ssl;
                 }
             };
@@ -512,19 +527,19 @@ public class HttpClientTLSTest
                         break;
                 }
 
-                // If the response is Content-Length delimited, allowing the
-                // missing TLS Close Message is fine because the application
-                // will see a EOFException anyway.
-                // If the response is connection delimited, allowing the
-                // missing TLS Close Message is bad because the application
-                // will see a successful response with truncated content.
+                // If the response is Content-Length delimited, the lack of
+                // the TLS Close Message is fine because the application
+                // will see a EOFException anyway: the Content-Length and
+                // the actual content bytes count won't match.
+                // If the response is connection delimited, the lack of the
+                // TLS Close Message is bad because the application will
+                // see a successful response, but with truncated content.
 
-                // Verify that by not allowing the missing
-                // TLS Close Message we get a response failure.
+                // Verify that by requiring the TLS Close Message we get
+                // a response failure.
 
                 byte[] half = new byte[8];
                 String response = "HTTP/1.1 200 OK\r\n" +
-//                        "Content-Length: " + (half.length * 2) + "\r\n" +
                     "Connection: close\r\n" +
                     "\r\n";
                 OutputStream output = sslSocket.getOutputStream();
@@ -564,4 +579,368 @@ public class HttpClientTLSTest
 
         assertTrue(latch.await(5, TimeUnit.SECONDS));
     }
+
+    @Test
+    public void testNeverUsedConnectionThenServerIdleTimeout() throws Exception
+    {
+        long idleTimeout = 2000;
+
+        SslContextFactory.Server serverTLSFactory = createServerSslContextFactory();
+        QueuedThreadPool serverThreads = new QueuedThreadPool();
+        serverThreads.setName("server");
+        server = new Server(serverThreads);
+        HttpConfiguration httpConfig = new HttpConfiguration();
+        httpConfig.addCustomizer(new SecureRequestCustomizer());
+        HttpConnectionFactory http = new HttpConnectionFactory(httpConfig);
+        AtomicLong serverBytes = new AtomicLong();
+        SslConnectionFactory ssl = new SslConnectionFactory(serverTLSFactory, http.getProtocol())
+        {
+            @Override
+            protected SslConnection newSslConnection(Connector connector, EndPoint endPoint, SSLEngine engine)
+            {
+                return new SslConnection(connector.getByteBufferPool(), connector.getExecutor(), endPoint, engine, isDirectBuffersForEncryption(), isDirectBuffersForDecryption())
+                {
+                    @Override
+                    protected int networkFill(ByteBuffer input) throws IOException
+                    {
+                        int n = super.networkFill(input);
+                        if (n > 0)
+                            serverBytes.addAndGet(n);
+                        return n;
+                    }
+                };
+            }
+        };
+        connector = new ServerConnector(server, 1, 1, ssl, http);
+        connector.setIdleTimeout(idleTimeout);
+        server.addConnector(connector);
+        server.setHandler(new EmptyServerHandler());
+        server.start();
+
+        SslContextFactory.Client clientTLSFactory = createClientSslContextFactory();
+        ClientConnector clientConnector = new ClientConnector();
+        clientConnector.setSelectors(1);
+        clientConnector.setSslContextFactory(clientTLSFactory);
+        QueuedThreadPool clientThreads = new QueuedThreadPool();
+        clientThreads.setName("client");
+        clientConnector.setExecutor(clientThreads);
+        AtomicLong clientBytes = new AtomicLong();
+        client = new HttpClient(new HttpClientTransportOverHTTP(clientConnector))
+        {
+            @Override
+            protected ClientConnectionFactory newSslClientConnectionFactory(SslContextFactory.Client sslContextFactory, ClientConnectionFactory connectionFactory)
+            {
+                if (sslContextFactory == null)
+                    sslContextFactory = getSslContextFactory();
+                return new SslClientConnectionFactory(sslContextFactory, getByteBufferPool(), getExecutor(), connectionFactory)
+                {
+                    @Override
+                    protected SslConnection newSslConnection(ByteBufferPool byteBufferPool, Executor executor, EndPoint endPoint, SSLEngine engine)
+                    {
+                        return new SslConnection(byteBufferPool, executor, endPoint, engine, isDirectBuffersForEncryption(), isDirectBuffersForDecryption())
+                        {
+                            @Override
+                            protected int networkFill(ByteBuffer input) throws IOException
+                            {
+                                int n = super.networkFill(input);
+                                if (n > 0)
+                                    clientBytes.addAndGet(n);
+                                return n;
+                            }
+                        };
+                    }
+                };
+            }
+        };
+        client.setExecutor(clientThreads);
+        client.start();
+
+        // Create a connection but don't use it.
+        Origin origin = new Origin(HttpScheme.HTTPS.asString(), "localhost", connector.getLocalPort());
+        HttpDestination destination = client.resolveDestination(new HttpDestination.Key(origin, null));
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
+        // Trigger the creation of a new connection, but don't use it.
+        connectionPool.tryCreate(-1);
+        // Verify that the connection has been created.
+        while (true)
+        {
+            Thread.sleep(50);
+            if (connectionPool.getConnectionCount() == 1)
+                break;
+        }
+
+        // Wait for the server to idle timeout the connection.
+        Thread.sleep(idleTimeout + idleTimeout / 2);
+
+        // The connection should be gone from the connection pool.
+        assertEquals(0, connectionPool.getConnectionCount(), connectionPool.dump());
+        assertEquals(0, serverBytes.get());
+        assertEquals(0, clientBytes.get());
+    }
+
+    @Test
+    public void testNeverUsedConnectionThenClientIdleTimeout() throws Exception
+    {
+        SslContextFactory.Server serverTLSFactory = createServerSslContextFactory();
+        QueuedThreadPool serverThreads = new QueuedThreadPool();
+        serverThreads.setName("server");
+        server = new Server(serverThreads);
+        HttpConfiguration httpConfig = new HttpConfiguration();
+        httpConfig.addCustomizer(new SecureRequestCustomizer());
+        HttpConnectionFactory http = new HttpConnectionFactory(httpConfig);
+        AtomicLong serverBytes = new AtomicLong();
+        SslConnectionFactory ssl = new SslConnectionFactory(serverTLSFactory, http.getProtocol())
+        {
+            @Override
+            protected SslConnection newSslConnection(Connector connector, EndPoint endPoint, SSLEngine engine)
+            {
+                return new SslConnection(connector.getByteBufferPool(), connector.getExecutor(), endPoint, engine, isDirectBuffersForEncryption(), isDirectBuffersForDecryption())
+                {
+                    @Override
+                    protected int networkFill(ByteBuffer input) throws IOException
+                    {
+                        int n = super.networkFill(input);
+                        if (n > 0)
+                            serverBytes.addAndGet(n);
+                        return n;
+                    }
+                };
+            }
+        };
+        connector = new ServerConnector(server, 1, 1, ssl, http);
+        server.addConnector(connector);
+        server.setHandler(new EmptyServerHandler());
+        server.start();
+
+        long idleTimeout = 2000;
+
+        SslContextFactory.Client clientTLSFactory = createClientSslContextFactory();
+        ClientConnector clientConnector = new ClientConnector();
+        clientConnector.setSelectors(1);
+        clientConnector.setSslContextFactory(clientTLSFactory);
+        QueuedThreadPool clientThreads = new QueuedThreadPool();
+        clientThreads.setName("client");
+        clientConnector.setExecutor(clientThreads);
+        AtomicLong clientBytes = new AtomicLong();
+        client = new HttpClient(new HttpClientTransportOverHTTP(clientConnector))
+        {
+            @Override
+            protected ClientConnectionFactory newSslClientConnectionFactory(SslContextFactory.Client sslContextFactory, ClientConnectionFactory connectionFactory)
+            {
+                if (sslContextFactory == null)
+                    sslContextFactory = getSslContextFactory();
+                return new SslClientConnectionFactory(sslContextFactory, getByteBufferPool(), getExecutor(), connectionFactory)
+                {
+                    @Override
+                    protected SslConnection newSslConnection(ByteBufferPool byteBufferPool, Executor executor, EndPoint endPoint, SSLEngine engine)
+                    {
+                        return new SslConnection(byteBufferPool, executor, endPoint, engine, isDirectBuffersForEncryption(), isDirectBuffersForDecryption())
+                        {
+                            @Override
+                            protected int networkFill(ByteBuffer input) throws IOException
+                            {
+                                int n = super.networkFill(input);
+                                if (n > 0)
+                                    clientBytes.addAndGet(n);
+                                return n;
+                            }
+                        };
+                    }
+                };
+            }
+        };
+        client.setIdleTimeout(idleTimeout);
+        client.setExecutor(clientThreads);
+        client.start();
+
+        // Create a connection but don't use it.
+        Origin origin = new Origin(HttpScheme.HTTPS.asString(), "localhost", connector.getLocalPort());
+        HttpDestination destination = client.resolveDestination(new HttpDestination.Key(origin, null));
+        DuplexConnectionPool connectionPool = (DuplexConnectionPool)destination.getConnectionPool();
+        // Trigger the creation of a new connection, but don't use it.
+        connectionPool.tryCreate(-1);
+        // Verify that the connection has been created.
+        while (true)
+        {
+            Thread.sleep(50);
+            if (connectionPool.getConnectionCount() == 1)
+                break;
+        }
+
+        // Wait for the client to idle timeout the connection.
+        Thread.sleep(idleTimeout + idleTimeout / 2);
+
+        // The connection should be gone from the connection pool.
+        assertEquals(0, connectionPool.getConnectionCount(), connectionPool.dump());
+        assertEquals(0, serverBytes.get());
+        assertEquals(0, clientBytes.get());
+    }
+
+    @Test
+    public void testSSLEngineClosedDuringHandshake() throws Exception
+    {
+        SslContextFactory.Server serverTLSFactory = createServerSslContextFactory();
+        startServer(serverTLSFactory, new EmptyServerHandler());
+
+        SslContextFactory.Client clientTLSFactory = createClientSslContextFactory();
+        ClientConnector clientConnector = new ClientConnector();
+        clientConnector.setSelectors(1);
+        clientConnector.setSslContextFactory(clientTLSFactory);
+        QueuedThreadPool clientThreads = new QueuedThreadPool();
+        clientThreads.setName("client");
+        clientConnector.setExecutor(clientThreads);
+        client = new HttpClient(new HttpClientTransportOverHTTP(clientConnector))
+        {
+            @Override
+            protected ClientConnectionFactory newSslClientConnectionFactory(SslContextFactory.Client sslContextFactory, ClientConnectionFactory connectionFactory)
+            {
+                if (sslContextFactory == null)
+                    sslContextFactory = getSslContextFactory();
+                return new SslClientConnectionFactory(sslContextFactory, getByteBufferPool(), getExecutor(), connectionFactory)
+                {
+                    @Override
+                    protected SslConnection newSslConnection(ByteBufferPool byteBufferPool, Executor executor, EndPoint endPoint, SSLEngine engine)
+                    {
+                        return new SslConnection(byteBufferPool, executor, endPoint, engine, isDirectBuffersForEncryption(), isDirectBuffersForDecryption())
+                        {
+                            @Override
+                            protected SSLEngineResult wrap(SSLEngine sslEngine, ByteBuffer[] input, ByteBuffer output) throws SSLException
+                            {
+                                sslEngine.closeOutbound();
+                                return super.wrap(sslEngine, input, output);
+                            }
+                        };
+                    }
+                };
+            }
+        };
+        client.setExecutor(clientThreads);
+        client.start();
+
+        ExecutionException failure = assertThrows(ExecutionException.class, () -> client.newRequest("localhost", connector.getLocalPort())
+            .scheme(HttpScheme.HTTPS.asString())
+            .send());
+        Throwable cause = failure.getCause();
+        assertThat(cause, Matchers.instanceOf(SSLHandshakeException.class));
+    }
+
+    @Test
+    public void testTLSLargeFragments() throws Exception
+    {
+        CountDownLatch serverLatch = new CountDownLatch(1);
+        SslContextFactory.Server serverTLSFactory = createServerSslContextFactory();
+        QueuedThreadPool serverThreads = new QueuedThreadPool();
+        serverThreads.setName("server");
+        server = new Server(serverThreads);
+        HttpConfiguration httpConfig = new HttpConfiguration();
+        httpConfig.addCustomizer(new SecureRequestCustomizer());
+        HttpConnectionFactory http = new HttpConnectionFactory(httpConfig);
+        SslConnectionFactory ssl = new SslConnectionFactory(serverTLSFactory, http.getProtocol())
+        {
+            @Override
+            protected SslConnection newSslConnection(Connector connector, EndPoint endPoint, SSLEngine engine)
+            {
+                return new SslConnection(connector.getByteBufferPool(), connector.getExecutor(), endPoint, engine, isDirectBuffersForEncryption(), isDirectBuffersForDecryption())
+                {
+                    @Override
+                    protected SSLEngineResult unwrap(SSLEngine sslEngine, ByteBuffer input, ByteBuffer output) throws SSLException
+                    {
+                        int inputBytes = input.remaining();
+                        SSLEngineResult result = super.unwrap(sslEngine, input, output);
+                        if (inputBytes == 5)
+                            serverLatch.countDown();
+                        return result;
+                    }
+                };
+            }
+        };
+        connector = new ServerConnector(server, 1, 1, ssl, http);
+        server.addConnector(connector);
+        server.setHandler(new EmptyServerHandler());
+        server.start();
+
+        long idleTimeout = 2000;
+
+        CountDownLatch clientLatch = new CountDownLatch(1);
+        SslContextFactory.Client clientTLSFactory = createClientSslContextFactory();
+        ClientConnector clientConnector = new ClientConnector();
+        clientConnector.setSelectors(1);
+        clientConnector.setSslContextFactory(clientTLSFactory);
+        QueuedThreadPool clientThreads = new QueuedThreadPool();
+        clientThreads.setName("client");
+        clientConnector.setExecutor(clientThreads);
+        client = new HttpClient(new HttpClientTransportOverHTTP(clientConnector))
+        {
+            @Override
+            protected ClientConnectionFactory newSslClientConnectionFactory(SslContextFactory.Client sslContextFactory, ClientConnectionFactory connectionFactory)
+            {
+                if (sslContextFactory == null)
+                    sslContextFactory = getSslContextFactory();
+                return new SslClientConnectionFactory(sslContextFactory, getByteBufferPool(), getExecutor(), connectionFactory)
+                {
+                    @Override
+                    protected SslConnection newSslConnection(ByteBufferPool byteBufferPool, Executor executor, EndPoint endPoint, SSLEngine engine)
+                    {
+                        return new SslConnection(byteBufferPool, executor, endPoint, engine, isDirectBuffersForEncryption(), isDirectBuffersForDecryption())
+                        {
+                            @Override
+                            protected SSLEngineResult wrap(SSLEngine sslEngine, ByteBuffer[] input, ByteBuffer output) throws SSLException
+                            {
+                                try
+                                {
+                                    clientLatch.countDown();
+                                    assertTrue(serverLatch.await(5, TimeUnit.SECONDS));
+                                    return super.wrap(sslEngine, input, output);
+                                }
+                                catch (InterruptedException x)
+                                {
+                                    throw new SSLException(x);
+                                }
+                            }
+                        };
+                    }
+                };
+            }
+        };
+        client.setIdleTimeout(idleTimeout);
+        client.setExecutor(clientThreads);
+        client.start();
+
+        String host = "localhost";
+        int port = connector.getLocalPort();
+
+        CountDownLatch responseLatch = new CountDownLatch(1);
+        client.newRequest(host, port)
+            .scheme(HttpScheme.HTTPS.asString())
+            .send(result ->
+            {
+                assertTrue(result.isSucceeded());
+                assertEquals(HttpStatus.OK_200, result.getResponse().getStatus());
+                responseLatch.countDown();
+            });
+        // Wait for the TLS buffers to be acquired by the client, then the
+        // HTTP request will be paused waiting for the TLS buffer to be expanded.
+        assertTrue(clientLatch.await(5, TimeUnit.SECONDS));
+
+        // Send the large frame bytes that will enlarge the TLS buffers.
+        try (Socket socket = new Socket(host, port))
+        {
+            OutputStream output = socket.getOutputStream();
+            byte[] largeFrameBytes = new byte[5];
+            largeFrameBytes[0] = 22; // Type = handshake
+            largeFrameBytes[1] = 3; // Major TLS version
+            largeFrameBytes[2] = 3; // Minor TLS version
+            // Frame length is 0x7FFF == 32767, i.e. a "large fragment".
+            // Maximum allowed by RFC 8446 is 16384, but SSLEngine supports up to 33093.
+            largeFrameBytes[3] = 0x7F; // Length hi byte
+            largeFrameBytes[4] = (byte)0xFF; // Length lo byte
+            output.write(largeFrameBytes);
+            output.flush();
+            // Just close the connection now, the large frame
+            // length was enough to trigger the buffer expansion.
+        }
+
+        // The HTTP request will resume and be forced to handle the TLS buffer expansion.
+        assertTrue(responseLatch.await(5, TimeUnit.SECONDS));
+    }
 }

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java

@@ -50,6 +50,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.LongConsumer;
 import javax.servlet.ServletException;
 import javax.servlet.ServletOutputStream;
 import javax.servlet.http.HttpServletRequest;
@@ -533,10 +534,10 @@ public class HttpClientTest extends AbstractHttpClientServerTest
     @ArgumentsSource(ScenarioProvider.class)
     public void test_ExchangeIsComplete_OnlyWhenBothRequestAndResponseAreComplete(Scenario scenario) throws Exception
     {
-        start(scenario, new AbstractHandler.ErrorDispatchHandler()
+        start(scenario, new AbstractHandler()
         {
             @Override
-            protected void doNonErrorHandle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException
+            public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException
             {
                 baseRequest.setHandled(true);
                 response.setContentLength(0);
@@ -1117,6 +1118,13 @@ public class HttpClientTest extends AbstractHttpClientServerTest
                 counter.incrementAndGet();
             }
 
+            @Override
+            public void onContent(Response response, LongConsumer demand, ByteBuffer content, Callback callback)
+            {
+                // Should not be invoked
+                counter.incrementAndGet();
+            }
+
             @Override
             public void onSuccess(Response response)
             {

jetty-client/src/test/java/org/eclipse/jetty/client/jmx/HttpClientJMXTest.java

@@ -39,7 +39,6 @@ public class HttpClientJMXTest
         String name = "foo";
         HttpClient httpClient = new HttpClient();
         httpClient.setName(name);
-        httpClient.start();
 
         try
         {
@@ -47,6 +46,7 @@ public class HttpClientJMXTest
             MBeanContainer mbeanContainer = new MBeanContainer(mbeanServer);
             // Adding MBeanContainer as a bean will trigger the registration of MBeans.
             httpClient.addBean(mbeanContainer);
+            httpClient.start();
 
             String domain = HttpClient.class.getPackage().getName();
             ObjectName pattern = new ObjectName(domain + ":type=" + HttpClient.class.getSimpleName().toLowerCase(Locale.ENGLISH) + ",*");

jetty-deploy/src/main/java/org/eclipse/jetty/deploy/providers/ScanningAppProvider.java

@@ -143,6 +143,7 @@ public abstract class ScanningAppProvider extends ContainerLifeCycle implements
         _scanner.setRecursive(_recursive);
         _scanner.setFilenameFilter(_filenameFilter);
         _scanner.setReportDirs(true);
+        _scanner.setScanDepth(1); //consider direct dir children of monitored dir
         _scanner.addListener(_scannerListener);
 
         addBean(_scanner);

jetty-deploy/src/main/java/org/eclipse/jetty/deploy/providers/WebAppProvider.java

@@ -81,6 +81,11 @@ public class WebAppProvider extends ScanningAppProvider
             String lowername = name.toLowerCase(Locale.ENGLISH);
 
             File file = new File(dir, name);
+            Resource r = Resource.newResource(file);
+            if (getMonitoredResources().contains(r) && r.isDirectory())
+            {
+                return false;
+            }
 
             // ignore hidden files
             if (lowername.startsWith("."))

jetty-deploy/src/test/resources/jetty.xml

@@ -84,7 +84,7 @@
       <Set name="responseHeaderSize">8192</Set>
       <Set name="sendServerVersion">true</Set>
       <Set name="sendDateHeader">false</Set>
-      <Set name="headerCacheSize">4096</Set>
+      <Set name="headerCacheSize">1024</Set>
 
       <!-- Uncomment to enable handling of X-Forwarded- style headers
       <Call name="addCustomizer">

jetty-distribution/pom.xml

@@ -121,7 +121,7 @@
                   <destFileName>test-spec.war</destFileName>
                 </artifactItem>
                 <artifactItem>
-                  <groupId>org.eclipse.jetty</groupId>
+                  <groupId>org.eclipse.jetty.tests</groupId>
                   <artifactId>test-proxy-webapp</artifactId>
                   <version>${project.version}</version>
                   <type>war</type>
@@ -416,7 +416,7 @@
       <optional>true</optional>
     </dependency>
     <dependency>
-      <groupId>org.eclipse.jetty</groupId>
+      <groupId>org.eclipse.jetty.tests</groupId>
       <artifactId>test-proxy-webapp</artifactId>
       <type>war</type>
       <version>${project.version}</version>

jetty-documentation/pom.xml

@@ -100,8 +100,8 @@
             <GITDOCURL>https://github.com/eclipse/jetty.project/tree/jetty-10.0.x-doc-refactor/jetty-documentation/src/main/asciidoc</GITDOCURL>
             <DISTGUIDE>../distribution-guide/index.html</DISTGUIDE>
             <EMBEDGUIDE>../embedded-guide/index.html</EMBEDGUIDE>
-            <QUICKGUIDE>../quickstart-guideindex.html</QUICKGUIDE>
-            <CONTRIBGUIDE>../contribution-guideindex.html</CONTRIBGUIDE>
+            <QUICKGUIDE>../quickstart-guide/index.html</QUICKGUIDE>
+            <CONTRIBGUIDE>../contribution-guide/index.html</CONTRIBGUIDE>
             <MVNCENTRAL>http://central.maven.org/maven2</MVNCENTRAL>
             <VERSION>${project.version}</VERSION>
             <TIMESTAMP>${maven.build.timestamp}</TIMESTAMP>

jetty-documentation/src/main/asciidoc/distribution-guide/annotations/using-annotations.adoc

@@ -139,7 +139,7 @@ Here is an example, setting the context attribute in code (although you can also
 ----
 WebAppContext context = new WebAppContext();
 context.setAttribute("org.eclipse.jetty.containerInitializerOrder",
-                     "org.eclipse.jetty.websocket.jsr356.server.deploy.WebSocketServerContainerInitializer, com.acme.Foo.MySCI, *");
+                     "org.eclipse.jetty.websocket.javax.server.JavaxWebSocketServletContainerInitializer, com.acme.Foo.MySCI, *");
 ----
 
 In this example, we ensure that the `WebSocketServerContainerInitializer` is the very first `ServletContainerInitializer` that is called, followed by MySCI and then any other `ServletContainerInitializer` instances that were discovered but not yet called.

jetty-documentation/src/main/asciidoc/distribution-guide/extras/dos-filter.adoc

@@ -33,7 +33,6 @@ The Denial of Service (DoS) filter limits exposure to request flooding, whether
 The DoS filter keeps track of the number of requests from a connection per second.
 If the requests exceed the limit, Jetty rejects, delays, or throttles the request, and sends a warning message.
 The filter works on the assumption that the attacker might be written in simple blocking style, so by suspending requests you are hopefully consuming the attacker's resources.
-The DoS filter is related to the QoS filter, using Continuations to prioritize requests and avoid thread starvation.
 
 [[dos-filter-using]]
 ==== Using the DoS Filter

jetty-documentation/src/main/asciidoc/distribution-guide/extras/gzip-filter.adoc

@@ -31,7 +31,6 @@
 
 The Jetty `GzipHandler` is a compression handler that you can apply to any dynamic resource (servlet).
 It fixes many of the bugs in commonly available compression filters: it works with asynchronous servlets; it handles all ways to set content length.
-It has been tested with Jetty continuations and suspending requests.
 Some user-agents might be excluded from compression to avoid common browser bugs (yes, this means IE!).
 
 The `GzipHandler` can be added to the entire server by enabling the `gzip.mod` module.

jetty-documentation/src/main/asciidoc/distribution-guide/http2/introduction.adoc

@@ -26,11 +26,21 @@ The requirements for running HTTP/2 are JDK 8 or greater, and typically also ALP
 A server deployed over TLS (SSL) normally advertises the HTTP/2 protocol via the TLS extension Application Layer Protocol Negotiation link:#alpn[(ALPN)].
 
 ____
-[IMPORTANT]
+[NOTE]
 To use HTTP/2 in Jetty via a TLS connector you need to add the link:#alpn-starting[ALPN boot jar] in the boot classpath.
 This is done automatically when using the Jetty distribution's start.jar link:#startup-modules[module system], but must be configured directly otherwise.
 ____
 
+[[http2-security-update]]
+==== Jetty HTTP/2 Security Update
+
+In mid-2019, there were a link:#security-reports[number of CVEs] were issued warning against vulnerable HTTP/2 implementations. These CVEs (CVE-2019-9511 thru CVE-2019-9518) generally centered around attackers manipulating and flooding HTTP/2 servers and creating a denial of service (DOS). These vulnerabilities were patched with Jetty 9.4.21.
+
+As a result of these CVEs, Jetty introduced a new, configurable denial of service (DOS) protection feature in Jetty 9.4.22.
+
+Jetty’s HTTP/2 implementation now features a new Rate Control parameter, `jetty.http2.rateControl.maxEventsPerSecond`, that defaults to 20 events per second, per connection for all pings, bad frames, settings frames, priority changes etc.
+
+
 [[http2-modules]]
 ==== Jetty HTTP/2 Sub Projects
 

jetty-documentation/src/main/asciidoc/distribution-guide/security/serving-aliased-files.adoc

@@ -19,8 +19,8 @@
 [[serving-aliased-files]]
 === Aliased Files and Symbolic links
 
-Web applications will often server static content from the file system provided by the operating system running underneath the JVM.
-However because file systems often implement multiple aliased names for the same file, then security constraints and other servlet URI space mappings my inadvertently be bypassed by aliases.
+Web applications will often serve static content from the file system provided by the operating system running underneath the JVM.
+However, because file systems often implement multiple aliased names for the same file, then security constraints and other servlet URI space mappings may inadvertently be bypassed by aliases.
 
 A key example of this is case insensitivity and 8.3 filenames implemented by the Windows file system.
 If a file within a web application called `/mysecretfile.txt` is protected by a security constraint on the URI `/mysecretfile.txt`, then a request to `/MySecretFile.TXT` will not match the URI constraint because URIs are case sensitive, but the Windows file system will report that a file does exist at that name and it will be served despite the security constraint.

jetty-documentation/src/main/asciidoc/distribution-guide/startup/start-jar.adoc

@@ -262,3 +262,16 @@ You might need to escape the slash "\|" to use this on some environments.
 maven.repo.uri=[url]::
 The url to use to download Maven dependencies.
 Default is https://repo1.maven.org/maven2/.
+
+==== Shaded Start.jar
+
+If you have a need for a shaded version of `start.jar` (such as for Gradle), you can achieve this via a Maven dependency.
+[source, xml, subs="{sub-order}"]
+....
+<dependency>
+    <groupId>org.eclipse.jetty</groupId>
+    <artifactId>jetty-start</artifactId>
+    <version>{VERSION}</version>
+    <classifier>shaded</classifier>
+</dependency>
+....

jetty-documentation/src/main/asciidoc/distribution-guide/startup/startup-base-vs-home.adoc

@@ -145,7 +145,7 @@ Properties:
  jetty.secure.port = 8443
  jetty.truststore = etc/keystore
  jetty.truststore.password = OBF:1vny1zlo1x8e1vnw1vn61x8g1zlu1vn4
- org.eclipse.jetty.websocket.jsr356 = false
+ org.eclipse.jetty.websocket.javax = false
  threads.max = 200
  threads.min = 10
  threads.timeout = 60000
@@ -235,7 +235,7 @@ etc/demo-rewrite-rules.xml
 
 # Websocket chat examples needs websocket enabled
 # Don't start for all contexts (set to true in test.xml context)
-org.eclipse.jetty.websocket.jsr356=false
+org.eclipse.jetty.websocket.javax=false
 --module=websocket
 
 # Create and configure the test realm

jetty-documentation/src/main/asciidoc/embedded-guide/server/continuations/continuations-intro.adoc

@@ -1,129 +0,0 @@
-//
-//  ========================================================================
-//  Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd.
-//  ========================================================================
-//  All rights reserved. This program and the accompanying materials
-//  are made available under the terms of the Eclipse Public License v1.0
-//  and Apache License v2.0 which accompanies this distribution.
-//
-//      The Eclipse Public License is available at
-//      http://www.eclipse.org/legal/epl-v10.html
-//
-//      The Apache License v2.0 is available at
-//      http://www.opensource.org/licenses/apache2.0.php
-//
-//  You may elect to redistribute this code under either of these licenses.
-//  ========================================================================
-//
-
-[[continuations-intro]]
-=== Introduction
-
-Continuations are a mechanism to implement Asynchronous servlets similar to asynchronous features in Servlet 3.0, but provides a simpler and portable interface.
-
-==== Why Asynchronous Servlets ?
-
-===== Not Asynchronous IO
-
-The concept of Asynchronous Servlets is often confused with Asynchronous IO or the use of NIO.
-However, Asynchronous Servlets are not primarily motivated by asynchronous IO, since:
-
-* HTTP Requests are mostly small and arrive in a single packet. Servlets rarely block on requests.
-
-* Many responses are small and fit within the server buffers, so servlets often do not block writing responses.
-
-* Even if we could expose asynchronous IO in a servlet, it is a hard paradigm to program. For example what would an application do if it read 2 bytes of a 3 byte UTF-8 character?
-It would have to buffer and wait for more bytes.
-This is best done by the container rather than the application.
-
-===== Asynchronous Waiting
-
-The main use-case for asynchronous servlets is waiting for non-IO events or resources.
-Many web applications need to wait at some stage during the processing of a HTTP request, for example:
-
-* Waiting for a resource to be available before processing the request (e.g., thread, JDBC Connection).
-
-* Waiting for an application event in an AJAX Comet application (e.g., chat message, price change).
-
-* Waiting for a response from a remote service (e.g., RESTful or SOAP call to a web service).
-
-The servlet API (pre 2.5) supports only a synchronous call style, so that any waiting that a servlet needs to do must be with blocking.
-Unfortunately this means that the thread allocated to the request must be held during that wait along with all its resources: kernel thread, stack memory and often pooled buffers, character converters, EE authentication context, etc.
-It is wasteful of system resources to hold these resources while waiting. Significantly better scalability and quality of service can be achieved if waiting is done asynchronously.
-
-==== Asynchronous Servlet Examples
-
-===== AJAX Comet Server Push
-
-Web 2.0 applications can use the http://en.wikipedia.org/wiki/Comet_(programming)[comet] technique (aka AJAX Push, Server Push, Long Polling) to dynamically update a web page without refreshing the entire page.
-
-Consider a stock portfolio web application. Each browser will send a long poll request to the server asking for any of the user's stock prices that have changed. The server will receive the long poll requests from all its clients, but will not immediately respond.
-Instead the server waits until a stock price changes, at which time it will send a response to each of the clients with that stock in their portfolio.
-The clients that receive the long poll response will immediately send another long poll request so they may obtain future price changes.
-
-Thus the server will typically hold a long poll request for every connected user, so if the servlet is not asynchronous, there would need more than 1000 threads available to handle 1000 simultaneous users.
-1000 threads can consume over 256MB of memory; that would be better used for the application rather than idly waiting for a price to change.
-
-If the servlet is asynchronous, then the number of threads needed is governed by the time to generate each response and the frequency of price changes.
-If every user receives a price every 10 seconds and the response takes 10ms to generate, then 1000 users can be serviced with just 1 thread, and the 256MB of stack be freed for other purposes.
-
-For more on comet see the http://cometd.org/[cometd] project that works asynchronously with Jetty.
-
-===== Asynchronous RESTful Web Service
-
-Consider a web application that accesses a remote web service (e.g., SOAP service or RESTful service).
-Typically a remote web service can take hundreds of milliseconds to produce a response -- eBay's RESTful web service frequently takes 350ms to respond with a list of auctions matching a given keyword -- while only a few 10s of milliseconds of CPU time are needed to locally process a request and generate a response.
-
-To handle 1000 requests per second, which each perform a 200ms web service call, a webapp would needs 1000*(200+20)/1000 = 220 threads and 110MB of stack memory.
-It would also be vulnerable to thread starvation if bursts occurred or the web service became slower. If handled asynchronously, the web application would not need to hold a thread while waiting for web service response.
-Even if the asynchronous mechanism cost 10ms (which it doesn't), then this webapp would need 1000*(20+10)/1000 = 30 threads and 15MB of stack memory.
-This is a 86% reduction in the resources required and 95MB more memory would be available for the application.
-Furthermore, if multiple web services request are required, the asynchronous approach allows these to be made in parallel rather than serially, without allocating additional threads.
-
-For an example of Jetty's solution, see the https://webtide.com/async-rest-jetty-9/[Asynchronous REST example]
-
-===== Quality of Service (e.g., JDBC Connection Pool)
-
-Consider a web application handling on average 400 requests per second, with each request interacting with the database for 50ms.
-To handle this load, 400*50/1000 = 20 JDBC connections are need on average.
-However, requests do not come at an even rate and there are often bursts and pauses.
-To protect a database from bursts, often a JDBC connection pool is applied to limit the simultaneous requests made on the database.
-So for this application, it would be reasonable to apply a JDBC pool of 30 connections, to provide for a 50% margin.
-
-If momentarily the request rate doubled, then the 30 connections would only be able to handle 600 requests per second, and 200 requests per second would join those waiting on the JDBC Connection pool.
-Then if the servlet container had a thread pool with 200 threads, that would be entirely consumed by threads waiting for JDBC connections in 1 second of this request rate.
-After 1s, the web application would be unable to process any requests at all because no threads would be available.
-Even requests that do not use the database would be blocked due to thread starvation.
-To double the thread pool would require an additional 100MB of stack memory and would only give the application another 1s of grace under load!
-
-This thread starvation situation can also occur if the database runs slowly or is momentarily unavailable.
-Thread starvation is a very frequently reported problem, and causes the entire web service to lock up and become unresponsive.
-If the web container was able to suspend the requests waiting for a JDBC connection without threads, then thread starvation would not occur, as only 30 threads would be consumed by requests accessing the database and the other 470 threads would be available to process the request that do not access the database.
-
-For an example of Jetty's solution, see the Quality of Service Filter.
-
-==== Servlet Threading Model
-
-The scalability issues of Java servlets are caused mainly by the server threading model:
-
-===== Thread per connection
-
-The traditional IO model of Java associated a thread with every TCP/IP connection.
-If you have a few very active threads, this model can scale to a very high number of requests per second.
-
-However, the traffic profile typical of many web applications is many persistent HTTP connections that are mostly idle while users read pages or search for the next link to click. With such profiles, the thread-per-connection model can have problems scaling to the thousands of threads required to support thousands of users on large scale deployments.
-
-===== Thread per request
-
-The Java NIO libraries support asynchronous IO, so that threads no longer need to be allocated to every connection.
-When the connection is idle (between requests), then the connection is added to an NIO select set, which allows one thread to scan many connections for activity.
-Only when IO is detected on a connection is a thread allocated to it.
-However, the servlet 2.5 API model still requires a thread to be allocated for the duration of the request handling.
-
-This thread-per-request model allows much greater scaling of connections (users) at the expense of a small reduction to maximum requests per second due to extra scheduling latency.
-
-===== Asynchronous Request handling
-
-The Jetty Continuation (and the servlet 3.0 asynchronous) API introduce a change in the servlet API that allows a request to be dispatched multiple times to a servlet.
-If the servlet does not have the resources required on a dispatch, then the request is suspended (or put into asynchronous mode), so that the servlet may return from the dispatch without a response being sent.
-When the waited-for resources become available, the request is re-dispatched to the servlet, with a new thread, and a response is generated.

jetty-documentation/src/main/asciidoc/embedded-guide/server/continuations/continuations-patterns.adoc

@@ -1,126 +0,0 @@
-//
-//  ========================================================================
-//  Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd.
-//  ========================================================================
-//  All rights reserved. This program and the accompanying materials
-//  are made available under the terms of the Eclipse Public License v1.0
-//  and Apache License v2.0 which accompanies this distribution.
-//
-//      The Eclipse Public License is available at
-//      http://www.eclipse.org/legal/epl-v10.html
-//
-//      The Apache License v2.0 is available at
-//      http://www.opensource.org/licenses/apache2.0.php
-//
-//  You may elect to redistribute this code under either of these licenses.
-//  ========================================================================
-//
-
-[[continuations-patterns]]
-=== Common Continuation Patterns
-
-==== Suspend Resume Pattern
-
-The suspend/resume style is used when a servlet and/or filter is used to generate the response after an asynchronous wait that is terminated by an asynchronous handler.
-Typically a request attribute is used to pass results and to indicate if the request has already been suspended.
-
-[source, java, subs="{sub-order}"]
-----
-void doGet(HttpServletRequest request, HttpServletResponse response)
-{
-     // if we need to get asynchronous results
-     Object results = request.getAttribute("results");
-     if (results==null)
-     {
-       final Continuation continuation = ContinuationSupport.getContinuation(request);
-
-       // if this is not a timeout
-       if (continuation.isExpired())
-       {
-         sendMyTimeoutResponse(response);
-         return;
-       }
-
-       // suspend the request
-       continuation.suspend(); // always suspend before registration
-
-       // register with async service.  The code here will depend on the
-       // the service used (see Jetty HttpClient for example)
-       myAsyncHandler.register(new MyHandler()
-       {
-          public void onMyEvent(Object result)
-          {
-            continuation.setAttribute("results",results);
-            continuation.resume();
-          }
-       });
-       return; // or continuation.undispatch();
-     }
-
-     // Send the results
-     sendMyResultResponse(response,results);
-}
-
-----
-
-This style is very good when the response needs the facilities of the servlet container (e.g., it uses a web framework) or if one event may resume many requests so the container's thread pool can be used to handle each of them.
-
-==== Suspend Continue Pattern
-
-The suspend/complete style is used when an asynchronous handler is used to generate the response:
-
-[source, java, subs="{sub-order}"]
-----
-void doGet(HttpServletRequest request, HttpServletResponse response)
-{
-     final Continuation continuation = ContinuationSupport.getContinuation(request);
-
-     // if this is not a timeout
-     if (continuation.isExpired())
-     {
-       sendMyTimeoutResponse(request,response);
-       return;
-     }
-
-     // suspend the request
-     continuation.suspend(); // response may be wrapped.
-
-     // register with async service.  The code here will depend on the
-     // the service used (see Jetty HttpClient for example)
-     myAsyncHandler.register(new MyHandler()
-     {
-       public void onMyEvent(Object result)
-       {
-         sendMyResultResponse(continuation.getServletResponse(),results);
-         continuation.complete();
-       }
-     });
-}
-
-----
-
-This style is very good when the response does not need the facilities of the servlet container (e.g., it does not use a web framework) and if an event will resume only one continuation.
-If many responses are to be sent (e.g., a chat room), then writing one response may block and cause a DOS on the other responses.
-
-==== Examples
-
-* The https://github.com/eclipse/jetty.project/blob/jetty-8/test-jetty-webapp/src/main/java/com/acme/ChatServlet.java[ChatServlet example] shows how the suspend/resume style can be used to directly code a chat room (See similar https://github.com/eclipse/jetty.project/blob/master/tests/test-webapps/test-jetty-webapp/src/main/java/com/acme/ChatServlet.java[example] using Async Servlets).
-The same principles are applied to frameworks like http://cometd.org/[cometd] which provide an richer environment for such applications, based on Continuations.
-
-* The link:{JDURL}/org/eclipse/jetty/servlets/QoSFilter.html[QoSFilter] uses suspend/resume style to limit the number of requests simultaneously within the filter.
-This can be used to protect a JDBC connection pool or other limited resource from too many simultaneous requests.
-
-+
-If too many requests are received, the extra requests wait for a short time on a semaphore, before being suspended.
-As requests within the filter return, they use a priority queue to resume the suspended requests.
-This allows your authenticated or priority users to get a better share of your server's resources when the machine is under load.
-+
-
-* The link:{JDURL}/org/eclipse/jetty/servlets/DoSFilter.html[DosFilter] is similar to the QoSFilter, but protects a web application from a denial of service attack, as much as is possible from within a web application.
-
-+
-If too many requests are detected coming from one source, then those requests are suspended and a warning generated.
-This works on the assumption that the attacker may be written in simple blocking style, so by suspending you are hopefully consuming their resources. True protection from DOS can only be achieved by network devices (or eugenics :)).
-+
-
-* The link:{JDURL}/org/eclipse/jetty/proxy/ProxyServlet.html[ProxyServlet] uses the suspend/complete style and the Jetty asynchronous HTTP client to implement a scalable Proxy server (or transparent proxy).

jetty-documentation/src/main/asciidoc/embedded-guide/server/continuations/continuations-using.adoc

@@ -1,116 +0,0 @@
-//
-//  ========================================================================
-//  Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd.
-//  ========================================================================
-//  All rights reserved. This program and the accompanying materials
-//  are made available under the terms of the Eclipse Public License v1.0
-//  and Apache License v2.0 which accompanies this distribution.
-//
-//      The Eclipse Public License is available at
-//      http://www.eclipse.org/legal/epl-v10.html
-//
-//      The Apache License v2.0 is available at
-//      http://www.opensource.org/licenses/apache2.0.php
-//
-//  You may elect to redistribute this code under either of these licenses.
-//  ========================================================================
-//
-
-[[continuations-using]]
-=== Using Continuations
-
-Asynchronous servlets were originally introduced with Jetty 6 Continuations, which were a Jetty specific mechanism.
-From Jetty 7 onwards, the Continuations API has been extended to be a general purpose API that will work asynchronously on any servlet-3.0 container, as well as on Jetty 6, 7, or 8.
-Continuations will also work in blocking mode with any servlet 2.5 container.
-
-==== Obtaining a Continuation
-
-The link:{JDURL}/org/eclipse/jetty/continuation/ContinuationSupport.html[ContinuationSupport] factory class can be used to obtain a continuation instance associated with a request:
-
-`Continuation continuation = ContinuationSupport.getContinuation(request);`
-
-==== Suspending a Request
-
-To suspend a request, the suspend method can be called on the continuation:
-
-[source, java, subs="{sub-order}"]
-----
-    void doGet(HttpServletRequest request, HttpServletResponse response)
-      {
-      ...
-      // optionally:
-      // continuation.setTimeout(long);
-      continuation.suspend();
-      ...
-      }
-
-----
-
-The lifecycle of the request will be extended beyond the return to the  container from the `Servlet.service(...)` method and `Filter.doFilter(...)` calls. When these dispatch methods return, the suspended request will not yet be committed and a response will not yet be sent to the HTTP client.
-
-Once the request has been suspended, the continuation should be registered with an asynchronous service so that it may be used by an asynchronous callback when the waited-for event happens.
-
-The request will be suspended until either `continuation.resume()` or `continuation.complete()` is called. If neither is called then the continuation will timeout.
-The timeout should be set before the suspend, by a call to `continuation.setTimeout(long)` if no timeout is set, then the default period is used.
-If no timeout listeners resume or complete the continuation, then the continuation is resumed with `continuation.isExpired()` true.
-
-Suspension is analogous to the servlet 3.0 `request.startAsync()` method. Unlike jetty 6 continuations, an exception is not thrown by suspend and the method should return normally.
-This allows the registration of the continuation to occur after suspension and avoids the need for a mutex.
-If an exception is desirable (to bypass code that is unaware of continuations and may try to commit the response), then `continuation.undispatch()` may be called to exit the current thread from the current dispatch by throwing a `ContinuationThrowable`.
-
-==== Resuming a Request
-
-Once an asynchronous event has occurred, the continuation can be resumed:
-
-[source, java, subs="{sub-order}"]
-----
-    void myAsyncCallback(Object results)
-    {
-    continuation.setAttribute("results",results);
-    continuation.resume();
-    }
-----
-
-When a continuation is resumed, the request is re-dispatched to the servlet container, almost as if the request had been received again.
-However during the re-dispatch, the `continuation.isInitial()` method returns false and any attributes set by the asynchronous handler are available.
-
-Continuation resume is analogous to Servlet 3.0 `AsyncContext.dispatch()`.
-
-==== Completing a Request
-
-As an alternative to resuming a request, an asynchronous handler may  write the response itself. After writing the response, the handler must indicate the request handling is complete by calling the complete method:
-
-[source, java, subs="{sub-order}"]
-----
-    void myAsyncCallback(Object results)
-    {
-      writeResults(continuation.getServletResponse(),results);
-      continuation.complete();
-    }
-----
-
-After complete is called, the container schedules the response to be  committed and flushed. Continuation complete is analogous to Servlet 3.0 `AsyncContext.complete()`.
-
-==== Continuation Listeners
-
-An application may monitor the status of a continuation by using a ContinuationListener:
-
-[source, java, subs="{sub-order}"]
-----
-    void doGet(HttpServletRequest request, HttpServletResponse response)
-    {
-      ...
-
-      Continuation continuation = ContinuationSupport.getContinuation(request);
-      continuation.addContinuationListener(new ContinuationListener()
-      {
-        public void onTimeout(Continuation continuation) { ... }
-        public void onComplete(Continuation continuation) { ... }
-      });
-
-      continuation.suspend();
-      ...
-    }
-----
-
-Continuation listeners are analogous to Servlet 3.0 AsyncListeners.

jetty-documentation/src/main/asciidoc/embedded-guide/server/maven/jetty-maven-plugin.adoc

@@ -213,7 +213,7 @@ Below is the relevant section taken from link:{GITBROWSEURL}/jetty-server/src/ma
       <Set name="responseHeaderSize"><Property name="jetty.httpConfig.responseHeaderSize" deprecated="jetty.response.header.size" default="8192" /></Set>
       <Set name="sendServerVersion"><Property name="jetty.httpConfig.sendServerVersion" deprecated="jetty.send.server.version" default="true" /></Set>
       <Set name="sendDateHeader"><Property name="jetty.httpConfig.sendDateHeader" deprecated="jetty.send.date.header" default="false" /></Set>
-      <Set name="headerCacheSize"><Property name="jetty.httpConfig.headerCacheSize" default="4096" /></Set>
+      <Set name="headerCacheSize"><Property name="jetty.httpConfig.headerCacheSize" default="1024" /></Set>
       <Set name="delayDispatchUntilContent"><Property name="jetty.httpConfig.delayDispatchUntilContent" deprecated="jetty.delayDispatchUntilContent" default="true"/></Set>
       <Set name="maxErrorDispatches"><Property name="jetty.httpConfig.maxErrorDispatches" default="10"/></Set>
       <Set name="persistentConnectionsEnabled"><Property name="jetty.httpConfig.persistentConnectionsEnabled" default="true"/></Set>

jetty-documentation/src/main/asciidoc/embedded-guide/server/server.adoc

@@ -26,7 +26,6 @@ include::handlers/chapter.adoc[]
 include::websockets/intro/chapter.adoc[]
 include::websockets/jetty/chapter.adoc[]
 include::ant/chapter.adoc[]
-include::continuations/chapter.adoc[]
 include::frameworks/chapter.adoc[]
 include::architecture/chapter.adoc[]
 include::platforms/chapter.adoc[]

jetty-documentation/src/main/asciidoc/embedded-guide/server/troubleshooting/security-reports.adoc

@@ -28,9 +28,24 @@ If you would like to report a security issue please follow these link:#security-
 |=======================================================================
 |yyyy/mm/dd |ID |Exploitable |Severity |Affects |Fixed Version |Comment
 
+|2019/08/13 |CVE-2019-9518 |Med |Med |< = 9.4.20 |9.4.21
+|https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-9518[Some HTTP/2 implementations are vulnerable to a flood of empty frames, potentially leading to a denial of service.]
+
+|2019/08/13 |CVE-2019-9516 |Med |Med |< = 9.4.20 |9.4.21
+|https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-9516[Some HTTP/2 implementations are vulnerable to a header leak, potentially leading to a denial of service.]
+
 |2019/08/13 |CVE-2019-9515 |Med |Med |< = 9.4.20 |9.4.21
 |https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-9515[Some HTTP/2 implementations are vulnerable to a settings flood, potentially leading to a denial of service when an attacker sent a stream of SETTINGS frames to the peer.]
 
+|2019/08/13 |CVE-2019-9514 |Med |Med |< = 9.4.20 |9.4.21
+|https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-9514[Some HTTP/2 implementations are vulnerable to a reset flood, potentially leading to a denial of service.]
+
+|2019/08/13 |CVE-2019-9512 |Low |Low |< = 9.4.20 |9.4.21
+|https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-9512[Some HTTP/2 implementations are vulnerable to ping floods which could lead to a denial of service.]
+
+|2019/08/13 |CVE-2019-9511 |Low |Low |< = 9.4.20 |9.4.21
+|https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-9511[Some HTTP/2 implementations are vulnerable to window size manipulation and stream prioritization manipulation which could lead to a denial of service.]
+
 |2019/04/11 |CVE-2019-10247 |Med |Med |< = 9.4.16 |9.2.28, 9.3.27, 9.4.17
 |https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-10247[If no webapp was mounted to the root namespace and a 404 was encountered, an HTML page would be generated displaying the fully qualified base resource location for each context.]
 

jetty-documentation/src/main/asciidoc/embedded-guide/server/websockets/intro/chapter.adoc

@@ -107,18 +107,18 @@ To enable Websocket, you need to enable the `websocket` link:#enabling-modules[m
 
 Once this module is enabled for your Jetty base, it will apply to all webapps deployed to that base. If you want to be more selective about which webapps use Websocket, then you can:
 
-Disable JSR-356 for a particular webapp:::
-  You can disable jsr-356 for a particular webapp by setting the link:#context_attributes[context attribute] `org.eclipse.jetty.websocket.jsr356` to `false`.
+Disable Websocket for a particular webapp:::
+  You can disable jsr-356 for a particular webapp by setting the link:#context_attributes[context attribute] `org.eclipse.jetty.websocket.javax` to `false`.
   This will mean that websockets are not available to your webapp, however deployment time   scanning for websocket-related classes such as endpoints will still occur.
   This can be a significant impost if your webapp contains a lot of classes and/or jar files.
   To completely disable websockets and avoid all setup costs associated with it for a particular webapp, use instead the context attribute `org.eclipse.jetty.containerInitializerExclusionPattern`, described next, which allows you to exclude the websocket ServletContainerInitializer that causes the scanning.
-Completely disable jsr-356 for a particular webapp:::
-  Set the `org.eclipse.jetty.containerInitializerExclusionPattern` link:#context_attributes[context attribute] to include `org.eclipse.jetty.websocket.jsr356.server.deploy.WebSocketServerContainerInitializer`.
+Completely disable Websocket for a particular webapp:::
+  Set the `org.eclipse.jetty.containerInitializerExclusionPattern` link:#context_attributes[context attribute] to include `org.eclipse.jetty.websocket.javax.server.JavaxWebSocketServletContainerInitializer`.
   Here's an example of doing this in code, although you can do the link:#intro-jetty-configuration-webapps[same in xml]:
 +
 [source, java, subs="{sub-order}"]
 ----
 WebAppContext context = new WebAppContext();
 context.setAttribute("org.eclipse.jetty.containerInitializerExclusionPattern",
-                     "org.eclipse.jetty.websocket.jsr356.server.deploy.WebSocketServerContainerInitializer|com.acme.*");
+                     "org.eclipse.jetty.websocket.javax.server.JavaxWebSocketServletContainerInitializer|com.acme.*");
 ----

jetty-documentation/src/main/asciidoc/embedded-guide/server/websockets/jetty/jetty-websocket-api-send-message.adoc

@@ -115,10 +115,54 @@ catch (IOException e)
 How to send a Text message in 2 parts, using the partial message support in RemoteEndpoint.
 This will block until each part of the message is sent, possibly throwing an IOException if unable to send the partial message.
 
+[[websocket-async-send]]
+==== Async Send Message
+
+There are also four (4) async send message methods available:
+
+* link:{JDURL}/org/eclipse/jetty/websocket/api/RemoteEndpoint.html#sendBytes(java.nio.ByteBuffer,org.eclipse.jetty.websocket.api.WriteCallback)[`RemoteEndpoint.sendBytes(ByteBuffer message, WriteCallback callback)`]
+* link:{JDURL}/org/eclipse/jetty/websocket/api/RemoteEndpoint.html#sendPartialBytes(java.nio.ByteBuffer,boolean,org.eclipse.jetty.websocket.api.WriteCallback)[`RemoteEndpoint.sendPartialBytes(ByteBuffer message, boolean isLast, WriteCallback callback)`]
+* link:{JDURL}/org/eclipse/jetty/websocket/api/RemoteEndpoint.html#sendString(java.lang.String,org.eclipse.jetty.websocket.api.WriteCallback)[`RemoteEndpoint.sendString(String message, WriteCallback callback)`]
+* link:{JDURL}/org/eclipse/jetty/websocket/api/RemoteEndpoint.html#sendPartialString(java.lang.String,boolean,org.eclipse.jetty.websocket.api.WriteCallback)[`RemoteEndpoint.sendPartialString(String message, boolean isLast, WriteCallback callback)`]
+
+All these async send methods use `WriteCallback`, which allows you to be notified when the write either succeeds or fails.
+
+[source, java, subs="{sub-order}"]
+----
+WriteCallback callback = new WriteCallback()
+{
+    @Override
+    public void writeSuccess()
+    {
+        // Notification that the write has succeeded.
+    }
+
+    @Override
+    public void writeFailed(Throwable x)
+    {
+       // Notification that the write has failed.
+       t.printStackTrace();
+    }
+};
+----
+
+The async send methods can be used in a similar way to the blocking send methods, however the method will return before the message is transmitted, and you are notified of the final result of the message transmission through the `WriteCallback`.
+The static `WriteCallback.NOOP` can be used to do nothing on success / failure of the callback.
+
+[source, java, subs="{sub-order}"]
+----
+RemoteEndpoint remote = session.getRemote();
+
+// Async Send of a BINARY message to remote endpoint
+ByteBuffer message = ByteBuffer.wrap(new byte[] { 0x11, 0x22, 0x33, 0x44 });
+remote.sendBytes(message, callback);
+----
+
+
 [[pingpong]]
 ==== Send Ping / Pong Control Frame
 
-You can also send Ping and Pong control frames using the RemoteEndpoint.
+You can also send Ping and Pong control frames using the `RemoteEndpoint`.
 
 [source, java, subs="{sub-order}"]
 ----
@@ -138,7 +182,7 @@ catch (IOException e)
 ----
 
 How to send a Ping control frame, with a payload of `"You There?"` (arriving at Remote Endpoint as a byte array payload).
-This will block until the message is sent, possibly throwing an IOException if unable to send the ping frame.
+This will block until the message is sent, possibly throwing an `IOException` if unable to send the ping frame.
 
 [source, java, subs="{sub-order}"]
 ----
@@ -158,143 +202,23 @@ catch (IOException e)
 ----
 
 How to send a Pong control frame, with a payload of `"Yup I'm here"` (arriving at Remote Endpoint as a byte array payload).
-This will block until the message is sent, possibly throwing an IOException if unable to send the pong frame.
+This will block until the message is sent, possibly throwing an `IOException` if unable to send the pong frame.
 
 To be correct in your usage of Pong frames, you should return the same byte array data that you received in the Ping frame.
 
-[[async]]
-==== Async Send Message
-
-However there are also 2 Async send message methods available:
-
-* link:{JDURL}/org/eclipse/jetty/websocket/api/RemoteEndpoint.html#sendBytesByFuture(java.nio.ByteBuffer)[`RemoteEndpoint.sendBytesByFuture(ByteBuffer message)`]
-* link:{JDURL}/org/eclipse/jetty/websocket/api/RemoteEndpoint.html#sendStringByFuture(java.lang.String)[`RemoteEndpoint.sendStringByFuture(String message)`]
-
-Both return a `Future<Void>` that can be used to test for success and failure of the message send using standard http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/Future.html[`java.util.concurrent.Future`] behavior.
+You can also asynchronously send Ping and Pong frames using the `WriteCallback`, this will return before the Ping/Pong is
+transmitted and notify you of the result in `WriteCallback` `writeSuccess()` or `writeFailed()`.
 
 [source, java, subs="{sub-order}"]
 ----
 RemoteEndpoint remote = session.getRemote();
 
-// Async Send of a BINARY message to remote endpoint
-ByteBuffer buf = ByteBuffer.wrap(new byte[] { 0x11, 0x22, 0x33, 0x44 });
-remote.sendBytesByFuture(buf);
+String pingData = "You There?";
+ByteBuffer pingPayload = ByteBuffer.wrap(data.getBytes());
+
+String pongData = "Yup, I'm here";
+ByteBuffer pongPayload = ByteBuffer.wrap(data.getBytes());
+
+remote.sendPing(pingPayload, WriteCallback.NOOP);
+remote.sendPong(pongPayload, WriteCallback.NOOP);
 ----
-
-How to send a simple Binary message using the RemoteEndpoint.
-The message will be enqueued for outgoing write, but you will not know if it succeeded or failed.
-
-[source, java, subs="{sub-order}"]
-----
-RemoteEndpoint remote = session.getRemote();
-
-// Async Send of a BINARY message to remote endpoint
-ByteBuffer buf = ByteBuffer.wrap(new byte[] { 0x11, 0x22, 0x33, 0x44 });
-try
-{
-    Future<Void> fut = remote.sendBytesByFuture(buf);
-    // wait for completion (forever)
-    fut.get();
-}
-catch (ExecutionException | InterruptedException e)
-{
-    // Send failed
-    e.printStackTrace();
-}
-----
-
-How to send a simple Binary message using the RemoteEndpoint, tracking the `Future<Void>` to know if the send succeeded or failed.
-
-[source, java, subs="{sub-order}"]
-----
-RemoteEndpoint remote = session.getRemote();
-
-// Async Send of a BINARY message to remote endpoint
-ByteBuffer buf = ByteBuffer.wrap(new byte[] { 0x11, 0x22, 0x33, 0x44 });
-Future<Void> fut = null;
-try
-{
-    fut = remote.sendBytesByFuture(buf);
-    // wait for completion (timeout)
-    fut.get(2,TimeUnit.SECONDS);
-}
-catch (ExecutionException | InterruptedException e)
-{
-    // Send failed
-    e.printStackTrace();
-}
-catch (TimeoutException e)
-{
-    // timeout
-    e.printStackTrace();
-    if (fut != null)
-    {
-        // cancel the message
-        fut.cancel(true);
-    }
-}
-----
-
-How to send a simple Binary message using the RemoteEndpoint, tracking the `Future<Void>` and waiting only prescribed amount of time for the send to complete, cancelling the message if the timeout occurs.
-
-[source, java, subs="{sub-order}"]
-----
-RemoteEndpoint remote = session.getRemote();
-
-// Async Send of a TEXT message to remote endpoint
-remote.sendStringByFuture("Hello World");
-----
-
-How to send a simple Text message using the RemoteEndpoint.
-The message will be enqueued for outgoing write, but you will not know if it succeeded or failed.
-
-[source, java, subs="{sub-order}"]
-----
-RemoteEndpoint remote = session.getRemote();
-
-// Async Send of a TEXT message to remote endpoint
-try
-{
-    Future<Void> fut = remote.sendStringByFuture("Hello World");
-    // wait for completion (forever)
-    fut.get();
-}
-catch (ExecutionException | InterruptedException e)
-{
-    // Send failed
-    e.printStackTrace();
-}
-----
-
-How to send a simple Binary message using the RemoteEndpoint, tracking the `Future<Void>` to know if the send succeeded or failed.
-
-[source, java, subs="{sub-order}"]
-----
-RemoteEndpoint remote = session.getRemote();
-
-// Async Send of a TEXT message to remote endpoint
-Future<Void> fut = null;
-try
-{
-    fut = remote.sendStringByFuture("Hello World");
-    // wait for completion (timeout)
-    fut.get(2,TimeUnit.SECONDS);
-}
-catch (ExecutionException | InterruptedException e)
-{
-    // Send failed
-    e.printStackTrace();
-}
-catch (TimeoutException e)
-{
-    // timeout
-    e.printStackTrace();
-    if (fut != null)
-    {
-        // cancel the message
-        fut.cancel(true);
-    }
-}
-----
-
-How to send a simple Binary message using the RemoteEndpoint, tracking the `Future<Void>` and waiting only prescribed amount of time for the send to complete, cancelling the message if the timeout occurs.

jetty-documentation/src/main/asciidoc/embedded-guide/server/websockets/jetty/jetty-websocket-api-session.adoc

@@ -54,12 +54,12 @@ What is the Local and Remote Address.
 
 [source, java, subs="{sub-order}"]
 ----
-InetSocketAddress remoteAddr = session.getRemoteAddress();
+SocketAddress remoteAddr = session.getRemoteAddress();
 ----
 
 Get and Set the Idle Timeout
 
 [source, java, subs="{sub-order}"]
 ----
-session.setIdleTimeout(2000); // 2 second timeout
+session.setIdleTimeout(Duration.ofMillis(2000));
 ----

jetty-documentation/src/main/asciidoc/embedded-guide/server/websockets/jetty/jetty-websocket-server-api.adoc

@@ -19,7 +19,7 @@
 [[jetty-websocket-server-api]]
 === Jetty WebSocket Server API
 
-Jetty provides the ability to wire up WebSocket endpoints to Servlet Path Specs via the use of a WebSocketServlet bridge servlet.
+Jetty provides the ability to wire up WebSocket endpoints to Servlet Path Specs via the use of a `JettyWebSocketServlet` bridge servlet.
 
 Internally, Jetty manages the HTTP Upgrade to WebSocket and migration from a HTTP Connection to a WebSocket Connection.
 
@@ -27,7 +27,7 @@ This will only work when running within the Jetty Container (unlike past Jetty t
 
 ==== The Jetty WebSocketServlet
 
-To wire up your WebSocket to a specific path via the WebSocketServlet, you will need to extend org.eclipse.jetty.websocket.servlet.WebSocketServlet and specify what WebSocket object should be created with incoming Upgrade requests.
+To wire up your WebSocket to a specific path via the `JettyWebSocketServlet`, you will need to extend `org.eclipse.jetty.websocket.servlet.JettyWebSocketServlet` and specify what `WebSocket` object should be created with incoming Upgrade requests.
 
 [source, java, subs="{sub-order}"]
 ----
@@ -36,8 +36,8 @@ include::{SRCDIR}/jetty-websocket/jetty-websocket-tests/src/test/java/org/eclips
 
 This example will create a Servlet mapped via the http://docs.oracle.com/javaee/6/api/javax/servlet/annotation/WebServlet.html[@WebServlet] annotation to the Servlet path spec of `"/echo"` (or you can do this manually in the `WEB-INF/web.xml` of your web application) which will create MyEchoSocket instances when encountering HTTP Upgrade requests.
 
-The link:{JDURL}/org/eclipse/jetty/websocket/servlet/WebSocketServlet.html#configure(org.eclipse.jetty.websocket.servlet.WebSocketServletFactory)[`WebSocketServlet.configure(WebSocketServletFactory factory)`] is where you put your specific configuration for your WebSocket.
-In the example we specify a 10 second idle timeout and register MyEchoSocket with the default WebSocketCreator the WebSocket class we want to be created on Upgrade.
+The link:{JDURL}/org/eclipse/jetty/websocket/servlet/JettyWebSocketServlet.html#configure(org.eclipse.jetty.websocket.servlet.JettyWebSocketServletFactory)[`JettyWebSocketServlet.configure(JettyWebSocketServletFactory factory)`] is where you put your specific configuration for your WebSocket.
+In the example we specify a 10 second idle timeout and register MyEchoSocket with the default JettyWebSocketCreator the WebSocket class we want to be created on Upgrade.
 
 ____
 [NOTE]
@@ -46,21 +46,21 @@ when configuring websockets. Be sure the websocket configuration is
 lower than your firewall or router.
 ____
 
-==== Using the WebSocketCreator
+==== Using the JettyWebSocketCreator
 
-All WebSocket's are created via whatever link:{JDURL}/org/eclipse/jetty/websocket/servlet/WebSocketCreator.html[WebSocketCreator] you have registered with the link:{JDURL}/org/eclipse/jetty/websocket/servlet/WebSocketServletFactory.html[WebSocketServletFactory].
+All WebSocket's are created via whatever link:{JDURL}/org/eclipse/jetty/websocket/servlet/JettyWebSocketCreator.html[JettyWebSocketCreator] you have registered with the link:{JDURL}/org/eclipse/jetty/websocket/servlet/JettyWebSocketServletFactory.html[JettyWebSocketServletFactory].
 
-By default, the WebSocketServletFactory is a simple WebSocketCreator capable of creating a single WebSocket object.
-Use link:{JDURL}/org/eclipse/jetty/websocket/servlet/WebSocketServletFactory.html#register(java.lang.Class)[`WebSocketCreator.register(Class<?> websocket)`] to tell the WebSocketServletFactory which class it should instantiate (make sure it has a default constructor).
+By default, the `JettyWebSocketServletFactory` is a simple `JettyWebSocketCreator` capable of creating a single WebSocket object.
+Use link:{JDURL}/org/eclipse/jetty/websocket/servlet/JettyWebSocketServletFactory.html#register(java.lang.Class)[`JettyWebSocketCreator.register(Class<?> websocket)`] to tell the `JettyWebSocketServletFactory` which class it should instantiate (make sure it has a default constructor).
 
-If you have a more complicated creation scenario, you might want to provide your own WebSocketCreator that bases the WebSocket it creates off of information present in the UpgradeRequest object.
+If you have a more complicated creation scenario, you might want to provide your own `JettyWebSocketCreator` that bases the WebSocket it creates off of information present in the `UpgradeRequest` object.
 
 [source, java, subs="{sub-order}"]
 ----
 include::{SRCDIR}/jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/examples/MyAdvancedEchoCreator.java[]
 ----
 
-Here we show a WebSocketCreator that will utilize the http://tools.ietf.org/html/rfc6455#section-1.9[WebSocket subprotocol] information from request to determine what WebSocket type should be
+Here we show a `JettyWebSocketCreator` that will utilize the http://tools.ietf.org/html/rfc6455#section-1.9[WebSocket subprotocol] information from request to determine what WebSocket type should be
 created.
 
 [source, java, subs="{sub-order}"]
@@ -68,9 +68,9 @@ created.
 include::{SRCDIR}/jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/examples/MyAdvancedEchoServlet.java[]
 ----
 
-When you want a custom WebSocketCreator, use link:{JDURL}/org/eclipse/jetty/websocket/servlet/WebSocketServletFactory.html#setCreator(org.eclipse.jetty.websocket.servlet.WebSocketCreator)[`WebSocketServletFactory.setCreator(WebSocketCreator creator)`] and the WebSocketServletFactory will use your creator for all incoming Upgrade requests on this servlet.
+When you want a custom `JettyWebSocketCreator`, use link:{JDURL}/org/eclipse/jetty/websocket/servlet/JettyWebSocketServletFactory.html#setCreator(org.eclipse.jetty.websocket.servlet.JettyWebSocketCreator)[`JettyWebSocketServletFactory.setCreator(JettyWebSocketCreator creator)`] and the `JettyWebSocketServletFactory` will use your creator for all incoming Upgrade requests on this servlet.
 
-Other uses for a WebSocketCreator:
+Other uses for a `JettyWebSocketCreator`:
 
 * Controlling the selection of WebSocket subprotocol
 * Performing any WebSocket origin you deem important.
@@ -78,4 +78,4 @@ Other uses for a WebSocketCreator:
 * Obtaining the Servlet HttpSession object (if it exists)
 * Specifying a response status code and reason
 
-If you don't want to accept the upgrade, simply return null from the link:{JDURL}/org/eclipse/jetty/websocket/servlet/WebSocketCreator.html#createWebSocket(org.eclipse.jetty.websocket.api.UpgradeRequest, org.eclipse.jetty.websocket.api.UpgradeResponse)[`WebSocketCreator.createWebSocket(UpgradeRequest req, UpgradeResponse resp)`] method.
+If you don't want to accept the upgrade, simply return null from the link:{JDURL}/org/eclipse/jetty/websocket/servlet/JettyWebSocketCreator.html#createWebSocket(org.eclipse.jetty.websocket.api.UpgradeRequest,org.eclipse.jetty.websocket.api.UpgradeResponse)[`JettyWebSocketCreator.createWebSocket(UpgradeRequest req, UpgradeResponse resp)`] method.

jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpChannelOverFCGI.java

@@ -81,6 +81,11 @@ public class HttpChannelOverFCGI extends HttpChannel
         return sender.isFailed() || receiver.isFailed();
     }
 
+    void receive()
+    {
+        connection.process();
+    }
+
     @Override
     public void send(HttpExchange exchange)
     {

jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpConnectionOverFCGI.java

@@ -49,8 +49,9 @@ import org.eclipse.jetty.http.HttpHeaderValue;
 import org.eclipse.jetty.io.AbstractConnection;
 import org.eclipse.jetty.io.ByteBufferPool;
 import org.eclipse.jetty.io.EndPoint;
+import org.eclipse.jetty.io.RetainableByteBuffer;
 import org.eclipse.jetty.util.BufferUtil;
-import org.eclipse.jetty.util.CompletableCallback;
+import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.Promise;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
@@ -68,7 +69,7 @@ public class HttpConnectionOverFCGI extends AbstractConnection implements IConne
     private final Flusher flusher;
     private final Delegate delegate;
     private final ClientParser parser;
-    private ByteBuffer buffer;
+    private RetainableByteBuffer networkBuffer;
 
     public HttpConnectionOverFCGI(EndPoint endPoint, HttpDestination destination, Promise<Connection> promise)
     {
@@ -114,69 +115,80 @@ public class HttpConnectionOverFCGI extends AbstractConnection implements IConne
     @Override
     public void onFillable()
     {
-        buffer = acquireBuffer();
-        process(buffer);
+        networkBuffer = newNetworkBuffer();
+        process();
     }
 
-    private ByteBuffer acquireBuffer()
+    private void reacquireNetworkBuffer()
+    {
+        if (networkBuffer == null)
+            throw new IllegalStateException();
+        if (networkBuffer.hasRemaining())
+            throw new IllegalStateException();
+        networkBuffer.release();
+        networkBuffer = newNetworkBuffer();
+        if (LOG.isDebugEnabled())
+            LOG.debug("Reacquired {}", networkBuffer);
+    }
+
+    private RetainableByteBuffer newNetworkBuffer()
     {
         HttpClient client = destination.getHttpClient();
         ByteBufferPool bufferPool = client.getByteBufferPool();
-        return bufferPool.acquire(client.getResponseBufferSize(), true);
+        // TODO: configure directness.
+        return new RetainableByteBuffer(bufferPool, client.getResponseBufferSize(), true);
     }
 
-    private void releaseBuffer(ByteBuffer buffer)
+    private void releaseNetworkBuffer()
     {
-        @SuppressWarnings("ReferenceEquality")
-        boolean isCurrentBuffer = (this.buffer == buffer);
-        assert (isCurrentBuffer);
-        HttpClient client = destination.getHttpClient();
-        ByteBufferPool bufferPool = client.getByteBufferPool();
-        bufferPool.release(buffer);
-        this.buffer = null;
+        if (networkBuffer == null)
+            throw new IllegalStateException();
+        if (networkBuffer.hasRemaining())
+            throw new IllegalStateException();
+        networkBuffer.release();
+        if (LOG.isDebugEnabled())
+            LOG.debug("Released {}", networkBuffer);
+        this.networkBuffer = null;
     }
 
-    private void process(ByteBuffer buffer)
+    void process()
     {
         try
         {
             EndPoint endPoint = getEndPoint();
-            boolean looping = false;
             while (true)
             {
-                if (!looping && parse(buffer))
+                if (parse(networkBuffer.getBuffer()))
                     return;
 
-                int read = endPoint.fill(buffer);
+                if (networkBuffer.getReferences() > 1)
+                    reacquireNetworkBuffer();
+
+                // The networkBuffer may have been reacquired.
+                int read = endPoint.fill(networkBuffer.getBuffer());
                 if (LOG.isDebugEnabled())
                     LOG.debug("Read {} bytes from {}", read, endPoint);
 
-                if (read > 0)
+                if (read == 0)
                 {
-                    if (parse(buffer))
-                        return;
-                }
-                else if (read == 0)
-                {
-                    releaseBuffer(buffer);
+                    releaseNetworkBuffer();
                     fillInterested();
                     return;
                 }
-                else
+                else if (read < 0)
                 {
-                    releaseBuffer(buffer);
+                    releaseNetworkBuffer();
                     shutdown();
                     return;
                 }
-
-                looping = true;
             }
         }
         catch (Exception x)
         {
             if (LOG.isDebugEnabled())
                 LOG.debug(x);
-            releaseBuffer(buffer);
+            networkBuffer.clear();
+            releaseNetworkBuffer();
             close(x);
         }
     }
@@ -404,13 +416,13 @@ public class HttpConnectionOverFCGI extends AbstractConnection implements IConne
         }
 
         @Override
-        public void onHeaders(int request)
+        public boolean onHeaders(int request)
         {
             HttpChannelOverFCGI channel = activeChannels.get(request);
             if (channel != null)
-                channel.responseHeaders();
-            else
+                return !channel.responseHeaders();
             noChannel(request);
+            return false;
         }
 
         @Override
@@ -423,26 +435,8 @@ public class HttpConnectionOverFCGI extends AbstractConnection implements IConne
                     HttpChannelOverFCGI channel = activeChannels.get(request);
                     if (channel != null)
                     {
-                        CompletableCallback callback = new CompletableCallback()
-                        {
-                            @Override
-                            public void resume()
-                            {
-                                if (LOG.isDebugEnabled())
-                                    LOG.debug("Content consumed asynchronously, resuming processing");
-                                process(HttpConnectionOverFCGI.this.buffer);
-                            }
-
-                            @Override
-                            public void abort(Throwable x)
-                            {
-                                close(x);
-                            }
-                        };
-                        // Do not short circuit these calls.
-                        boolean proceed = channel.content(buffer, callback);
-                        boolean async = callback.tryComplete();
-                        return !proceed || async;
+                        networkBuffer.retain();
+                        return !channel.content(buffer, Callback.from(networkBuffer::release, HttpConnectionOverFCGI.this::close));
                     }
                     else
                     {

jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpReceiverOverFCGI.java

@@ -33,6 +33,12 @@ public class HttpReceiverOverFCGI extends HttpReceiver
         super(channel);
     }
 
+    @Override
+    protected HttpChannelOverFCGI getHttpChannel()
+    {
+        return (HttpChannelOverFCGI)super.getHttpChannel();
+    }
+
     @Override
     protected boolean responseBegin(HttpExchange exchange)
     {
@@ -68,4 +74,10 @@ public class HttpReceiverOverFCGI extends HttpReceiver
     {
         return super.responseFailure(failure);
     }
+
+    @Override
+    protected void receive()
+    {
+        getHttpChannel().receive();
+    }
 }

jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/parser/ClientParser.java

@@ -80,9 +80,9 @@ public class ClientParser extends Parser
         }
 
         @Override
-        public void onHeaders(int request)
+        public boolean onHeaders(int request)
         {
-            listener.onHeaders(request);
+            return listener.onHeaders(request);
         }
 
         @Override

jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/parser/Parser.java

@@ -135,7 +135,11 @@ public abstract class Parser
     {
         public void onHeader(int request, HttpField field);
 
-        public void onHeaders(int request);
+        /**
+         * @param request the request id
+         * @return true to signal to the parser to stop parsing, false to continue parsing
+         */
+        public boolean onHeaders(int request);
 
         /**
          * @param request the request id
@@ -158,8 +162,9 @@ public abstract class Parser
             }
 
             @Override
-            public void onHeaders(int request)
+            public boolean onHeaders(int request)
             {
+                return false;
             }
 
             @Override

jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/parser/ResponseContentParser.java

@@ -82,7 +82,7 @@ public class ResponseContentParser extends StreamContentParser
         parsers.remove(request);
     }
 
-    private class ResponseParser implements HttpParser.ResponseHandler
+    private static class ResponseParser implements HttpParser.ResponseHandler
     {
         private final HttpFields fields = new HttpFields();
         private ClientParser.Listener listener;
@@ -90,6 +90,7 @@ public class ResponseContentParser extends StreamContentParser
         private final FCGIHttpParser httpParser;
         private State state = State.HEADERS;
         private boolean seenResponseCode;
+        private boolean stalled;
 
         private ResponseParser(ClientParser.Listener listener, int request)
         {
@@ -111,7 +112,11 @@ public class ResponseContentParser extends StreamContentParser
                     case HEADERS:
                     {
                         if (httpParser.parseNext(buffer))
+                        {
                             state = State.CONTENT_MODE;
+                            if (stalled)
+                                return true;
+                        }
                         remaining = buffer.remaining();
                         break;
                     }
@@ -151,21 +156,7 @@ public class ResponseContentParser extends StreamContentParser
         }
 
         @Override
-        public int getHeaderCacheSize()
-        {
-            // TODO: configure this
-            return 4096;
-        }
-
-        @Override
-        public boolean isHeaderCacheCaseSensitive()
-        {
-            // TODO get from configuration
-            return false;
-        }
-
-        @Override
-        public boolean startResponse(HttpVersion version, int status, String reason)
+        public void startResponse(HttpVersion version, int status, String reason)
         {
             // The HTTP request line does not exist in FCGI responses
             throw new IllegalStateException();
@@ -247,16 +238,17 @@ public class ResponseContentParser extends StreamContentParser
             }
         }
 
-        private void notifyHeaders()
+        private boolean notifyHeaders()
         {
             try
             {
-                listener.onHeaders(request);
+                return listener.onHeaders(request);
             }
             catch (Throwable x)
             {
                 if (LOG.isDebugEnabled())
                     LOG.debug("Exception while invoking listener " + listener, x);
+                return false;
             }
         }
 
@@ -269,8 +261,10 @@ public class ResponseContentParser extends StreamContentParser
                 notifyBegin(200, "OK");
                 notifyHeaders(fields);
             }
-            notifyHeaders();
-            // Return from HTTP parsing so that we can parse the content.
+            // Remember whether we have demand.
+            stalled = notifyHeaders();
+            // Always return from HTTP parsing so that we
+            // can parse the content with the FCGI parser.
             return true;
         }
 

jetty-fcgi/fcgi-client/src/test/java/org/eclipse/jetty/fcgi/generator/ClientGeneratorTest.java

@@ -109,10 +109,11 @@ public class ClientGeneratorTest
             }
 
             @Override
-            public void onHeaders(int request)
+            public boolean onHeaders(int request)
             {
                 assertEquals(id, request);
                 params.set(params.get() * primes[4]);
+                return false;
             }
         });
 

jetty-fcgi/fcgi-client/src/test/java/org/eclipse/jetty/fcgi/parser/ClientParserTest.java

@@ -91,10 +91,11 @@ public class ClientParserTest
             }
 
             @Override
-            public void onHeaders(int request)
+            public boolean onHeaders(int request)
             {
                 assertEquals(id, request);
                 params.set(params.get() * primes[2]);
+                return false;
             }
         });
 

jetty-fcgi/fcgi-server/src/main/java/org/eclipse/jetty/fcgi/server/HttpTransportOverFCGI.java

@@ -50,12 +50,6 @@ public class HttpTransportOverFCGI implements HttpTransport
         this.request = request;
     }
 
-    @Override
-    public boolean isOptimizedForDirectBuffers()
-    {
-        return false;
-    }
-
     @Override
     public void send(MetaData.Request request, MetaData.Response response, ByteBuffer content, boolean lastContent, Callback callback)
     {

jetty-fcgi/fcgi-server/src/main/java/org/eclipse/jetty/fcgi/server/ServerFCGIConnection.java

@@ -144,7 +144,7 @@ public class ServerFCGIConnection extends AbstractConnection
         }
 
         @Override
-        public void onHeaders(int request)
+        public boolean onHeaders(int request)
         {
             HttpChannelOverFCGI channel = channels.get(request);
             if (LOG.isDebugEnabled())
@@ -154,6 +154,7 @@ public class ServerFCGIConnection extends AbstractConnection
                 channel.onRequest();
                 channel.dispatch();
             }
+            return false;
         }
 
         @Override

jetty-home/pom.xml

@@ -723,6 +723,11 @@
       <artifactId>jetty-alpn-java-server</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-openid</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-alpn-conscrypt-server</artifactId>

jetty-home/src/main/resources/etc/jetty-setuid.xml

@@ -6,7 +6,7 @@
 <!-- ================================================================ -->
 <Configure id="Server" class="org.eclipse.jetty.server.Server">
 
-  <Call name="addLifeCycleListener">
+  <Call name="addEventListener">
     <Arg>
       <New class="org.eclipse.jetty.setuid.SetUIDListener">
         <Set name="startServerAsPrivileged"><Property name="jetty.setuid.startServerAsPrivileged" default="false"/></Set>

jetty-home/src/main/resources/etc/jetty-started.xml

@@ -5,7 +5,7 @@
 <!-- Mixin the Start FileNoticeLifeCycleListener                     -->
 <!-- =============================================================== -->
 <Configure id="Server" class="org.eclipse.jetty.server.Server">
-  <Call name="addLifeCycleListener">
+  <Call name="addEventListener">
     <Arg>
       <New class="org.eclipse.jetty.util.component.FileNoticeLifeCycleListener">
         <Arg><Property name="jetty.state" default="./jetty.state"/></Arg>

jetty-http/src/main/java/org/eclipse/jetty/http/CookieCompliance.java

@@ -34,7 +34,7 @@ import static java.util.EnumSet.noneOf;
  */
 public class CookieCompliance implements ComplianceViolation.Mode
 {
-    enum Violation implements ComplianceViolation
+    public enum Violation implements ComplianceViolation
     {
         COMMA_NOT_VALID_OCTET("https://tools.ietf.org/html/rfc6265#section-4.1.1", "Comma not valid as cookie-octet or separator"),
         RESERVED_NAMES_NOT_DOLLAR_PREFIXED("https://tools.ietf.org/html/rfc6265#section-4.1.1", "Reserved names no longer use '$' prefix");
@@ -57,13 +57,13 @@ public class CookieCompliance implements ComplianceViolation.Mode
         @Override
         public String getURL()
         {
-            return null;
+            return url;
         }
 
         @Override
         public String getDescription()
         {
-            return null;
+            return description;
         }
     }
 
@@ -87,9 +87,8 @@ public class CookieCompliance implements ComplianceViolation.Mode
 
     private CookieCompliance(String name, Set<Violation> violations)
     {
-        Objects.nonNull(violations);
         _name = name;
-        _violations = unmodifiableSet(copyOf(violations));
+        _violations = unmodifiableSet(copyOf(Objects.requireNonNull(violations)));
     }
 
     @Override

jetty-http/src/main/java/org/eclipse/jetty/http/HttpCookie.java

@@ -24,7 +24,7 @@ import java.util.concurrent.TimeUnit;
 import org.eclipse.jetty.util.QuotedStringTokenizer;
 import org.eclipse.jetty.util.StringUtil;
 
-// TODO consider replacing this with java.net.HttpCookie
+// TODO consider replacing this with java.net.HttpCookie (once it supports RFC6265)
 public class HttpCookie
 {
     private static final String __COOKIE_DELIM = "\",;\\ \t";
@@ -33,14 +33,14 @@ public class HttpCookie
     /**
      *If this string is found within the comment parsed with {@link #isHttpOnlyInComment(String)} the check will return true
      **/
-    private static final String HTTP_ONLY_COMMENT = "__HTTP_ONLY__";
+    public static final String HTTP_ONLY_COMMENT = "__HTTP_ONLY__";
     /**
      *These strings are used by {@link #getSameSiteFromComment(String)} to check for a SameSite specifier in the comment
      **/
     private static final String SAME_SITE_COMMENT = "__SAME_SITE_";
-    private static final String SAME_SITE_NONE_COMMENT = SAME_SITE_COMMENT + "NONE__";
-    private static final String SAME_SITE_LAX_COMMENT = SAME_SITE_COMMENT + "LAX__";
-    private static final String SAME_SITE_STRICT_COMMENT = SAME_SITE_COMMENT + "STRICT__";
+    public static final String SAME_SITE_NONE_COMMENT = SAME_SITE_COMMENT + "NONE__";
+    public static final String SAME_SITE_LAX_COMMENT = SAME_SITE_COMMENT + "LAX__";
+    public static final String SAME_SITE_STRICT_COMMENT = SAME_SITE_COMMENT + "STRICT__";
 
     public enum SameSite
     {
@@ -428,17 +428,17 @@ public class HttpCookie
     {
         if (comment != null)
         {
-            if (comment.contains(SAME_SITE_NONE_COMMENT))
+            if (comment.contains(SAME_SITE_STRICT_COMMENT))
             {
-                return SameSite.NONE;
+                return SameSite.STRICT;
             }
             if (comment.contains(SAME_SITE_LAX_COMMENT))
             {
                 return SameSite.LAX;
             }
-            if (comment.contains(SAME_SITE_STRICT_COMMENT))
+            if (comment.contains(SAME_SITE_NONE_COMMENT))
             {
-                return SameSite.STRICT;
+                return SameSite.NONE;
             }
         }
 
@@ -462,6 +462,44 @@ public class HttpCookie
         return strippedComment.length() == 0 ? null : strippedComment;
     }
 
+    public static String getCommentWithAttributes(String comment, boolean httpOnly, SameSite sameSite)
+    {
+        if (comment == null && sameSite == null)
+            return null;
+
+        StringBuilder builder = new StringBuilder();
+        if (StringUtil.isNotBlank(comment))
+        {
+            comment = getCommentWithoutAttributes(comment);
+            if (StringUtil.isNotBlank(comment))
+                builder.append(comment);
+        }
+        if (httpOnly)
+            builder.append(HTTP_ONLY_COMMENT);
+
+        if (sameSite != null)
+        {
+            switch (sameSite)
+            {
+                case NONE:
+                    builder.append(SAME_SITE_NONE_COMMENT);
+                    break;
+                case STRICT:
+                    builder.append(SAME_SITE_STRICT_COMMENT);
+                    break;
+                case LAX:
+                    builder.append(SAME_SITE_LAX_COMMENT);
+                    break;
+                default:
+                    throw new IllegalArgumentException(sameSite.toString());
+            }
+        }
+
+        if (builder.length() == 0)
+            return null;
+        return builder.toString();
+    }
+
     public static class SetCookieHttpField extends HttpField
     {
         final HttpCookie _cookie;

jetty-http/src/main/java/org/eclipse/jetty/http/HttpField.java

@@ -69,6 +69,11 @@ public class HttpField
         return _name;
     }
 
+    public String getLowerCaseName()
+    {
+        return _header != null ? _header.lowerCaseName() : StringUtil.asciiToLowerCase(_name);
+    }
+
     public String getValue()
     {
         return _value;

jetty-http/src/main/java/org/eclipse/jetty/http/HttpGenerator.java

@@ -78,7 +78,7 @@ public class HttpGenerator
         FLUSH,                  // The buffers previously generated should be flushed 
         CONTINUE,               // Continue generating the message
         SHUTDOWN_OUT,           // Need EOF to be signaled
-        DONE                    // Message generation complete
+        DONE                    // The current phase of generation is complete
     }
 
     // other statics
@@ -686,19 +686,26 @@ public class HttpGenerator
             _endOfContent = EndOfContent.NO_CONTENT;
 
             // But it is an error if there actually is content
-            if (_contentPrepared > 0 || contentLength > 0)
+            if (_contentPrepared > 0)
+                throw new BadMessageException(INTERNAL_SERVER_ERROR_500, "Content for no content response");
+
+            if (contentLengthField)
+            {
+                if (response != null && response.getStatus() == HttpStatus.NOT_MODIFIED_304)
+                    putContentLength(header, contentLength);
+                else if (contentLength > 0)
                 {
                     if (_contentPrepared == 0 && last)
                     {
                         // TODO discard content for backward compatibility with 9.3 releases
                         // TODO review if it is still needed in 9.4 or can we just throw.
                         content.clear();
-                    contentLength = 0;
                     }
                     else
                         throw new BadMessageException(INTERNAL_SERVER_ERROR_500, "Content for no content response");
                 }
             }
+        }
         // Else if we are HTTP/1.1 and the content length is unknown and we are either persistent
         // or it is a request with content (which cannot EOF) or the app has requested chunking
         else if (http11 && (chunkedHint || contentLength < 0 && (_persistent || assumedContentRequest)))

jetty-http/src/main/java/org/eclipse/jetty/http/HttpHeader.java

@@ -150,6 +150,7 @@ public enum HttpHeader
     }
 
     private final String _string;
+    private final String _lowerCase;
     private final byte[] _bytes;
     private final byte[] _bytesColonSpace;
     private final ByteBuffer _buffer;
@@ -157,11 +158,17 @@ public enum HttpHeader
     HttpHeader(String s)
     {
         _string = s;
+        _lowerCase = StringUtil.asciiToLowerCase(s);
         _bytes = StringUtil.getBytes(s);
         _bytesColonSpace = StringUtil.getBytes(s + ": ");
         _buffer = ByteBuffer.wrap(_bytes);
     }
 
+    public String lowerCaseName()
+    {
+        return _lowerCase;
+    }
+
     public ByteBuffer toBuffer()
     {
         return _buffer.asReadOnlyBuffer();

jetty-http/src/main/java/org/eclipse/jetty/http/HttpParser.java

@@ -24,7 +24,6 @@ import java.util.EnumSet;
 import java.util.List;
 import java.util.Locale;
 
-import org.eclipse.jetty.http.HttpCompliance.Violation;
 import org.eclipse.jetty.http.HttpTokens.EndOfContent;
 import org.eclipse.jetty.util.ArrayTernaryTrie;
 import org.eclipse.jetty.util.ArrayTrie;
@@ -35,6 +34,8 @@ import org.eclipse.jetty.util.Utf8StringBuilder;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 
+import static org.eclipse.jetty.http.HttpCompliance.RFC7230;
+import static org.eclipse.jetty.http.HttpCompliance.Violation;
 import static org.eclipse.jetty.http.HttpCompliance.Violation.CASE_SENSITIVE_FIELD_NAME;
 import static org.eclipse.jetty.http.HttpCompliance.Violation.MULTIPLE_CONTENT_LENGTHS;
 import static org.eclipse.jetty.http.HttpCompliance.Violation.NO_COLON_AFTER_FIELD_NAME;
@@ -136,6 +137,7 @@ public class HttpParser
         CHUNK_SIZE,
         CHUNK_PARAMS,
         CHUNK,
+        CONTENT_END,
         TRAILER,
         END,
         CLOSE,  // The associated stream/endpoint should be closed
@@ -160,7 +162,6 @@ public class HttpParser
     private int _headerBytes;
     private boolean _host;
     private boolean _headerComplete;
-
     private volatile State _state = State.START;
     private volatile FieldState _fieldState = FieldState.FIELD;
     private volatile boolean _eof;
@@ -170,6 +171,7 @@ public class HttpParser
     private Utf8StringBuilder _uri = new Utf8StringBuilder(INITIAL_URI_LENGTH); // Tune?
     private EndOfContent _endOfContent;
     private boolean _hasContentLength;
+    private boolean _hasTransferEncoding;
     private long _contentLength = -1;
     private long _contentPosition;
     private int _chunkLength;
@@ -178,9 +180,10 @@ public class HttpParser
     private boolean _cr;
     private ByteBuffer _contentChunk;
     private Trie<HttpField> _fieldCache;
-
     private int _length;
     private final StringBuilder _string = new StringBuilder();
+    private int _headerCacheSize = 1024;
+    private boolean _headerCacheCaseSensitive;
 
     static
     {
@@ -237,7 +240,7 @@ public class HttpParser
 
     private static HttpCompliance compliance()
     {
-        return HttpCompliance.RFC7230;
+        return RFC7230;
     }
 
     public HttpParser(RequestHandler handler)
@@ -290,6 +293,26 @@ public class HttpParser
         return _handler;
     }
 
+    public int getHeaderCacheSize()
+    {
+        return _headerCacheSize;
+    }
+
+    public void setHeaderCacheSize(int headerCacheSize)
+    {
+        _headerCacheSize = headerCacheSize;
+    }
+
+    public boolean isHeaderCacheCaseSensitive()
+    {
+        return _headerCacheCaseSensitive;
+    }
+
+    public void setHeaderCacheCaseSensitive(boolean headerCacheCaseSensitive)
+    {
+        _headerCacheCaseSensitive = headerCacheCaseSensitive;
+    }
+
     protected void checkViolation(Violation violation) throws BadMessageException
     {
         if (violation.isAllowedBy(_complianceMode))
@@ -528,16 +551,19 @@ public class HttpParser
     {
         boolean handleHeader = _handler.headerComplete();
         _headerComplete = true;
-        boolean handleContent = _handler.contentComplete();
-        boolean handleMessage = _handler.messageComplete();
-        return handleHeader || handleContent || handleMessage;
+        if (handleHeader)
+            return true;
+        setState(State.CONTENT_END);
+        return handleContentMessage();
     }
 
     private boolean handleContentMessage()
     {
         boolean handleContent = _handler.contentComplete();
-        boolean handleMessage = _handler.messageComplete();
-        return handleContent || handleMessage;
+        if (handleContent)
+            return true;
+        setState(State.END);
+        return _handler.messageComplete();
     }
 
     /* Parse a request or response line
@@ -707,7 +733,7 @@ public class HttpParser
 
                         case LF:
                             setState(State.HEADER);
-                            handle |= _responseHandler.startResponse(_version, _responseStatus, null);
+                            _responseHandler.startResponse(_version, _responseStatus, null);
                             break;
 
                         default:
@@ -725,10 +751,11 @@ public class HttpParser
                         case LF:
                             // HTTP/0.9
                             checkViolation(Violation.HTTP_0_9);
-                            handle = _requestHandler.startRequest(_methodString, _uri.toString(), HttpVersion.HTTP_0_9);
-                            setState(State.END);
+                            _requestHandler.startRequest(_methodString, _uri.toString(), HttpVersion.HTTP_0_9);
+                            setState(State.CONTENT);
+                            _endOfContent = EndOfContent.NO_CONTENT;
                             BufferUtil.clear(buffer);
-                            handle |= handleHeaderContentMessage();
+                            handle = handleHeaderContentMessage();
                             break;
 
                         case ALPHA:
@@ -804,16 +831,17 @@ public class HttpParser
                             if (_responseHandler != null)
                             {
                                 setState(State.HEADER);
-                                handle |= _responseHandler.startResponse(_version, _responseStatus, null);
+                                _responseHandler.startResponse(_version, _responseStatus, null);
                             }
                             else
                             {
                                 // HTTP/0.9
                                 checkViolation(Violation.HTTP_0_9);
-                                handle = _requestHandler.startRequest(_methodString, _uri.toString(), HttpVersion.HTTP_0_9);
-                                setState(State.END);
+                                _requestHandler.startRequest(_methodString, _uri.toString(), HttpVersion.HTTP_0_9);
+                                setState(State.CONTENT);
+                                _endOfContent = EndOfContent.NO_CONTENT;
                                 BufferUtil.clear(buffer);
-                                handle |= handleHeaderContentMessage();
+                                handle = handleHeaderContentMessage();
                             }
                             break;
 
@@ -834,15 +862,13 @@ public class HttpParser
                             checkVersion();
 
                             // Should we try to cache header fields?
-                            if (_fieldCache == null && _version.getVersion() >= HttpVersion.HTTP_1_1.getVersion() && _handler.getHeaderCacheSize() > 0)
-                            {
-                                int headerCache = _handler.getHeaderCacheSize();
+                            int headerCache = getHeaderCacheSize();
+                            if (_fieldCache == null && _version.getVersion() >= HttpVersion.HTTP_1_1.getVersion() && headerCache > 0)
                                 _fieldCache = new ArrayTernaryTrie<>(headerCache);
-                            }
 
                             setState(State.HEADER);
 
-                            handle |= _requestHandler.startRequest(_methodString, _uri.toString(), _version);
+                            _requestHandler.startRequest(_methodString, _uri.toString(), _version);
                             continue;
 
                         case ALPHA:
@@ -864,7 +890,7 @@ public class HttpParser
                         case LF:
                             String reason = takeString();
                             setState(State.HEADER);
-                            handle |= _responseHandler.startResponse(_version, _responseStatus, reason);
+                            _responseHandler.startResponse(_version, _responseStatus, reason);
                             continue;
 
                         case ALPHA:
@@ -916,6 +942,9 @@ public class HttpParser
                 switch (_header)
                 {
                     case CONTENT_LENGTH:
+                        if (_hasTransferEncoding)
+                            checkViolation(TRANSFER_ENCODING_WITH_CONTENT_LENGTH);
+
                         if (_hasContentLength)
                         {
                             checkViolation(MULTIPLE_CONTENT_LENGTHS);
@@ -924,9 +953,6 @@ public class HttpParser
                         }
                         _hasContentLength = true;
 
-                        if (_endOfContent == EndOfContent.CHUNKED_CONTENT)
-                            checkViolation(TRANSFER_ENCODING_WITH_CONTENT_LENGTH);
-
                         if (_endOfContent != EndOfContent.CHUNKED_CONTENT)
                         {
                             _contentLength = convertContentLength(_valueString);
@@ -938,9 +964,15 @@ public class HttpParser
                         break;
 
                     case TRANSFER_ENCODING:
+                        _hasTransferEncoding = true;
+
                         if (_hasContentLength)
                             checkViolation(TRANSFER_ENCODING_WITH_CONTENT_LENGTH);
 
+                        // we encountered another Transfer-Encoding header, but chunked was already set
+                        if (_endOfContent == EndOfContent.CHUNKED_CONTENT)
+                            throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Bad Transfer-Encoding, chunked not last");
+
                         if (HttpHeaderValue.CHUNKED.is(_valueString))
                         {
                             _endOfContent = EndOfContent.CHUNKED_CONTENT;
@@ -949,15 +981,26 @@ public class HttpParser
                         else
                         {
                             List<String> values = new QuotedCSV(_valueString).getValues();
-                            if (!values.isEmpty() && HttpHeaderValue.CHUNKED.is(values.get(values.size() - 1)))
+                            int chunked = -1;
+                            int len = values.size();
+                            for (int i = 0; i < len; i++)
                             {
+                                if (HttpHeaderValue.CHUNKED.is(values.get(i)))
+                                {
+                                    if (chunked != -1)
+                                        throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Bad Transfer-Encoding, multiple chunked tokens");
+                                    chunked = i;
+                                    // declared chunked
                                     _endOfContent = EndOfContent.CHUNKED_CONTENT;
                                     _contentLength = -1;
                                 }
-                            else if (values.stream().anyMatch(HttpHeaderValue.CHUNKED::is))
-                                throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Bad chunking");
+                                // we have a non-chunked token after a declared chunked token
+                                else if (_endOfContent == EndOfContent.CHUNKED_CONTENT)
+                                {
+                                    throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Bad Transfer-Encoding, chunked not last");
+                                }
+                            }
                         }
-
                         break;
 
                     case HOST:
@@ -1098,6 +1141,17 @@ public class HttpParser
                                 return _handler.messageComplete();
                             }
 
+                            // We found Transfer-Encoding headers, but none declared the 'chunked' token
+                            if (_hasTransferEncoding && _endOfContent != EndOfContent.CHUNKED_CONTENT)
+                            {
+                                if (_responseHandler == null || _endOfContent != EndOfContent.EOF_CONTENT)
+                                {
+                                    // Transfer-Encoding chunked not specified
+                                    // https://tools.ietf.org/html/rfc7230#section-3.3.1
+                                    throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Bad Transfer-Encoding, chunked not last");
+                                }
+                            }
+
                             // Was there a required host header?
                             if (!_host && _version == HttpVersion.HTTP_1_1 && _requestHandler != null)
                             {
@@ -1140,11 +1194,6 @@ public class HttpParser
                                     _headerComplete = true;
                                     return handle;
                                 }
-                                case NO_CONTENT:
-                                {
-                                    setState(State.END);
-                                    return handleHeaderContentMessage();
-                                }
                                 default:
                                 {
                                     setState(State.CONTENT);
@@ -1190,7 +1239,7 @@ public class HttpParser
                                         }
                                     }
 
-                                    if (v != null && _handler.isHeaderCacheCaseSensitive())
+                                    if (v != null && isHeaderCacheCaseSensitive())
                                     {
                                         String ev = BufferUtil.toString(buffer, buffer.position() + n.length() + 1, v.length(), StandardCharsets.ISO_8859_1);
                                         if (!v.equals(ev))
@@ -1443,10 +1492,18 @@ public class HttpParser
                 // Handle HEAD response
                 if (_responseStatus > 0 && _headResponse)
                 {
-                    setState(State.END);
+                    if (_state != State.CONTENT_END)
+                    {
+                        setState(State.CONTENT_END);
                         return handleContentMessage();
                     }
                     else
+                    {
+                        setState(State.END);
+                        return _handler.messageComplete();
+                    }
+                }
+                else
                 {
                     if (parseContent(buffer))
                         return true;
@@ -1463,11 +1520,18 @@ public class HttpParser
             // handle end states
             if (_state == State.END)
             {
-                // eat white space
-                while (buffer.remaining() > 0 && buffer.get(buffer.position()) <= HttpTokens.SPACE)
+                // Eat CR or LF white space, but not SP.
+                int whiteSpace = 0;
+                while (buffer.remaining() > 0)
                 {
+                    byte b = buffer.get(buffer.position());
+                    if (b != HttpTokens.CARRIAGE_RETURN && b != HttpTokens.LINE_FEED)
+                        break;
                     buffer.get();
+                    ++whiteSpace;
                 }
+                if (debugEnabled && whiteSpace > 0)
+                    LOG.debug("Discarded {} CR or LF characters", whiteSpace);
             }
             else if (isClose() || isClosed())
             {
@@ -1475,18 +1539,13 @@ public class HttpParser
             }
 
             // Handle EOF
-            if (_eof && !buffer.hasRemaining())
+            if (isAtEOF() && !buffer.hasRemaining())
             {
                 switch (_state)
                 {
                     case CLOSED:
                         break;
 
-                    case START:
-                        setState(State.CLOSED);
-                        _handler.earlyEOF();
-                        break;
-
                     case END:
                     case CLOSE:
                         setState(State.CLOSED);
@@ -1497,13 +1556,18 @@ public class HttpParser
                         if (_fieldState == FieldState.FIELD)
                         {
                             // Be forgiving of missing last CRLF
+                            setState(State.CONTENT_END);
+                            boolean handle = handleContentMessage();
+                            if (handle && _state == State.CONTENT_END)
+                                return true;
                             setState(State.CLOSED);
-                            return handleContentMessage();
+                            return handle;
                         }
                         setState(State.CLOSED);
                         _handler.earlyEOF();
                         break;
 
+                    case START:
                     case CONTENT:
                     case CHUNKED_CONTENT:
                     case CHUNK_SIZE:
@@ -1549,18 +1613,28 @@ public class HttpParser
     protected boolean parseContent(ByteBuffer buffer)
     {
         int remaining = buffer.remaining();
-        if (remaining == 0 && _state == State.CONTENT)
+        if (remaining == 0)
         {
+            switch (_state)
+            {
+                case CONTENT:
                     long content = _contentLength - _contentPosition;
-            if (content == 0)
+                    if (_endOfContent == EndOfContent.NO_CONTENT || content == 0)
                     {
-                setState(State.END);
+                        setState(State.CONTENT_END);
                         return handleContentMessage();
                     }
+                    break;
+                case CONTENT_END:
+                    setState(_endOfContent == EndOfContent.EOF_CONTENT ? State.CLOSED : State.END);
+                    return _handler.messageComplete();
+                default:
+                    // No bytes to parse, return immediately.
+                    return false;
+            }
         }
 
-        // Handle _content
-        byte ch;
+        // Handle content.
         while (_state.ordinal() < State.TRAILER.ordinal() && remaining > 0)
         {
             switch (_state)
@@ -1576,9 +1650,9 @@ public class HttpParser
                 case CONTENT:
                 {
                     long content = _contentLength - _contentPosition;
-                    if (content == 0)
+                    if (_endOfContent == EndOfContent.NO_CONTENT || content == 0)
                     {
-                        setState(State.END);
+                        setState(State.CONTENT_END);
                         return handleContentMessage();
                     }
                     else
@@ -1601,7 +1675,7 @@ public class HttpParser
 
                         if (_contentPosition == _contentLength)
                         {
-                            setState(State.END);
+                            setState(State.CONTENT_END);
                             return handleContentMessage();
                         }
                     }
@@ -1726,10 +1800,10 @@ public class HttpParser
                     break;
                 }
 
-                case CLOSED:
+                case CONTENT_END:
                 {
-                    BufferUtil.clear(buffer);
-                    return false;
+                    setState(_endOfContent == EndOfContent.EOF_CONTENT ? State.CLOSED : State.END);
+                    return _handler.messageComplete();
                 }
 
                 default:
@@ -1779,6 +1853,7 @@ public class HttpParser
         _endOfContent = EndOfContent.UNKNOWN_CONTENT;
         _contentLength = -1;
         _hasContentLength = false;
+        _hasTransferEncoding = false;
         _contentPosition = 0;
         _responseStatus = 0;
         _contentChunk = null;
@@ -1812,8 +1887,8 @@ public class HttpParser
         return String.format("%s{s=%s,%d of %d}",
             getClass().getSimpleName(),
             _state,
-            _contentPosition,
-            _contentLength);
+            getContentRead(),
+            getContentLength());
     }
 
     /* Event Handler interface
@@ -1863,13 +1938,6 @@ public class HttpParser
         default void badMessage(BadMessageException failure)
         {
         }
-
-        /**
-         * @return the size in bytes of the per parser header cache
-         */
-        int getHeaderCacheSize();
-
-        boolean isHeaderCacheCaseSensitive();
     }
 
     public interface RequestHandler extends HttpHandler
@@ -1880,9 +1948,8 @@ public class HttpParser
          * @param method The method
          * @param uri The raw bytes of the URI.  These are copied into a ByteBuffer that will not be changed until this parser is reset and reused.
          * @param version the http version in use
-         * @return true if handling parsing should return.
          */
-        boolean startRequest(String method, String uri, HttpVersion version);
+        void startRequest(String method, String uri, HttpVersion version);
     }
 
     public interface ResponseHandler extends HttpHandler
@@ -1893,9 +1960,8 @@ public class HttpParser
          * @param version the http version in use
          * @param status the response status
          * @param reason the response reason phrase
-         * @return true if handling parsing should return
          */
-        boolean startResponse(HttpVersion version, int status, String reason);
+        void startResponse(HttpVersion version, int status, String reason);
     }
 
     @SuppressWarnings("serial")

jetty-http/src/test/java/org/eclipse/jetty/http/HttpCookieTest.java

@@ -18,17 +18,25 @@
 
 package org.eclipse.jetty.http;
 
+import java.util.stream.Stream;
+
 import org.hamcrest.Matchers;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
 
 public class HttpCookieTest
 {
@@ -93,7 +101,6 @@ public class HttpCookieTest
         httpCookie = new HttpCookie("everything", "value", "domain", "path", 0, true, true, null, -1);
         assertEquals("everything=value; Path=path; Domain=domain; Expires=Thu, 01-Jan-1970 00:00:00 GMT; Max-Age=0; Secure; HttpOnly", httpCookie.getRFC6265SetCookie());
 
-
         httpCookie = new HttpCookie("everything", "value", "domain", "path", 0, true, true, null, -1, HttpCookie.SameSite.NONE);
         assertEquals("everything=value; Path=path; Domain=domain; Expires=Thu, 01-Jan-1970 00:00:00 GMT; Max-Age=0; Secure; HttpOnly; SameSite=None", httpCookie.getRFC6265SetCookie());
 
@@ -102,8 +109,11 @@ public class HttpCookieTest
 
         httpCookie = new HttpCookie("everything", "value", "domain", "path", 0, true, true, null, -1, HttpCookie.SameSite.STRICT);
         assertEquals("everything=value; Path=path; Domain=domain; Expires=Thu, 01-Jan-1970 00:00:00 GMT; Max-Age=0; Secure; HttpOnly; SameSite=Strict", httpCookie.getRFC6265SetCookie());
+    }
 
-        String[] badNameExamples = {
+    public static Stream<String> rfc6265BadNameSource()
+    {
+        return Stream.of(
             "\"name\"",
             "name\t",
             "na me",
@@ -113,25 +123,32 @@ public class HttpCookieTest
             "{name}",
             "[name]",
             "\""
-        };
+        );
+    }
 
-        for (String badNameExample : badNameExamples)
+    @ParameterizedTest
+    @MethodSource("rfc6265BadNameSource")
+    public void testSetRFC6265CookieBadName(String badNameExample)
     {
-            try
+        IllegalArgumentException ex = assertThrows(IllegalArgumentException.class,
+            () ->
             {
-                httpCookie = new HttpCookie(badNameExample, "value", null, "/", 1, true, true, null, -1);
+                HttpCookie httpCookie = new HttpCookie(badNameExample, "value", null, "/", 1, true, true, null, -1);
                 httpCookie.getRFC6265SetCookie();
-                fail(badNameExample);
-            }
-            catch (IllegalArgumentException ex)
-            {
-                // System.err.printf("%s: %s%n", ex.getClass().getSimpleName(), ex.getMessage());
-                assertThat("Testing bad name: [" + badNameExample + "]", ex.getMessage(),
-                    allOf(containsString("RFC6265"), containsString("RFC2616")));
-            }
+            });
+        // make sure that exception mentions just how mad of a name it truly is
+        assertThat("message", ex.getMessage(),
+            allOf(
+                // violation of Cookie spec
+                containsString("RFC6265"),
+                // violation of HTTP spec
+                containsString("RFC2616")
+            ));
     }
 
-        String[] badValueExamples = {
+    public static Stream<String> rfc6265BadValueSource()
+    {
+        return Stream.of(
             "va\tlue",
             "\t",
             "value\u0000",
@@ -143,39 +160,44 @@ public class HttpCookieTest
             "val\\ue",
             "val\"ue",
             "\""
-        };
+        );
+    }
 
-        for (String badValueExample : badValueExamples)
+    @ParameterizedTest
+    @MethodSource("rfc6265BadValueSource")
+    public void testSetRFC6265CookieBadValue(String badValueExample)
     {
-            try
+        IllegalArgumentException ex = assertThrows(IllegalArgumentException.class,
+            () ->
             {
-                httpCookie = new HttpCookie("name", badValueExample, null, "/", 1, true, true, null, -1);
+                HttpCookie httpCookie = new HttpCookie("name", badValueExample, null, "/", 1, true, true, null, -1);
                 httpCookie.getRFC6265SetCookie();
-                fail();
-            }
-            catch (IllegalArgumentException ex)
-            {
-                // System.err.printf("%s: %s%n", ex.getClass().getSimpleName(), ex.getMessage());
-                assertThat("Testing bad value [" + badValueExample + "]", ex.getMessage(), Matchers.containsString("RFC6265"));
-            }
+            });
+        assertThat("message", ex.getMessage(), containsString("RFC6265"));
     }
 
-        String[] goodNameExamples = {
+    public static Stream<String> rfc6265GoodNameSource()
+    {
+        return Stream.of(
             "name",
             "n.a.m.e",
             "na-me",
             "+name",
             "na*me",
             "na$me",
-            "#name"
-        };
+            "#name");
+    }
 
-        for (String goodNameExample : goodNameExamples)
+    @ParameterizedTest
+    @MethodSource("rfc6265GoodNameSource")
+    public void testSetRFC6265CookieGoodName(String goodNameExample)
     {
-            httpCookie = new HttpCookie(goodNameExample, "value", null, "/", 1, true, true, null, -1);
+        new HttpCookie(goodNameExample, "value", null, "/", 1, true, true, null, -1);
         // should not throw an exception
     }
 
+    public static Stream<String> rfc6265GoodValueSource()
+    {
         String[] goodValueExamples = {
             "value",
             "",
@@ -185,37 +207,150 @@ public class HttpCookieTest
             "val/ue",
             "v.a.l.u.e"
         };
+        return Stream.of(goodValueExamples);
+    }
 
-        for (String goodValueExample : goodValueExamples)
+    @ParameterizedTest
+    @MethodSource("rfc6265GoodValueSource")
+    public void testSetRFC6265CookieGoodValue(String goodValueExample)
     {
-            httpCookie = new HttpCookie("name", goodValueExample, null, "/", 1, true, true, null, -1);
+        new HttpCookie("name", goodValueExample, null, "/", 1, true, true, null, -1);
         // should not throw an exception
     }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+        "__HTTP_ONLY__",
+        "__HTTP_ONLY__comment",
+        "comment__HTTP_ONLY__"
+    })
+    public void testIsHttpOnlyInCommentTrue(String comment)
+    {
+        assertTrue(HttpCookie.isHttpOnlyInComment(comment), "Comment \"" + comment + "\"");
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+        "comment",
+        "",
+        "__",
+        "__HTTP__ONLY__",
+        "__http_only__",
+        "HTTP_ONLY",
+        "__HTTP__comment__ONLY__"
+    })
+    public void testIsHttpOnlyInCommentFalse(String comment)
+    {
+        assertFalse(HttpCookie.isHttpOnlyInComment(comment), "Comment \"" + comment + "\"");
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+        "__SAME_SITE_NONE__",
+        "__SAME_SITE_NONE____SAME_SITE_NONE__"
+    })
+    public void testGetSameSiteFromCommentNONE(String comment)
+    {
+        assertEquals(HttpCookie.getSameSiteFromComment(comment), HttpCookie.SameSite.NONE, "Comment \"" + comment + "\"");
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+        "__SAME_SITE_LAX__",
+        "__SAME_SITE_LAX____SAME_SITE_NONE__",
+        "__SAME_SITE_NONE____SAME_SITE_LAX__",
+        "__SAME_SITE_LAX____SAME_SITE_NONE__"
+    })
+    public void testGetSameSiteFromCommentLAX(String comment)
+    {
+        assertEquals(HttpCookie.getSameSiteFromComment(comment), HttpCookie.SameSite.LAX, "Comment \"" + comment + "\"");
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+        "__SAME_SITE_STRICT__",
+        "__SAME_SITE_NONE____SAME_SITE_STRICT____SAME_SITE_LAX__",
+        "__SAME_SITE_STRICT____SAME_SITE_LAX____SAME_SITE_NONE__",
+        "__SAME_SITE_STRICT____SAME_SITE_STRICT__"
+    })
+    public void testGetSameSiteFromCommentSTRICT(String comment)
+    {
+        assertEquals(HttpCookie.getSameSiteFromComment(comment), HttpCookie.SameSite.STRICT, "Comment \"" + comment + "\"");
+    }
+
+    /**
+     * A comment that does not have a declared SamesSite attribute defined
+     */
+    @ParameterizedTest
+    @ValueSource(strings = {
+        "__HTTP_ONLY__",
+        "comment",
+        // not jetty attributes
+        "SameSite=None",
+        "SameSite=Lax",
+        "SameSite=Strict",
+        // incomplete jetty attributes
+        "SAME_SITE_NONE",
+        "SAME_SITE_LAX",
+        "SAME_SITE_STRICT",
+    })
+    public void testGetSameSiteFromCommentUndefined(String comment)
+    {
+        assertNull(HttpCookie.getSameSiteFromComment(comment), "Comment \"" + comment + "\"");
+    }
+
+    public static Stream<Arguments> getCommentWithoutAttributesSource()
+    {
+        return Stream.of(
+            // normal - only attribute comment
+            Arguments.of("__SAME_SITE_LAX__", null),
+            // normal - no attribute comment
+            Arguments.of("comment", "comment"),
+            // mixed - attributes at end
+            Arguments.of("comment__SAME_SITE_NONE__", "comment"),
+            Arguments.of("comment__HTTP_ONLY____SAME_SITE_NONE__", "comment"),
+            // mixed - attributes at start
+            Arguments.of("__SAME_SITE_NONE__comment", "comment"),
+            Arguments.of("__HTTP_ONLY____SAME_SITE_NONE__comment", "comment"),
+            // mixed - attributes at start and end
+            Arguments.of("__SAME_SITE_NONE__comment__HTTP_ONLY__", "comment"),
+            Arguments.of("__HTTP_ONLY__comment__SAME_SITE_NONE__", "comment")
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("getCommentWithoutAttributesSource")
+    public void testGetCommentWithoutAttributes(String rawComment, String expectedComment)
+    {
+        String actualComment = HttpCookie.getCommentWithoutAttributes(rawComment);
+        if (expectedComment == null)
+        {
+            assertNull(actualComment);
+        }
+        else
+        {
+            assertEquals(actualComment, expectedComment);
+        }
     }
 
     @Test
-    public void testGetHttpOnlyFromComment()
+    public void testGetCommentWithAttributes()
     {
-        assertTrue(HttpCookie.isHttpOnlyInComment("__HTTP_ONLY__"));
-        assertTrue(HttpCookie.isHttpOnlyInComment("__HTTP_ONLY__comment"));
-        assertFalse(HttpCookie.isHttpOnlyInComment("comment"));
-    }
+        assertThat(HttpCookie.getCommentWithAttributes(null, false, null), nullValue());
+        assertThat(HttpCookie.getCommentWithAttributes("", false, null), nullValue());
+        assertThat(HttpCookie.getCommentWithAttributes("hello", false, null), is("hello"));
 
-    @Test
-    public void testGetSameSiteFromComment()
-    {
-        assertEquals(HttpCookie.getSameSiteFromComment("__SAME_SITE_NONE__"), HttpCookie.SameSite.NONE);
-        assertEquals(HttpCookie.getSameSiteFromComment("__SAME_SITE_LAX__"), HttpCookie.SameSite.LAX);
-        assertEquals(HttpCookie.getSameSiteFromComment("__SAME_SITE_STRICT__"), HttpCookie.SameSite.STRICT);
-        assertEquals(HttpCookie.getSameSiteFromComment("__SAME_SITE_NONE____SAME_SITE_STRICT__"), HttpCookie.SameSite.NONE);
-        assertNull(HttpCookie.getSameSiteFromComment("comment"));
-    }
+        assertThat(HttpCookie.getCommentWithAttributes(null, true, HttpCookie.SameSite.STRICT),
+            is("__HTTP_ONLY____SAME_SITE_STRICT__"));
+        assertThat(HttpCookie.getCommentWithAttributes("", true, HttpCookie.SameSite.NONE),
+            is("__HTTP_ONLY____SAME_SITE_NONE__"));
+        assertThat(HttpCookie.getCommentWithAttributes("hello", true, HttpCookie.SameSite.LAX),
+            is("hello__HTTP_ONLY____SAME_SITE_LAX__"));
 
-    @Test
-    public void getCommentWithoutAttributes()
-    {
-        assertEquals(HttpCookie.getCommentWithoutAttributes("comment__SAME_SITE_NONE__"), "comment");
-        assertEquals(HttpCookie.getCommentWithoutAttributes("comment__HTTP_ONLY____SAME_SITE_NONE__"), "comment");
-        assertNull(HttpCookie.getCommentWithoutAttributes("__SAME_SITE_LAX__"));
+        assertThat(HttpCookie.getCommentWithAttributes("__HTTP_ONLY____SAME_SITE_LAX__", false, null), nullValue());
+        assertThat(HttpCookie.getCommentWithAttributes("__HTTP_ONLY____SAME_SITE_LAX__", true, HttpCookie.SameSite.NONE),
+            is("__HTTP_ONLY____SAME_SITE_NONE__"));
+        assertThat(HttpCookie.getCommentWithAttributes("__HTTP_ONLY____SAME_SITE_LAX__hello", true, HttpCookie.SameSite.LAX),
+            is("hello__HTTP_ONLY____SAME_SITE_LAX__"));
     }
 }

jetty-http/src/test/java/org/eclipse/jetty/http/HttpGeneratorServerHTTPTest.java

@@ -32,6 +32,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class HttpGeneratorServerHTTPTest
@@ -70,7 +71,7 @@ public class HttpGeneratorServerHTTPTest
         assertEquals("OK??Test", _reason);
 
         if (_content == null)
-            assertTrue(run.result._body == null, msg);
+            assertNull(run.result._body, msg);
         else
             assertThat(msg, run.result._contentLength, either(equalTo(_content.length())).or(equalTo(-1)));
     }
@@ -248,10 +249,9 @@ public class HttpGeneratorServerHTTPTest
         }
 
         @Override
-        public boolean startResponse(HttpVersion version, int status, String reason)
+        public void startResponse(HttpVersion version, int status, String reason)
         {
             _reason = reason;
-            return false;
         }
 
         @Override
@@ -259,18 +259,6 @@ public class HttpGeneratorServerHTTPTest
         {
             throw failure;
         }
-
-        @Override
-        public int getHeaderCacheSize()
-        {
-            return 4096;
-        }
-
-        @Override
-        public boolean isHeaderCacheCaseSensitive()
-        {
-            return false;
-        }
     }
 
     public static final String CONTENT = "The quick brown fox jumped over the lazy dog.\nNow is the time for all good men to come to the aid of the party\nThe moon is blue to a fish in love.\n";

jetty-http2/http2-client/pom.xml

@@ -20,7 +20,7 @@
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
           <argLine>
-            @{argLine} ${jetty.surefire.argLine} --add-reads org.eclipse.jetty.http2.client=jetty.servlet.api --add-modules jetty.servlet.api
+            @{argLine} ${jetty.surefire.argLine} --add-reads org.eclipse.jetty.http2.client=jetty.servlet.api,org.eclipse.jetty.http2.hpack --add-modules jetty.servlet.api
           </argLine>
         </configuration>
       </plugin>

jetty-http2/http2-client/src/main/java/org/eclipse/jetty/http2/client/HTTP2ClientConnectionFactory.java

@@ -73,7 +73,7 @@ public class HTTP2ClientConnectionFactory implements ClientConnectionFactory
 
         final HTTP2ClientConnection connection = new HTTP2ClientConnection(client, byteBufferPool, executor, endPoint,
             parser, session, client.getInputBufferSize(), promise, listener);
-        connection.addListener(connectionListener);
+        connection.addEventListener(connectionListener);
         return customize(connection, context);
     }
 

jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/DataDemandTest.java

@@ -0,0 +1,386 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.http2.client;
+
+import java.nio.ByteBuffer;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.eclipse.jetty.http.HttpFields;
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.http.MetaData;
+import org.eclipse.jetty.http2.FlowControlStrategy;
+import org.eclipse.jetty.http2.HTTP2Session;
+import org.eclipse.jetty.http2.ISession;
+import org.eclipse.jetty.http2.api.Session;
+import org.eclipse.jetty.http2.api.Stream;
+import org.eclipse.jetty.http2.api.server.ServerSessionListener;
+import org.eclipse.jetty.http2.frames.DataFrame;
+import org.eclipse.jetty.http2.frames.HeadersFrame;
+import org.eclipse.jetty.http2.generator.Generator;
+import org.eclipse.jetty.io.ByteBufferPool;
+import org.eclipse.jetty.io.MappedByteBufferPool;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.FuturePromise;
+import org.eclipse.jetty.util.Promise;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+public class DataDemandTest extends AbstractTest
+{
+    @Test
+    public void testExplicitDemand() throws Exception
+    {
+        int length = FlowControlStrategy.DEFAULT_WINDOW_SIZE - 1;
+        AtomicReference<Stream> serverStreamRef = new AtomicReference<>();
+        Queue<DataFrame> serverQueue = new ConcurrentLinkedQueue<>();
+        start(new ServerSessionListener.Adapter()
+        {
+            @Override
+            public Stream.Listener onNewStream(Stream stream, HeadersFrame frame)
+            {
+                serverStreamRef.set(stream);
+                return new Stream.Listener.Adapter()
+                {
+                    @Override
+                    public void onDataDemanded(Stream stream, DataFrame frame, Callback callback)
+                    {
+                        // Don't demand and don't complete callbacks.
+                        serverQueue.offer(frame);
+                    }
+                };
+            }
+        });
+
+        Session client = newClient(new Session.Listener.Adapter());
+        MetaData.Request post = newRequest("POST", new HttpFields());
+        FuturePromise<Stream> promise = new FuturePromise<>();
+        Queue<DataFrame> clientQueue = new ConcurrentLinkedQueue<>();
+        client.newStream(new HeadersFrame(post, null, false), promise, new Stream.Listener.Adapter()
+        {
+            @Override
+            public void onDataDemanded(Stream stream, DataFrame frame, Callback callback)
+            {
+                clientQueue.offer(frame);
+            }
+        });
+        Stream clientStream = promise.get(5, TimeUnit.SECONDS);
+        // Send a single frame larger than the default frame size,
+        // so that it will be split on the server in multiple frames.
+        clientStream.data(new DataFrame(clientStream.getId(), ByteBuffer.allocate(length), true), Callback.NOOP);
+
+        // The server should receive only 1 DATA frame because it does explicit demand.
+        // Wait a bit more to be sure it only receives 1 DATA frame.
+        Thread.sleep(1000);
+        assertEquals(1, serverQueue.size());
+
+        Stream serverStream = serverStreamRef.get();
+        assertNotNull(serverStream);
+
+        // Demand more DATA frames.
+        int count = 2;
+        serverStream.demand(count);
+        Thread.sleep(1000);
+        // The server should have received `count` more DATA frames.
+        assertEquals(1 + count, serverQueue.size());
+
+        // Demand all the rest.
+        serverStream.demand(Long.MAX_VALUE);
+        int loops = 0;
+        while (true)
+        {
+            if (++loops > 100)
+                fail();
+
+            Thread.sleep(100);
+
+            long sum = serverQueue.stream()
+                .mapToLong(frame -> frame.getData().remaining())
+                .sum();
+            if (sum == length)
+                break;
+        }
+
+        // Even if demanded, the flow control window should not have
+        // decreased because the callbacks have not been completed.
+        int recvWindow = ((ISession)serverStream.getSession()).updateRecvWindow(0);
+        assertEquals(FlowControlStrategy.DEFAULT_WINDOW_SIZE - length, recvWindow);
+
+        // Send a large DATA frame to the client.
+        serverStream.data(new DataFrame(serverStream.getId(), ByteBuffer.allocate(length), true), Callback.NOOP);
+
+
+        // The client should receive only 1 DATA frame because it does explicit demand.
+        // Wait a bit more to be sure it only receives 1 DATA frame.
+        Thread.sleep(1000);
+        assertEquals(1, clientQueue.size());
+
+        // Demand more DATA frames.
+        clientStream.demand(count);
+        Thread.sleep(1000);
+        // The client should have received `count` more DATA frames.
+        assertEquals(1 + count, clientQueue.size());
+
+        // Demand all the rest.
+        clientStream.demand(Long.MAX_VALUE);
+        loops = 0;
+        while (true)
+        {
+            if (++loops > 100)
+                fail();
+
+            Thread.sleep(100);
+
+            long sum = clientQueue.stream()
+                .mapToLong(frame -> frame.getData().remaining())
+                .sum();
+            if (sum == length)
+                break;
+        }
+
+        // Both the client and server streams should be gone now.
+        assertNull(clientStream.getSession().getStream(clientStream.getId()));
+        assertNull(serverStream.getSession().getStream(serverStream.getId()));
+    }
+
+    @Test
+    public void testOnBeforeData() throws Exception
+    {
+        start(new ServerSessionListener.Adapter()
+        {
+            @Override
+            public Stream.Listener onNewStream(Stream stream, HeadersFrame frame)
+            {
+                MetaData.Response response = new MetaData.Response(HttpVersion.HTTP_2, HttpStatus.OK_200, new HttpFields());
+                stream.headers(new HeadersFrame(stream.getId(), response, null, false), Callback.from(() -> sendData(stream), x -> {}));
+                return null;
+            }
+
+            private void sendData(Stream stream)
+            {
+                stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(1024 * 1024), true), Callback.NOOP);
+            }
+        });
+
+        Session client = newClient(new Session.Listener.Adapter());
+        MetaData.Request post = newRequest("GET", new HttpFields());
+        FuturePromise<Stream> promise = new FuturePromise<>();
+        CountDownLatch responseLatch = new CountDownLatch(1);
+        CountDownLatch beforeDataLatch = new CountDownLatch(1);
+        CountDownLatch latch = new CountDownLatch(1);
+        client.newStream(new HeadersFrame(post, null, true), promise, new Stream.Listener.Adapter()
+        {
+            @Override
+            public void onHeaders(Stream stream, HeadersFrame frame)
+            {
+                MetaData.Response response = (MetaData.Response)frame.getMetaData();
+                assertEquals(HttpStatus.OK_200, response.getStatus());
+                responseLatch.countDown();
+            }
+
+            @Override
+            public void onBeforeData(Stream stream)
+            {
+                beforeDataLatch.countDown();
+                // Don't demand.
+            }
+
+            @Override
+            public void onData(Stream stream, DataFrame frame, Callback callback)
+            {
+                callback.succeeded();
+                if (frame.isEndStream())
+                    latch.countDown();
+            }
+        });
+        Stream clientStream = promise.get(5, TimeUnit.SECONDS);
+        assertTrue(responseLatch.await(5, TimeUnit.SECONDS));
+        assertTrue(beforeDataLatch.await(5, TimeUnit.SECONDS));
+        // Should not receive DATA frames until demanded.
+        assertFalse(latch.await(1, TimeUnit.SECONDS));
+        // Now demand the first DATA frame.
+        clientStream.demand(1);
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testDemandFromOnHeaders() throws Exception
+    {
+        start(new ServerSessionListener.Adapter()
+        {
+            @Override
+            public Stream.Listener onNewStream(Stream stream, HeadersFrame frame)
+            {
+                MetaData.Response response = new MetaData.Response(HttpVersion.HTTP_2, HttpStatus.OK_200, new HttpFields());
+                stream.headers(new HeadersFrame(stream.getId(), response, null, false), Callback.from(() -> sendData(stream), x -> {}));
+                return null;
+            }
+
+            private void sendData(Stream stream)
+            {
+                stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(1024 * 1024), true), Callback.NOOP);
+            }
+        });
+
+        Session client = newClient(new Session.Listener.Adapter());
+        MetaData.Request post = newRequest("GET", new HttpFields());
+        CountDownLatch latch = new CountDownLatch(1);
+        client.newStream(new HeadersFrame(post, null, true), new Promise.Adapter<>(), new Stream.Listener.Adapter()
+        {
+            @Override
+            public void onHeaders(Stream stream, HeadersFrame frame)
+            {
+                stream.demand(1);
+            }
+
+            @Override
+            public void onBeforeData(Stream stream)
+            {
+                // Do not demand from here, we have already demanded in onHeaders().
+            }
+
+            @Override
+            public void onData(Stream stream, DataFrame frame, Callback callback)
+            {
+                callback.succeeded();
+                if (frame.isEndStream())
+                    latch.countDown();
+            }
+        });
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testOnBeforeDataDoesNotReenter() throws Exception
+    {
+        start(new ServerSessionListener.Adapter()
+        {
+            @Override
+            public Stream.Listener onNewStream(Stream stream, HeadersFrame frame)
+            {
+                MetaData.Response response = new MetaData.Response(HttpVersion.HTTP_2, HttpStatus.OK_200, new HttpFields());
+                stream.headers(new HeadersFrame(stream.getId(), response, null, false), Callback.from(() -> sendData(stream), x -> {}));
+                return null;
+            }
+
+            private void sendData(Stream stream)
+            {
+                stream.data(new DataFrame(stream.getId(), ByteBuffer.allocate(1024 * 1024), true), Callback.NOOP);
+            }
+        });
+
+        Session client = newClient(new Session.Listener.Adapter());
+        MetaData.Request post = newRequest("GET", new HttpFields());
+        CountDownLatch latch = new CountDownLatch(1);
+        client.newStream(new HeadersFrame(post, null, true), new Promise.Adapter<>(), new Stream.Listener.Adapter()
+        {
+            private boolean inBeforeData;
+
+            @Override
+            public void onBeforeData(Stream stream)
+            {
+                inBeforeData = true;
+                stream.demand(1);
+                inBeforeData = false;
+            }
+
+            @Override
+            public void onData(Stream stream, DataFrame frame, Callback callback)
+            {
+                assertFalse(inBeforeData);
+                callback.succeeded();
+                if (frame.isEndStream())
+                    latch.countDown();
+            }
+        });
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testSynchronousDemandDoesNotStackOverflow() throws Exception
+    {
+        start(new ServerSessionListener.Adapter()
+        {
+            @Override
+            public Stream.Listener onNewStream(Stream stream, HeadersFrame frame)
+            {
+                return new Stream.Listener.Adapter()
+                {
+                    @Override
+                    public void onDataDemanded(Stream stream, DataFrame frame, Callback callback)
+                    {
+                        callback.succeeded();
+                        stream.demand(1);
+                        if (frame.isEndStream())
+                        {
+                            MetaData.Response response = new MetaData.Response(HttpVersion.HTTP_2, HttpStatus.OK_200, new HttpFields());
+                            stream.headers(new HeadersFrame(stream.getId(), response, null, true), Callback.NOOP);
+                        }
+                    }
+                };
+            }
+        });
+
+        Session client = newClient(new Session.Listener.Adapter());
+        MetaData.Request post = newRequest("POST", new HttpFields());
+        FuturePromise<Stream> promise = new FuturePromise<>();
+        CountDownLatch latch = new CountDownLatch(1);
+        client.newStream(new HeadersFrame(post, null, false), promise, new Stream.Listener.Adapter()
+        {
+            @Override
+            public void onHeaders(Stream stream, HeadersFrame frame)
+            {
+                if (frame.isEndStream())
+                {
+                    MetaData.Response response = (MetaData.Response)frame.getMetaData();
+                    assertEquals(HttpStatus.OK_200, response.getStatus());
+                    latch.countDown();
+                }
+            }
+        });
+        Stream clientStream = promise.get(5, TimeUnit.SECONDS);
+
+        // Generate a lot of small DATA frames and write them in a single
+        // write so that the server will continuously be notified and demand,
+        // which will test that it won't throw StackOverflowError.
+        MappedByteBufferPool byteBufferPool = new MappedByteBufferPool();
+        Generator generator = new Generator(byteBufferPool);
+        ByteBufferPool.Lease lease = new ByteBufferPool.Lease(byteBufferPool);
+        for (int i = 512; i >= 0; --i)
+            generator.data(lease, new DataFrame(clientStream.getId(), ByteBuffer.allocate(1), i == 0), 1);
+
+        // Since this is a naked write, we need to wait that the
+        // client finishes writing the SETTINGS reply to the server
+        // during connection initialization, or we risk a WritePendingException.
+        Thread.sleep(1000);
+        ((HTTP2Session)clientStream.getSession()).getEndPoint().write(Callback.NOOP, lease.getByteBuffers().toArray(new ByteBuffer[0]));
+
+        assertTrue(latch.await(15, TimeUnit.SECONDS));
+    }
+}

jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/HTTP2Test.java

@@ -26,6 +26,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
@@ -44,7 +45,9 @@ import org.eclipse.jetty.http2.api.server.ServerSessionListener;
 import org.eclipse.jetty.http2.frames.DataFrame;
 import org.eclipse.jetty.http2.frames.GoAwayFrame;
 import org.eclipse.jetty.http2.frames.HeadersFrame;
+import org.eclipse.jetty.http2.frames.ResetFrame;
 import org.eclipse.jetty.http2.frames.SettingsFrame;
+import org.eclipse.jetty.http2.hpack.HpackException;
 import org.eclipse.jetty.http2.parser.RateControl;
 import org.eclipse.jetty.http2.parser.ServerParser;
 import org.eclipse.jetty.http2.server.RawHTTP2ServerConnectionFactory;
@@ -58,8 +61,12 @@ import org.eclipse.jetty.util.Jetty;
 import org.eclipse.jetty.util.Promise;
 import org.junit.jupiter.api.Test;
 
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class HTTP2Test extends AbstractTest
@@ -792,6 +799,100 @@ public class HTTP2Test extends AbstractTest
         assertTrue(goAwayLatch.await(5, TimeUnit.SECONDS));
     }
 
+    @Test
+    public void testClientInvalidHeader() throws Exception
+    {
+        start(new EmptyHttpServlet());
+
+        // A bad header in the request should fail on the client.
+        Session session = newClient(new Session.Listener.Adapter());
+        HttpFields requestFields = new HttpFields();
+        requestFields.put(":custom", "special");
+        MetaData.Request metaData = newRequest("GET", requestFields);
+        HeadersFrame request = new HeadersFrame(metaData, null, true);
+        FuturePromise<Stream> promise = new FuturePromise<>();
+        session.newStream(request, promise, new Stream.Listener.Adapter());
+        ExecutionException x = assertThrows(ExecutionException.class, () -> promise.get(5, TimeUnit.SECONDS));
+        assertThat(x.getCause(), instanceOf(HpackException.StreamException.class));
+    }
+
+    @Test
+    public void testServerInvalidHeader() throws Exception
+    {
+        start(new EmptyHttpServlet()
+        {
+            @Override
+            protected void service(HttpServletRequest request, HttpServletResponse response)
+            {
+                response.setHeader(":custom", "special");
+            }
+        });
+
+        // Good request with bad header in the response.
+        Session session = newClient(new Session.Listener.Adapter());
+        MetaData.Request metaData = newRequest("GET", new HttpFields());
+        HeadersFrame request = new HeadersFrame(metaData, null, true);
+        FuturePromise<Stream> promise = new FuturePromise<>();
+        CountDownLatch resetLatch = new CountDownLatch(1);
+        session.newStream(request, promise, new Stream.Listener.Adapter()
+        {
+            @Override
+            public void onReset(Stream stream, ResetFrame frame)
+            {
+                resetLatch.countDown();
+            }
+        });
+        Stream stream = promise.get(5, TimeUnit.SECONDS);
+        assertNotNull(stream);
+
+        assertTrue(resetLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testServerInvalidHeaderFlushed() throws Exception
+    {
+        CountDownLatch serverFailure = new CountDownLatch(1);
+        start(new EmptyHttpServlet()
+        {
+            @Override
+            protected void service(HttpServletRequest request, HttpServletResponse response) throws IOException
+            {
+                response.setHeader(":custom", "special");
+                try
+                {
+                    response.flushBuffer();
+                }
+                catch (IOException x)
+                {
+                    assertThat(x.getCause(), instanceOf(HpackException.StreamException.class));
+                    serverFailure.countDown();
+                    throw x;
+                }
+            }
+        });
+
+        // Good request with bad header in the response.
+        Session session = newClient(new Session.Listener.Adapter());
+        MetaData.Request metaData = newRequest("GET", "/flush", new HttpFields());
+        HeadersFrame request = new HeadersFrame(metaData, null, true);
+        FuturePromise<Stream> promise = new FuturePromise<>();
+        CountDownLatch resetLatch = new CountDownLatch(1);
+        session.newStream(request, promise, new Stream.Listener.Adapter()
+        {
+            @Override
+            public void onReset(Stream stream, ResetFrame frame)
+            {
+                // Cannot receive a 500 because we force the flush on the server, so
+                // the response is committed even if the server was not able to write it.
+                resetLatch.countDown();
+            }
+        });
+        Stream stream = promise.get(5, TimeUnit.SECONDS);
+        assertNotNull(stream);
+        assertTrue(serverFailure.await(5, TimeUnit.SECONDS));
+        assertTrue(resetLatch.await(5, TimeUnit.SECONDS));
+    }
+
     private static void sleep(long time)
     {
         try

jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/TrailersTest.java

@@ -37,6 +37,7 @@ import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.http.MetaData;
+import org.eclipse.jetty.http2.HTTP2Session;
 import org.eclipse.jetty.http2.api.Session;
 import org.eclipse.jetty.http2.api.Stream;
 import org.eclipse.jetty.http2.api.server.ServerSessionListener;
@@ -298,7 +299,34 @@ public class TrailersTest extends AbstractTest
     }
 
     @Test
-    public void testRequestTrailerInvalidHpack() throws Exception
+    public void testRequestTrailerInvalidHpackSent() throws Exception
+    {
+        start(new EmptyHttpServlet());
+
+        Session session = newClient(new Session.Listener.Adapter());
+        MetaData.Request request = newRequest("POST", new HttpFields());
+        HeadersFrame requestFrame = new HeadersFrame(request, null, false);
+        FuturePromise<Stream> promise = new FuturePromise<>();
+        session.newStream(requestFrame, promise, new Stream.Listener.Adapter());
+        Stream stream = promise.get(5, TimeUnit.SECONDS);
+        ByteBuffer data = ByteBuffer.wrap(StringUtil.getUtf8Bytes("hello"));
+        Callback.Completable completable = new Callback.Completable();
+        stream.data(new DataFrame(stream.getId(), data, false), completable);
+        CountDownLatch failureLatch = new CountDownLatch(1);
+        completable.thenRun(() ->
+        {
+            // Invalid trailer: cannot contain pseudo headers.
+            HttpFields trailerFields = new HttpFields();
+            trailerFields.put(HttpHeader.C_METHOD, "GET");
+            MetaData trailer = new MetaData(HttpVersion.HTTP_2, trailerFields);
+            HeadersFrame trailerFrame = new HeadersFrame(stream.getId(), trailer, null, true);
+            stream.headers(trailerFrame, Callback.from(Callback.NOOP::succeeded, x -> failureLatch.countDown()));
+        });
+        assertTrue(failureLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testRequestTrailerInvalidHpackReceived() throws Exception
     {
         CountDownLatch serverLatch = new CountDownLatch(1);
         start(new HttpServlet()
@@ -344,6 +372,8 @@ public class TrailersTest extends AbstractTest
         stream.data(new DataFrame(stream.getId(), data, false), completable);
         completable.thenRun(() ->
         {
+            // Disable checks for invalid headers.
+            ((HTTP2Session)session).getGenerator().setValidateHpackEncoding(false);
             // Invalid trailer: cannot contain pseudo headers.
             HttpFields trailerFields = new HttpFields();
             trailerFields.put(HttpHeader.C_METHOD, "GET");

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Connection.java

@@ -56,6 +56,8 @@ public class HTTP2Connection extends AbstractConnection implements WriteFlusher.
     private final ISession session;
     private final int bufferSize;
     private final ExecutionStrategy strategy;
+    private boolean useInputDirectByteBuffers;
+    private boolean useOutputDirectByteBuffers;
 
     public HTTP2Connection(ByteBufferPool byteBufferPool, Executor executor, EndPoint endPoint, Parser parser, ISession session, int bufferSize)
     {
@@ -99,6 +101,26 @@ public class HTTP2Connection extends AbstractConnection implements WriteFlusher.
             producer.setInputBuffer(buffer);
     }
 
+    public boolean isUseInputDirectByteBuffers()
+    {
+        return useInputDirectByteBuffers;
+    }
+
+    public void setUseInputDirectByteBuffers(boolean useInputDirectByteBuffers)
+    {
+        this.useInputDirectByteBuffers = useInputDirectByteBuffers;
+    }
+
+    public boolean isUseOutputDirectByteBuffers()
+    {
+        return useOutputDirectByteBuffers;
+    }
+
+    public void setUseOutputDirectByteBuffers(boolean useOutputDirectByteBuffers)
+    {
+        this.useOutputDirectByteBuffers = useOutputDirectByteBuffers;
+    }
+
     @Override
     public void onOpen()
     {
@@ -309,7 +331,7 @@ public class HTTP2Connection extends AbstractConnection implements WriteFlusher.
             if (currentBuffer == null)
                 throw new IllegalStateException();
 
-            if (currentBuffer.getBuffer().hasRemaining())
+            if (currentBuffer.hasRemaining())
                 throw new IllegalStateException();
 
             currentBuffer.release();
@@ -389,7 +411,7 @@ public class HTTP2Connection extends AbstractConnection implements WriteFlusher.
     {
         private NetworkBuffer()
         {
-            super(byteBufferPool, bufferSize, false);
+            super(byteBufferPool, bufferSize, isUseInputDirectByteBuffers());
         }
 
         private void put(ByteBuffer source)

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Flusher.java

@@ -30,6 +30,7 @@ import java.util.Set;
 
 import org.eclipse.jetty.http2.frames.Frame;
 import org.eclipse.jetty.http2.frames.WindowUpdateFrame;
+import org.eclipse.jetty.http2.hpack.HpackException;
 import org.eclipse.jetty.io.ByteBufferPool;
 import org.eclipse.jetty.io.EofException;
 import org.eclipse.jetty.util.Callback;
@@ -207,6 +208,13 @@ public class HTTP2Flusher extends IteratingCallback implements Dumpable
                         }
                     }
                 }
+                catch (HpackException.StreamException failure)
+                {
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("Failure generating " + entry, failure);
+                    entry.failed(failure);
+                    pending.remove();
+                }
                 catch (Throwable failure)
                 {
                     // Failure to generate the entry is catastrophic.
@@ -397,7 +405,7 @@ public class HTTP2Flusher extends IteratingCallback implements Dumpable
             return 0;
         }
 
-        protected abstract boolean generate(ByteBufferPool.Lease lease);
+        protected abstract boolean generate(ByteBufferPool.Lease lease) throws HpackException;
 
         public abstract long onFlushed(long bytes) throws IOException;
 

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Session.java

@@ -52,6 +52,7 @@ import org.eclipse.jetty.http2.frames.ResetFrame;
 import org.eclipse.jetty.http2.frames.SettingsFrame;
 import org.eclipse.jetty.http2.frames.WindowUpdateFrame;
 import org.eclipse.jetty.http2.generator.Generator;
+import org.eclipse.jetty.http2.hpack.HpackException;
 import org.eclipse.jetty.http2.parser.Parser;
 import org.eclipse.jetty.io.ByteBufferPool;
 import org.eclipse.jetty.io.EndPoint;
@@ -62,7 +63,6 @@ import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.CountingCallback;
 import org.eclipse.jetty.util.MathUtils;
 import org.eclipse.jetty.util.Promise;
-import org.eclipse.jetty.util.Retainable;
 import org.eclipse.jetty.util.annotation.ManagedAttribute;
 import org.eclipse.jetty.util.annotation.ManagedObject;
 import org.eclipse.jetty.util.component.ContainerLifeCycle;
@@ -1236,7 +1236,7 @@ public abstract class HTTP2Session extends ContainerLifeCycle implements ISessio
         }
 
         @Override
-        protected boolean generate(ByteBufferPool.Lease lease)
+        protected boolean generate(ByteBufferPool.Lease lease) throws HpackException
         {
             frameBytes = generator.control(lease, frame);
             beforeSend();
@@ -1482,7 +1482,7 @@ public abstract class HTTP2Session extends ContainerLifeCycle implements ISessio
         }
     }
 
-    private class DataCallback extends Callback.Nested implements Retainable
+    private class DataCallback extends Callback.Nested
     {
         private final IStream stream;
         private final int flowControlLength;
@@ -1494,14 +1494,6 @@ public abstract class HTTP2Session extends ContainerLifeCycle implements ISessio
             this.flowControlLength = flowControlLength;
         }
 
-        @Override
-        public void retain()
-        {
-            Callback callback = getCallback();
-            if (callback instanceof Retainable)
-                ((Retainable)callback).retain();
-        }
-
         @Override
         public void succeeded()
         {

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Stream.java

@@ -21,6 +21,8 @@ package org.eclipse.jetty.http2;
 import java.io.EOFException;
 import java.io.IOException;
 import java.nio.channels.WritePendingException;
+import java.util.ArrayDeque;
+import java.util.Queue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
@@ -42,16 +44,20 @@ import org.eclipse.jetty.http2.frames.ResetFrame;
 import org.eclipse.jetty.http2.frames.WindowUpdateFrame;
 import org.eclipse.jetty.io.IdleTimeout;
 import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.MathUtils;
 import org.eclipse.jetty.util.Promise;
 import org.eclipse.jetty.util.component.Dumpable;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
+import org.eclipse.jetty.util.thread.AutoLock;
 import org.eclipse.jetty.util.thread.Scheduler;
 
 public class HTTP2Stream extends IdleTimeout implements IStream, Callback, Dumpable
 {
     private static final Logger LOG = Log.getLogger(HTTP2Stream.class);
 
+    private final AutoLock lock = new AutoLock();
+    private final Queue<DataEntry> dataQueue = new ArrayDeque<>();
     private final AtomicReference<Object> attachment = new AtomicReference<>();
     private final AtomicReference<ConcurrentMap<String, Object>> attributes = new AtomicReference<>();
     private final AtomicReference<CloseState> closeState = new AtomicReference<>(CloseState.NOT_CLOSED);
@@ -67,6 +73,9 @@ public class HTTP2Stream extends IdleTimeout implements IStream, Callback, Dumpa
     private Listener listener;
     private boolean remoteReset;
     private long dataLength;
+    private long dataDemand;
+    private boolean dataInitial;
+    private boolean dataProcess;
 
     public HTTP2Stream(Scheduler scheduler, ISession session, int streamId, MetaData.Request request, boolean local)
     {
@@ -76,6 +85,7 @@ public class HTTP2Stream extends IdleTimeout implements IStream, Callback, Dumpa
         this.request = request;
         this.local = local;
         this.dataLength = Long.MIN_VALUE;
+        this.dataInitial = true;
     }
 
     @Override
@@ -343,10 +353,88 @@ public class HTTP2Stream extends IdleTimeout implements IStream, Callback, Dumpa
             }
         }
 
+        boolean initial;
+        boolean proceed = false;
+        DataEntry entry = new DataEntry(frame, callback);
+        try (AutoLock l = lock.lock())
+        {
+            dataQueue.offer(entry);
+            initial = dataInitial;
+            if (initial)
+            {
+                dataInitial = false;
+                // Fake that we are processing data so we return
+                // from onBeforeData() before calling onData().
+                dataProcess = true;
+            }
+            else if (!dataProcess)
+            {
+                dataProcess = proceed = dataDemand > 0;
+            }
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("{} data processing of {} for {}", initial ? "Starting" : proceed ? "Proceeding" : "Stalling", frame, this);
+        if (initial)
+        {
+            notifyBeforeData(this);
+            try (AutoLock l = lock.lock())
+            {
+                dataProcess = proceed = dataDemand > 0;
+            }
+        }
+        if (proceed)
+            processData();
+    }
+
+    @Override
+    public void demand(long n)
+    {
+        if (n <= 0)
+            throw new IllegalArgumentException("Invalid demand " + n);
+        long demand;
+        boolean proceed = false;
+        try (AutoLock l = lock.lock())
+        {
+            demand = dataDemand = MathUtils.cappedAdd(dataDemand, n);
+            if (!dataProcess)
+                dataProcess = proceed = !dataQueue.isEmpty();
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("Demand {}/{}, {} data processing for {}", n, demand, proceed ? "proceeding" : "stalling", this);
+        if (proceed)
+            processData();
+    }
+
+    private void processData()
+    {
+        while (true)
+        {
+            DataEntry dataEntry;
+            try (AutoLock l = lock.lock())
+            {
+                if (dataQueue.isEmpty() || dataDemand == 0)
+                {
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("Stalling data processing for {}", this);
+                    dataProcess = false;
+                    return;
+                }
+                --dataDemand;
+                dataEntry = dataQueue.poll();
+            }
+            DataFrame frame = dataEntry.frame;
             if (updateClose(frame.isEndStream(), CloseState.Event.RECEIVED))
                 session.removeStream(this);
+            notifyDataDemanded(this, frame, dataEntry.callback);
+        }
+    }
 
-        notifyData(this, frame, callback);
+    private long demand()
+    {
+        try (AutoLock l = lock.lock())
+        {
+            return dataDemand;
+        }
     }
 
     private void onReset(ResetFrame frame, Callback callback)
@@ -573,14 +661,34 @@ public class HTTP2Stream extends IdleTimeout implements IStream, Callback, Dumpa
         }
     }
 
-    private void notifyData(Stream stream, DataFrame frame, Callback callback)
+    private void notifyBeforeData(Stream stream)
     {
         Listener listener = this.listener;
         if (listener != null)
         {
             try
             {
-                listener.onData(stream, frame, callback);
+                listener.onBeforeData(stream);
+            }
+            catch (Throwable x)
+            {
+                LOG.info("Failure while notifying listener " + listener, x);
+            }
+        }
+        else
+        {
+            stream.demand(1);
+        }
+    }
+
+    private void notifyDataDemanded(Stream stream, DataFrame frame, Callback callback)
+    {
+        Listener listener = this.listener;
+        if (listener != null)
+        {
+            try
+            {
+                listener.onDataDemanded(stream, frame, callback);
             }
             catch (Throwable x)
             {
@@ -591,6 +699,7 @@ public class HTTP2Stream extends IdleTimeout implements IStream, Callback, Dumpa
         else
         {
             callback.succeeded();
+            stream.demand(1);
         }
     }
 
@@ -682,16 +791,29 @@ public class HTTP2Stream extends IdleTimeout implements IStream, Callback, Dumpa
     @Override
     public String toString()
     {
-        return String.format("%s@%x#%d{sendWindow=%s,recvWindow=%s,reset=%b/%b,%s,age=%d,attachment=%s}",
+        return String.format("%s@%x#%d{sendWindow=%s,recvWindow=%s,demand=%d,reset=%b/%b,%s,age=%d,attachment=%s}",
             getClass().getSimpleName(),
             hashCode(),
             getId(),
             sendWindow,
             recvWindow,
+            demand(),
             localReset,
             remoteReset,
             closeState,
             TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - timeStamp),
             attachment);
     }
+
+    private static class DataEntry
+    {
+        private final DataFrame frame;
+        private final Callback callback;
+
+        private DataEntry(DataFrame frame, Callback callback)
+        {
+            this.frame = frame;
+            this.callback = callback;
+        }
+    }
 }

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2StreamEndPoint.java

@@ -493,12 +493,6 @@ public abstract class HTTP2StreamEndPoint implements EndPoint
             LOG.debug("onClose {}", this);
     }
 
-    @Override
-    public boolean isOptimizedForDirectBuffers()
-    {
-        return true;
-    }
-
     @Override
     public void upgrade(Connection newConnection)
     {

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/api/Stream.java

@@ -29,7 +29,7 @@ import org.eclipse.jetty.util.Promise;
  * <p>A {@link Stream} represents a bidirectional exchange of data on top of a {@link Session}.</p>
  * <p>Differently from socket streams, where the input and output streams are permanently associated
  * with the socket (and hence with the connection that the socket represents), there can be multiple
- * HTTP/2 streams present concurrent for an HTTP/2 session.</p>
+ * HTTP/2 streams present concurrently for an HTTP/2 session.</p>
  * <p>A {@link Stream} maps to an HTTP request/response cycle, and after the request/response cycle is
  * completed, the stream is closed and removed from the session.</p>
  * <p>Like {@link Session}, {@link Stream} is the active part and by calling its API applications
@@ -129,9 +129,25 @@ public interface Stream
      */
     public void setIdleTimeout(long idleTimeout);
 
+    /**
+     * <p>Demands {@code n} more {@code DATA} frames for this stream.</p>
+     *
+     * @param n the increment of the demand, must be greater than zero
+     * @see Listener#onDataDemanded(Stream, DataFrame, Callback)
+     */
+    public void demand(long n);
+
     /**
      * <p>A {@link Stream.Listener} is the passive counterpart of a {@link Stream} and receives
      * events happening on an HTTP/2 stream.</p>
+     * <p>HTTP/2 data is flow controlled - this means that only a finite number of data events
+     * are delivered, until the flow control window is exhausted.</p>
+     * <p>Applications control the delivery of data events by requesting them via
+     * {@link Stream#demand(long)}; the first event is always delivered, while subsequent
+     * events must be explicitly demanded.</p>
+     * <p>Applications control the HTTP/2 flow control by completing the callback associated
+     * with data events - this allows the implementation to recycle the data buffer and
+     * eventually to enlarge the flow control window so that the sender can send more data.</p>
      *
      * @see Stream
      */
@@ -164,15 +180,42 @@ public interface Stream
          */
         public Listener onPush(Stream stream, PushPromiseFrame frame);
 
+        /**
+         * <p>Callback method invoked before notifying the first DATA frame.</p>
+         * <p>The default implementation initializes the demand for DATA frames.</p>
+         *
+         * @param stream the stream
+         */
+        public default void onBeforeData(Stream stream)
+        {
+            stream.demand(1);
+        }
+
         /**
          * <p>Callback method invoked when a DATA frame has been received.</p>
          *
          * @param stream the stream
          * @param frame the DATA frame received
          * @param callback the callback to complete when the bytes of the DATA frame have been consumed
+         * @see #onDataDemanded(Stream, DataFrame, Callback)
          */
         public void onData(Stream stream, DataFrame frame, Callback callback);
 
+        /**
+         * <p>Callback method invoked when a DATA frame has been demanded.</p>
+         * <p>Implementations of this method must arrange to call (within the
+         * method or otherwise asynchronously) {@link #demand(long)}.</p>
+         *
+         * @param stream the stream
+         * @param frame the DATA frame received
+         * @param callback the callback to complete when the bytes of the DATA frame have been consumed
+         */
+        public default void onDataDemanded(Stream stream, DataFrame frame, Callback callback)
+        {
+            onData(stream, frame, callback);
+            stream.demand(1);
+        }
+
         /**
          * <p>Callback method invoked when a RST_STREAM frame has been received for this stream.</p>
          *

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/generator/FrameGenerator.java

@@ -20,8 +20,11 @@ package org.eclipse.jetty.http2.generator;
 
 import java.nio.ByteBuffer;
 
+import org.eclipse.jetty.http.MetaData;
 import org.eclipse.jetty.http2.frames.Frame;
 import org.eclipse.jetty.http2.frames.FrameType;
+import org.eclipse.jetty.http2.hpack.HpackEncoder;
+import org.eclipse.jetty.http2.hpack.HpackException;
 import org.eclipse.jetty.io.ByteBufferPool;
 
 public abstract class FrameGenerator
@@ -33,7 +36,7 @@ public abstract class FrameGenerator
         this.headerGenerator = headerGenerator;
     }
 
-    public abstract int generate(ByteBufferPool.Lease lease, Frame frame);
+    public abstract int generate(ByteBufferPool.Lease lease, Frame frame) throws HpackException;
 
     protected ByteBuffer generateHeader(ByteBufferPool.Lease lease, FrameType frameType, int length, int flags, int streamId)
     {
@@ -44,4 +47,24 @@ public abstract class FrameGenerator
     {
         return headerGenerator.getMaxFrameSize();
     }
+
+    public boolean isUseDirectByteBuffers()
+    {
+        return headerGenerator.isUseDirectByteBuffers();
+    }
+
+    protected ByteBuffer encode(HpackEncoder encoder, ByteBufferPool.Lease lease, MetaData metaData, int maxFrameSize) throws HpackException
+    {
+        ByteBuffer hpacked = lease.acquire(maxFrameSize, isUseDirectByteBuffers());
+        try
+        {
+            encoder.encode(hpacked, metaData);
+            return hpacked;
+        }
+        catch (HpackException x)
+        {
+            lease.release(hpacked);
+            throw x;
+        }
+    }
 }

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/generator/Generator.java

@@ -22,6 +22,7 @@ import org.eclipse.jetty.http2.frames.DataFrame;
 import org.eclipse.jetty.http2.frames.Frame;
 import org.eclipse.jetty.http2.frames.FrameType;
 import org.eclipse.jetty.http2.hpack.HpackEncoder;
+import org.eclipse.jetty.http2.hpack.HpackException;
 import org.eclipse.jetty.io.ByteBufferPool;
 
 public class Generator
@@ -38,10 +39,15 @@ public class Generator
     }
 
     public Generator(ByteBufferPool byteBufferPool, int maxDynamicTableSize, int maxHeaderBlockFragment)
+    {
+        this(byteBufferPool, true, maxDynamicTableSize, maxHeaderBlockFragment);
+    }
+
+    public Generator(ByteBufferPool byteBufferPool, boolean useDirectByteBuffers, int maxDynamicTableSize, int maxHeaderBlockFragment)
     {
         this.byteBufferPool = byteBufferPool;
 
-        headerGenerator = new HeaderGenerator();
+        headerGenerator = new HeaderGenerator(useDirectByteBuffers);
         hpackEncoder = new HpackEncoder(maxDynamicTableSize);
 
         this.generators = new FrameGenerator[FrameType.values().length];
@@ -65,6 +71,11 @@ public class Generator
         return byteBufferPool;
     }
 
+    public void setValidateHpackEncoding(boolean validateEncoding)
+    {
+        hpackEncoder.setValidateEncoding(validateEncoding);
+    }
+
     public void setHeaderTableSize(int headerTableSize)
     {
         hpackEncoder.setRemoteMaxDynamicTableSize(headerTableSize);
@@ -75,7 +86,7 @@ public class Generator
         headerGenerator.setMaxFrameSize(maxFrameSize);
     }
 
-    public int control(ByteBufferPool.Lease lease, Frame frame)
+    public int control(ByteBufferPool.Lease lease, Frame frame) throws HpackException
     {
         return generators[frame.getType().getType()].generate(lease, frame);
     }

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/generator/HeaderGenerator.java

@@ -27,10 +27,26 @@ import org.eclipse.jetty.io.ByteBufferPool;
 public class HeaderGenerator
 {
     private int maxFrameSize = Frame.DEFAULT_MAX_LENGTH;
+    private final boolean useDirectByteBuffers;
+
+    public HeaderGenerator()
+    {
+        this(true);
+    }
+
+    public HeaderGenerator(boolean useDirectByteBuffers)
+    {
+        this.useDirectByteBuffers = useDirectByteBuffers;
+    }
+
+    public boolean isUseDirectByteBuffers()
+    {
+        return useDirectByteBuffers;
+    }
 
     public ByteBuffer generate(ByteBufferPool.Lease lease, FrameType frameType, int capacity, int length, int flags, int streamId)
     {
-        ByteBuffer header = lease.acquire(capacity, true);
+        ByteBuffer header = lease.acquire(capacity, isUseDirectByteBuffers());
         header.put((byte)((length & 0x00_FF_00_00) >>> 16));
         header.put((byte)((length & 0x00_00_FF_00) >>> 8));
         header.put((byte)((length & 0x00_00_00_FF)));

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/generator/HeadersGenerator.java

@@ -27,6 +27,7 @@ import org.eclipse.jetty.http2.frames.FrameType;
 import org.eclipse.jetty.http2.frames.HeadersFrame;
 import org.eclipse.jetty.http2.frames.PriorityFrame;
 import org.eclipse.jetty.http2.hpack.HpackEncoder;
+import org.eclipse.jetty.http2.hpack.HpackException;
 import org.eclipse.jetty.io.ByteBufferPool;
 import org.eclipse.jetty.util.BufferUtil;
 
@@ -50,13 +51,13 @@ public class HeadersGenerator extends FrameGenerator
     }
 
     @Override
-    public int generate(ByteBufferPool.Lease lease, Frame frame)
+    public int generate(ByteBufferPool.Lease lease, Frame frame) throws HpackException
     {
         HeadersFrame headersFrame = (HeadersFrame)frame;
         return generateHeaders(lease, headersFrame.getStreamId(), headersFrame.getMetaData(), headersFrame.getPriority(), headersFrame.isEndStream());
     }
 
-    public int generateHeaders(ByteBufferPool.Lease lease, int streamId, MetaData metaData, PriorityFrame priority, boolean endStream)
+    public int generateHeaders(ByteBufferPool.Lease lease, int streamId, MetaData metaData, PriorityFrame priority, boolean endStream) throws HpackException
     {
         if (streamId < 0)
             throw new IllegalArgumentException("Invalid stream id: " + streamId);
@@ -66,10 +67,7 @@ public class HeadersGenerator extends FrameGenerator
         if (priority != null)
             flags = Flags.PRIORITY;
 
-        int maxFrameSize = getMaxFrameSize();
-        ByteBuffer hpacked = lease.acquire(maxFrameSize, false);
-        BufferUtil.clearToFill(hpacked);
-        encoder.encode(hpacked, metaData);
+        ByteBuffer hpacked = encode(encoder, lease, metaData, getMaxFrameSize());
         int hpackedLength = hpacked.position();
         BufferUtil.flipToFlush(hpacked, 0);
 

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/generator/PushPromiseGenerator.java

@@ -26,6 +26,7 @@ import org.eclipse.jetty.http2.frames.Frame;
 import org.eclipse.jetty.http2.frames.FrameType;
 import org.eclipse.jetty.http2.frames.PushPromiseFrame;
 import org.eclipse.jetty.http2.hpack.HpackEncoder;
+import org.eclipse.jetty.http2.hpack.HpackException;
 import org.eclipse.jetty.io.ByteBufferPool;
 import org.eclipse.jetty.util.BufferUtil;
 
@@ -40,13 +41,13 @@ public class PushPromiseGenerator extends FrameGenerator
     }
 
     @Override
-    public int generate(ByteBufferPool.Lease lease, Frame frame)
+    public int generate(ByteBufferPool.Lease lease, Frame frame) throws HpackException
     {
         PushPromiseFrame pushPromiseFrame = (PushPromiseFrame)frame;
         return generatePushPromise(lease, pushPromiseFrame.getStreamId(), pushPromiseFrame.getPromisedStreamId(), pushPromiseFrame.getMetaData());
     }
 
-    public int generatePushPromise(ByteBufferPool.Lease lease, int streamId, int promisedStreamId, MetaData metaData)
+    public int generatePushPromise(ByteBufferPool.Lease lease, int streamId, int promisedStreamId, MetaData metaData) throws HpackException
     {
         if (streamId < 0)
             throw new IllegalArgumentException("Invalid stream id: " + streamId);
@@ -58,9 +59,7 @@ public class PushPromiseGenerator extends FrameGenerator
         int extraSpace = 4;
         maxFrameSize -= extraSpace;
 
-        ByteBuffer hpacked = lease.acquire(maxFrameSize, false);
-        BufferUtil.clearToFill(hpacked);
-        encoder.encode(hpacked, metaData);
+        ByteBuffer hpacked = encode(encoder, lease, metaData, maxFrameSize);
         int hpackedLength = hpacked.position();
         BufferUtil.flipToFlush(hpacked, 0);
 

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/parser/RateControl.java

@@ -18,6 +18,8 @@
 
 package org.eclipse.jetty.http2.parser;
 
+import org.eclipse.jetty.io.EndPoint;
+
 /**
  * Controls rate of events via {@link #onEvent(Object)}.
  */
@@ -36,4 +38,19 @@ public interface RateControl
      * @return true IFF the rate is within limits
      */
     public boolean onEvent(Object event);
+
+    /**
+     * Factory to create RateControl instances.
+     */
+    public interface Factory
+    {
+        /**
+         * @return a new RateControl instance for the given EndPoint
+         * @param endPoint the EndPoint for which the RateControl is created
+         */
+        public default RateControl newRateControl(EndPoint endPoint)
+        {
+            return NO_RATE_CONTROL;
+        }
+    }
 }

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/parser/SettingsBodyParser.java

@@ -83,6 +83,11 @@ public class SettingsBodyParser extends BodyParser
 
     @Override
     public boolean parse(ByteBuffer buffer)
+    {
+        return parse(buffer, getStreamId(), getBodyLength());
+    }
+
+    private boolean parse(ByteBuffer buffer, int streamId, int bodyLength)
     {
         while (buffer.hasRemaining())
         {
@@ -91,9 +96,9 @@ public class SettingsBodyParser extends BodyParser
                 case PREPARE:
                 {
                     // SPEC: wrong streamId is treated as connection error.
-                    if (getStreamId() != 0)
+                    if (streamId != 0)
                         return connectionFailure(buffer, ErrorCode.PROTOCOL_ERROR.code, "invalid_settings_frame");
-                    length = getBodyLength();
+                    length = bodyLength;
                     settings = new HashMap<>();
                     state = State.SETTING_ID;
                     break;
@@ -216,6 +221,13 @@ public class SettingsBodyParser extends BodyParser
         return true;
     }
 
+    /**
+     * <p>Parses the given buffer containing the whole body of a {@code SETTINGS} frame
+     * (without header bytes), typically from the {@code HTTP2-Settings} header.</p>
+     *
+     * @param buffer the buffer containing the body of {@code SETTINGS} frame
+     * @return the {@code SETTINGS} frame from the parsed body bytes
+     */
     public static SettingsFrame parseBody(final ByteBuffer buffer)
     {
         AtomicReference<SettingsFrame> frameRef = new AtomicReference<>();
@@ -234,7 +246,7 @@ public class SettingsBodyParser extends BodyParser
             }
         });
         if (buffer.hasRemaining())
-            parser.parse(buffer);
+            parser.parse(buffer, 0, buffer.remaining());
         else
             parser.emptyBody(buffer);
         return frameRef.get();