Merge remote-tracking branch 'origin/jetty-10.0.x' into jetty-10.0.x-WebSocketServlet

Jenkinsfile

@@ -8,45 +8,51 @@ pipeline {
     stage("Parallel Stage") {
       parallel {
         stage("Build / Test - JDK11") {
-          agent { node { label 'linux' } }
+          agent {
+            node { label 'linux' }
+          }
           options { timeout(time: 120, unit: 'MINUTES') }
           steps {
-            mavenBuild("jdk11", "-T3 -Pmongodb clean install", "maven3", true) // -Pautobahn
-            // Collect up the jacoco execution results (only on main build)
-            jacoco inclusionPattern: '**/org/eclipse/jetty/**/*.class',
-              exclusionPattern: '' +
-                // build tools
-                '**/org/eclipse/jetty/ant/**' +
-                ',**/org/eclipse/jetty/maven/**' +
-                ',**/org/eclipse/jetty/jspc/**' +
-                // example code / documentation
-                ',**/org/eclipse/jetty/embedded/**' +
-                ',**/org/eclipse/jetty/asyncrest/**' +
-                ',**/org/eclipse/jetty/demo/**' +
-                // special environments / late integrations
-                ',**/org/eclipse/jetty/gcloud/**' +
-                ',**/org/eclipse/jetty/infinispan/**' +
-                ',**/org/eclipse/jetty/osgi/**' +
-                ',**/org/eclipse/jetty/spring/**' +
-                ',**/org/eclipse/jetty/http/spi/**' +
-                // test classes
-                ',**/org/eclipse/jetty/tests/**' +
-                ',**/org/eclipse/jetty/test/**',
-              execPattern: '**/target/jacoco.exec',
-              classPattern: '**/target/classes',
-              sourcePattern: '**/src/main/java'
-            warnings consoleParsers: [[parserName: 'Maven'], [parserName: 'Java']]
-            junit testResults: '**/target/surefire-reports/*.xml,**/target/invoker-reports/TEST*.xml,**/target/autobahntestsuite-reports/*.xml'
+            container('jetty-build') {
+              mavenBuild("jdk11", "-T3 -Pmongodb clean install", "maven3", true) // -Pautobahn
+              // Collect up the jacoco execution results (only on main build)
+              jacoco inclusionPattern: '**/org/eclipse/jetty/**/*.class',
+                exclusionPattern: '' +
+                  // build tools
+                  '**/org/eclipse/jetty/ant/**' +
+                  ',**/org/eclipse/jetty/maven/**' +
+                  ',**/org/eclipse/jetty/jspc/**' +
+                  // example code / documentation
+                  ',**/org/eclipse/jetty/embedded/**' +
+                  ',**/org/eclipse/jetty/asyncrest/**' +
+                  ',**/org/eclipse/jetty/demo/**' +
+                  // special environments / late integrations
+                  ',**/org/eclipse/jetty/gcloud/**' +
+                  ',**/org/eclipse/jetty/infinispan/**' +
+                  ',**/org/eclipse/jetty/osgi/**' +
+                  ',**/org/eclipse/jetty/spring/**' +
+                  ',**/org/eclipse/jetty/http/spi/**' +
+                  // test classes
+                  ',**/org/eclipse/jetty/tests/**' +
+                  ',**/org/eclipse/jetty/test/**',
+                execPattern: '**/target/jacoco.exec',
+                classPattern: '**/target/classes',
+                sourcePattern: '**/src/main/java'
+              warnings consoleParsers: [[parserName: 'Maven'], [parserName: 'Java']]
+              junit testResults: '**/target/surefire-reports/*.xml,**/target/invoker-reports/TEST*.xml,**/target/autobahntestsuite-reports/*.xml'
+            }
           }
         }
 
-        stage("Build / Test - JDK13") {
+        stage("Build / Test - JDK14") {
           agent { node { label 'linux' } }
           steps {
-            timeout(time: 120, unit: 'MINUTES') {
-              mavenBuild("jdk13", "-T3 -Pmongodb clean install", "maven3", true)
-              warnings consoleParsers: [[parserName: 'Maven'], [parserName: 'Java']]
-              junit testResults: '**/target/surefire-reports/*.xml,**/target/invoker-reports/TEST*.xml'
+            container('jetty-build') {
+              timeout( time: 120, unit: 'MINUTES' ) {
+                mavenBuild( "jdk14", "-T3 -Pmongodb clean install", "maven3", true )
+                warnings consoleParsers: [[parserName: 'Maven'], [parserName: 'Java']]
+                junit testResults: '**/target/surefire-reports/*.xml,**/target/invoker-reports/TEST*.xml'
+              }
             }
           }
         }
@@ -54,9 +60,13 @@ pipeline {
         stage("Build Javadoc") {
           agent { node { label 'linux' } }
           steps {
-            timeout(time: 30, unit: 'MINUTES') {
-              mavenBuild("jdk11", "package source:jar javadoc:jar javadoc:aggregate-jar -Peclipse-release  -DskipTests -Dpmd.skip=true -Dcheckstyle.skip=true", "maven3", true)
-              warnings consoleParsers: [[parserName: 'Maven'], [parserName: 'JavaDoc'], [parserName: 'Java']]
+            container('jetty-build') {
+              timeout( time: 30, unit: 'MINUTES' ) {
+                mavenBuild( "jdk11",
+                            "package source:jar javadoc:jar javadoc:aggregate-jar -Peclipse-release  -DskipTests -Dpmd.skip=true -Dcheckstyle.skip=true",
+                            "maven3", true )
+                warnings consoleParsers: [[parserName: 'Maven'], [parserName: 'JavaDoc'], [parserName: 'Java']]
+              }
             }
           }
         }

VERSION.txt

@@ -355,6 +355,46 @@ jetty-10.0.0-alpha0 - 11 July 2019
  + 3849 ClosedChannelException from jetty-test-webapp javax websocket chat
    example
 
+jetty-9.4.28.v20200408 - 08 April 2020
+ + 847 Setting async timeout on WebSocketClient does not seem to timeout writes
+ + 2896 Wrong Certificate Selected When Using Multiple Virtual Host Names in
+   Conscrypt
+ + 4443 Track backport of ALPN APIs to Java 8
+ + 4529 ErrorHandler showing servlet info, can not be disabled unless
+   overriding most of its functionality
+ + 4542 servlet context root mapping incorrect
+ + 4619 Inconsistent library versions notice.
+ + 4620 Using console-capture with StdErrLog results in empty log file
+ + 4621 jetty-jaspi in jetty-all uber aggregate artifact requires
+   javax.security.auth.message.AuthException which cannot be included
+ + 4628 Add support for conditional module dependencies in jetty-start
+ + 4631 Startup XmlConfiguration WARN on Arg threadpool
+ + 4638 maxFormContentSize fix in Issue #3856 broke JenkinsCI/Winstone
+ + 4644 no injection of env-entry if env-entry-value is whitespace only or
+   missing
+ + 4645 Empty "X-Forwarded-Port" header results in NumberFormatException
+ + 4647 Hazelcast remote.xml configuration file do not configure hazelcast
+   remote addresses
+ + 4650 Do not use ServiceLoader every time a WebSocketSession is created
+ + 4654 Hazelcast configurationLocation is not configurable via mod files
+ + 4662 Jetty 9.4.x calls ServletContextListener.contextDestroyed() too early
+ + 4671 CustomRequestLog throws NullPointerException when no request cookie is
+   present
+ + 4673 Short reads break form-data multipart parsing
+ + 4676 ALPN support for Java 15
+ + 4682 "UnreadableSessionDataException Unreadable session ..." after upgrading
+   to 9.4.27
+ + 4693 Version 9.4.25 breaks binary compatibility by renaming
+   Response.closeOutput()
+ + 4699 ServletContainerInitializer.onStartUp is not called with maven jar
+   packaging using Jetty Maven Plugin
+ + 4711 Reset trailers on recycled response
+ + 4714 Low setMaxConcurrentStreams causes "1/unexpected_data_frame" errors
+ + 4735 Get env variables in PHP scripts served through FastCGIProxyServlet
+ + 4737 PreDestroy not called for non-async and run-as servlets
+ + 4739 @RunAs not honoured on servlets
+ + 4751 Refresh NetworkTraffic* classes
+
 jetty-9.4.27.v20200227 - 27 February 2020
  + 3247 Generate jetty-maven-plugin website
  + 4247 Cookie security attributes are going to mandated by Google Chrome
@@ -398,13 +438,13 @@ jetty-9.4.25.v20191220 - 20 December 2019
  + 4269 ResponseWriter should not throw RuntimeIOExceptions
  + 4323 QOS Filter does not handle IllegalStateException and never releases
    passes
- + 4329 rewrite prevents URL session tracking.
+ + 4329 rewrite prevents URL session tracking
  + 4331 Improve handling of HttpOutput.close() for pending writes
  + 4350 Deprecated MultiPartInputStreamParser still used in jetty-server
    (MultiPartsUtilParser) but OSGi ExportPackage suppressed
  + 4351 Servlet.service called before Servlet.init is finished when servlet is
    lazily initialized
- + 4363 jetty-maven-plugin no longer processes supplied context.xml-file.
+ + 4363 jetty-maven-plugin no longer processes supplied context.xml-file
  + 4366 HTTP client uses SOCKS4 proxy hostname for SSL hostname verification
  + 4374 Jetty client: Response.AsyncContentListener.onContent is not called
  + 4376 Async Content Complete bug results in

examples/embedded/src/main/java/org/eclipse/jetty/embedded/ExampleServer.java

@@ -19,11 +19,10 @@
 package org.eclipse.jetty.embedded;
 
 import org.eclipse.jetty.server.Connector;
-import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.handler.DefaultHandler;
-import org.eclipse.jetty.server.handler.HandlerCollection;
+import org.eclipse.jetty.server.handler.HandlerList;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 
 public class ExampleServer
@@ -41,9 +40,7 @@ public class ExampleServer
         context.addServlet(HelloServlet.class, "/hello");
         context.addServlet(AsyncEchoServlet.class, "/echo/*");
 
-        HandlerCollection handlers = new HandlerCollection();
-        handlers.setHandlers(new Handler[]{context, new DefaultHandler()});
-        server.setHandler(handlers);
+        server.setHandler(new HandlerList(context, new DefaultHandler()));
 
         return server;
     }

examples/embedded/src/main/java/org/eclipse/jetty/embedded/FastFileServer.java

@@ -31,7 +31,6 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.http.MimeTypes;
-import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.HttpOutput;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.Server;
@@ -62,12 +61,9 @@ public class FastFileServer
     {
         Server server = new Server(port);
 
-        HandlerList handlers = new HandlerList();
-        handlers.setHandlers(new Handler[]{
+        server.setHandler(new HandlerList(
             new FastFileHandler(resourceBase),
-            new DefaultHandler()
-        });
-        server.setHandler(handlers);
+            new DefaultHandler()));
         return server;
     }
 

examples/embedded/src/main/java/org/eclipse/jetty/embedded/FileServer.java

@@ -21,7 +21,6 @@ package org.eclipse.jetty.embedded;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
-import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.handler.DefaultHandler;
 import org.eclipse.jetty.server.handler.HandlerList;
@@ -53,9 +52,7 @@ public class FileServer
         resourceHandler.setBaseResource(baseResource);
 
         // Add the ResourceHandler to the server.
-        HandlerList handlers = new HandlerList();
-        handlers.setHandlers(new Handler[]{resourceHandler, new DefaultHandler()});
-        server.setHandler(handlers);
+        server.setHandler(new HandlerList(resourceHandler, new DefaultHandler()));
 
         return server;
     }

examples/embedded/src/main/java/org/eclipse/jetty/embedded/JarServer.java

@@ -51,10 +51,7 @@ public class JarServer
         context.setBaseResource(base);
         context.addServlet(new ServletHolder(new DefaultServlet()), "/");
 
-        HandlerList handlers = new HandlerList();
-        handlers.addHandler(context);
-        handlers.addHandler(new DefaultHandler());
-        server.setHandler(handlers);
+        server.setHandler(new HandlerList(context, new DefaultHandler()));
         return server;
     }
 

examples/embedded/src/main/java/org/eclipse/jetty/embedded/LikeJettyXml.java

@@ -40,7 +40,6 @@ import org.eclipse.jetty.security.HashLoginService;
 import org.eclipse.jetty.server.AsyncRequestLogWriter;
 import org.eclipse.jetty.server.CustomRequestLog;
 import org.eclipse.jetty.server.DebugListener;
-import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.LowResourceMonitor;
@@ -51,7 +50,7 @@ import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.SslConnectionFactory;
 import org.eclipse.jetty.server.handler.ContextHandlerCollection;
 import org.eclipse.jetty.server.handler.DefaultHandler;
-import org.eclipse.jetty.server.handler.HandlerCollection;
+import org.eclipse.jetty.server.handler.HandlerList;
 import org.eclipse.jetty.server.handler.StatisticsHandler;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
@@ -105,10 +104,8 @@ public class LikeJettyXml
         // httpConfig.addCustomizer(new ForwardedRequestCustomizer());
 
         // Handler Structure
-        HandlerCollection handlers = new HandlerCollection();
         ContextHandlerCollection contexts = new ContextHandlerCollection();
-        handlers.setHandlers(new Handler[]{contexts, new DefaultHandler()});
-        server.setHandler(handlers);
+        server.setHandler(new HandlerList(contexts, new DefaultHandler()));
 
         // === jetty-jmx.xml ===
         MBeanContainer mbContainer = new MBeanContainer(

examples/embedded/src/main/java/org/eclipse/jetty/embedded/ManyHandlers.java

@@ -124,7 +124,7 @@ public class ManyHandlers
         CustomRequestLog ncsaLog = new CustomRequestLog(requestLogFile.getAbsolutePath());
         server.setRequestLog(ncsaLog);
 
-        // create the handler collections
+        // create the handlers list
         HandlerList handlers = new HandlerList();
 
         // wrap contexts around specific handlers

examples/embedded/src/main/java/org/eclipse/jetty/embedded/SimplestServer.java

@@ -23,6 +23,7 @@ import org.eclipse.jetty.server.Server;
 /**
  * The simplest possible Jetty server.
  */
+// TODO: remove this class, only used in documentation.
 public class SimplestServer
 {
     public static Server createServer(int port)

examples/embedded/src/main/java/org/eclipse/jetty/embedded/WebSocketJsrServer.java

@@ -50,11 +50,8 @@ public class WebSocketJsrServer
     {
         Server server = new Server(port);
 
-        HandlerList handlers = new HandlerList();
-
         ServletContextHandler context = new ServletContextHandler();
         context.setContextPath("/");
-        handlers.addHandler(context);
 
         // Enable javax.websocket configuration for the context
         JavaxWebSocketServletContainerInitializer.configure(context,
@@ -65,8 +62,7 @@ public class WebSocketJsrServer
             }
         );
 
-        handlers.addHandler(new DefaultHandler());
-        server.setHandler(handlers);
+        server.setHandler(new HandlerList(context, new DefaultHandler()));
 
         return server;
     }

examples/embedded/src/main/resources/exampleserver.xml

@@ -25,7 +25,7 @@
   </New>
 
   <Set name="handler">
-    <New class="org.eclipse.jetty.server.handler.HandlerCollection">
+    <New class="org.eclipse.jetty.server.handler.HandlerList">
       <Set name="handlers">
         <Array type="org.eclipse.jetty.server.Handler">
           <Item>

examples/embedded/src/main/resources/jetty-otherserver.xml

@@ -3,7 +3,7 @@
 
 <Configure id="OtherServer" class="org.eclipse.jetty.server.Server">
     <Set name="handler">
-      <New id="Handlers" class="org.eclipse.jetty.server.handler.HandlerCollection">
+      <New id="Handlers" class="org.eclipse.jetty.server.handler.HandlerList">
         <Set name="handlers">
          <Array type="org.eclipse.jetty.server.Handler">
            <Item>

examples/embedded/src/test/java/org/eclipse/jetty/embedded/ExampleServerTest.java

@@ -21,7 +21,7 @@ package org.eclipse.jetty.embedded;
 import java.net.URI;
 
 import org.eclipse.jetty.client.api.ContentResponse;
-import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.client.util.StringRequestContent;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.server.Server;
@@ -75,7 +75,7 @@ public class ExampleServerTest extends AbstractEmbeddedTest
         String postBody = "Greetings from " + ExampleServerTest.class;
         ContentResponse response = client.newRequest(uri)
             .method(HttpMethod.POST)
-            .content(new StringContentProvider(postBody))
+            .body(new StringRequestContent(postBody))
             .send();
 
         // Check the response status code

jetty-alpn/jetty-alpn-client/src/main/java/org/eclipse/jetty/alpn/client/ALPNClientConnection.java

@@ -44,9 +44,6 @@ public class ALPNClientConnection extends NegotiatingClientConnection
 
     public void selected(String protocol)
     {
-        if (protocol == null || !protocols.contains(protocol))
-            close();
-        else
-            completed(protocol);
+        completed(protocol);
     }
 }

jetty-alpn/jetty-alpn-client/src/main/java/org/eclipse/jetty/alpn/client/ALPNClientConnectionFactory.java

@@ -110,12 +110,4 @@ public class ALPNClientConnectionFactory extends NegotiatingClientConnectionFact
         }
         throw new IllegalStateException("No ALPNProcessor for " + engine);
     }
-
-    public static class ALPN extends Info
-    {
-        public ALPN(Executor executor, ClientConnectionFactory factory, List<String> protocols)
-        {
-            super(List.of("alpn"), new ALPNClientConnectionFactory(executor, factory, protocols));
-        }
-    }
 }

jetty-alpn/jetty-alpn-java-client/src/main/java/org/eclipse/jetty/alpn/java/client/JDK9ClientALPNProcessor.java

@@ -75,8 +75,11 @@ public class JDK9ClientALPNProcessor implements ALPNProcessor.Client
         {
             String protocol = alpnConnection.getSSLEngine().getApplicationProtocol();
             if (LOG.isDebugEnabled())
-                LOG.debug("selected protocol {}", protocol);
-            alpnConnection.selected(protocol);
+                LOG.debug("selected protocol '{}'", protocol);
+            if (protocol != null && !protocol.isEmpty())
+                alpnConnection.selected(protocol);
+            else
+                alpnConnection.selected(null);
         }
     }
 }

jetty-annotations/src/main/java/org/eclipse/jetty/annotations/RunAsAnnotationHandler.java

@@ -21,7 +21,6 @@ package org.eclipse.jetty.annotations;
 import javax.servlet.Servlet;
 
 import org.eclipse.jetty.annotations.AnnotationIntrospector.AbstractIntrospectableAnnotationHandler;
-import org.eclipse.jetty.plus.annotation.RunAsCollection;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.webapp.Descriptor;
 import org.eclipse.jetty.webapp.MetaData;
@@ -61,14 +60,7 @@ public class RunAsAnnotationHandler extends AbstractIntrospectableAnnotationHand
                     if (d == null)
                     {
                         metaData.setOrigin(holder.getName() + ".servlet.run-as", runAs, clazz);
-                        org.eclipse.jetty.plus.annotation.RunAs ra = new org.eclipse.jetty.plus.annotation.RunAs(clazz.getName(), role);
-                        RunAsCollection raCollection = (RunAsCollection)_context.getAttribute(RunAsCollection.RUNAS_COLLECTION);
-                        if (raCollection == null)
-                        {
-                            raCollection = new RunAsCollection();
-                            _context.setAttribute(RunAsCollection.RUNAS_COLLECTION, raCollection);
-                        }
-                        raCollection.add(ra);
+                        holder.setRunAsRole(role);
                     }
                 }
             }

jetty-annotations/src/test/java/org/eclipse/jetty/annotations/TestAnnotationConfiguration.java

@@ -118,25 +118,25 @@ public class TestAnnotationConfiguration
     @Test
     public void testAnnotationScanControl() throws Exception
     {
-        //check that a 2.5 webapp won't discover annotations
+        //check that a 2.5 webapp with configurationDiscovered will discover annotations
         TestableAnnotationConfiguration config25 = new TestableAnnotationConfiguration();
         WebAppContext context25 = new WebAppContext();
         context25.setClassLoader(Thread.currentThread().getContextClassLoader());
         context25.setAttribute(AnnotationConfiguration.MULTI_THREADED, Boolean.FALSE);
         context25.setAttribute(AnnotationConfiguration.MAX_SCAN_WAIT, 0);
+        context25.setConfigurationDiscovered(false);
         context25.getMetaData().setWebDescriptor(new WebDescriptor(Resource.newResource(web25)));
         context25.getServletContext().setEffectiveMajorVersion(2);
         context25.getServletContext().setEffectiveMinorVersion(5);
         config25.configure(context25);
         config25.assertAnnotationDiscovery(false);
 
-        //check that a 2.5 webapp with configurationDiscovered will discover annotations
+        //check that a 2.5 webapp discover annotations
         TestableAnnotationConfiguration config25b = new TestableAnnotationConfiguration();
         WebAppContext context25b = new WebAppContext();
         context25b.setClassLoader(Thread.currentThread().getContextClassLoader());
         context25b.setAttribute(AnnotationConfiguration.MULTI_THREADED, Boolean.FALSE);
         context25b.setAttribute(AnnotationConfiguration.MAX_SCAN_WAIT, 0);
-        context25b.setConfigurationDiscovered(true);
         context25b.getMetaData().setWebDescriptor(new WebDescriptor(Resource.newResource(web25)));
         context25b.getServletContext().setEffectiveMajorVersion(2);
         context25b.getServletContext().setEffectiveMinorVersion(5);
@@ -293,6 +293,7 @@ public class TestAnnotationConfiguration
             AnnotationConfiguration config = new AnnotationConfiguration();
             WebAppContext context = new WebAppContext();
             List<ServletContainerInitializer> scis;
+            context.setConfigurationDiscovered(false);
             context.setClassLoader(webAppLoader);
             context.getMetaData().setWebDescriptor(new WebDescriptor(Resource.newResource(web25)));
             context.getMetaData().setWebInfClassesResources(classes);

jetty-annotations/src/test/java/org/eclipse/jetty/annotations/TestRunAsAnnotation.java

@@ -0,0 +1,63 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.annotations;
+
+import java.io.File;
+
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
+import org.eclipse.jetty.util.resource.Resource;
+import org.eclipse.jetty.webapp.WebAppContext;
+import org.eclipse.jetty.webapp.WebDescriptor;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TestRunAsAnnotation
+{
+    @Test
+    public void testRunAsAnnotation() throws Exception
+    {
+        WebAppContext wac = new WebAppContext();
+        
+        //pre-add a servlet but not by descriptor
+        ServletHolder holder = new ServletHolder();
+        holder.setName("foo1");
+        holder.setHeldClass(ServletC.class);
+        holder.setInitOrder(1); //load on startup
+        wac.getServletHandler().addServletWithMapping(holder, "/foo/*");
+        
+        //add another servlet of the same class, but as if by descriptor
+        ServletHolder holder2 = new ServletHolder();
+        holder2.setName("foo2");
+        holder2.setHeldClass(ServletC.class);
+        holder2.setInitOrder(1);
+        wac.getServletHandler().addServletWithMapping(holder2, "/foo2/*");
+        Resource fakeXml = Resource.newResource(new File(MavenTestingUtils.getTargetTestingDir("run-as"), "fake.xml"));
+        wac.getMetaData().setOrigin(holder2.getName() + ".servlet.run-as", new WebDescriptor(fakeXml));
+        
+        AnnotationIntrospector parser = new AnnotationIntrospector(wac);
+        RunAsAnnotationHandler handler = new RunAsAnnotationHandler(wac);
+        parser.registerHandler(handler);
+        parser.introspect(new ServletC(), null);
+        
+        assertEquals("admin", holder.getRunAsRole());
+        assertEquals(null, holder2.getRunAsRole());
+    }
+}

jetty-ant/src/main/java/org/eclipse/jetty/ant/ServerProxyImpl.java

@@ -30,7 +30,6 @@ import org.eclipse.jetty.ant.types.ContextHandlers;
 import org.eclipse.jetty.ant.utils.ServerProxy;
 import org.eclipse.jetty.ant.utils.TaskLog;
 import org.eclipse.jetty.security.LoginService;
-import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.RequestLog;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
@@ -39,6 +38,7 @@ import org.eclipse.jetty.server.handler.ContextHandler;
 import org.eclipse.jetty.server.handler.ContextHandlerCollection;
 import org.eclipse.jetty.server.handler.DefaultHandler;
 import org.eclipse.jetty.server.handler.HandlerCollection;
+import org.eclipse.jetty.server.handler.HandlerList;
 import org.eclipse.jetty.util.Scanner;
 import org.eclipse.jetty.util.resource.PathResource;
 import org.eclipse.jetty.util.resource.Resource;
@@ -406,23 +406,15 @@ public class ServerProxyImpl implements ServerProxy
         if (requestLog != null)
             server.setRequestLog(requestLog);
 
-        contexts = (ContextHandlerCollection)server
-            .getChildHandlerByClass(ContextHandlerCollection.class);
+        contexts = server.getChildHandlerByClass(ContextHandlerCollection.class);
         if (contexts == null)
         {
             contexts = new ContextHandlerCollection();
-            HandlerCollection handlers = (HandlerCollection)server
-                .getChildHandlerByClass(HandlerCollection.class);
+            HandlerCollection handlers = server.getChildHandlerByClass(HandlerCollection.class);
             if (handlers == null)
-            {
-                handlers = new HandlerCollection();
-                server.setHandler(handlers);
-                handlers.setHandlers(new Handler[]{contexts, new DefaultHandler()});
-            }
+                server.setHandler(new HandlerList(contexts, new DefaultHandler()));
             else
-            {
                 handlers.addHandler(contexts);
-            }
         }
 
         //if there are any extra contexts to deploy

jetty-client/src/main/java/org/eclipse/jetty/client/AbstractConnectorHttpClientTransport.java

@@ -72,7 +72,7 @@ public abstract class AbstractConnectorHttpClientTransport extends AbstractHttpC
         context.put(ClientConnector.CLIENT_CONNECTION_FACTORY_CONTEXT_KEY, destination.getClientConnectionFactory());
         @SuppressWarnings("unchecked")
         Promise<Connection> promise = (Promise<Connection>)context.get(HTTP_CONNECTION_PROMISE_CONTEXT_KEY);
-        context.put(ClientConnector.CONNECTION_PROMISE_CONTEXT_KEY, promise);
+        context.put(ClientConnector.CONNECTION_PROMISE_CONTEXT_KEY, Promise.from(ioConnection -> {}, promise::failed));
         connector.connect(address, context);
     }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/AsyncContentProvider.java

@@ -21,10 +21,14 @@ package org.eclipse.jetty.client;
 import java.util.EventListener;
 
 import org.eclipse.jetty.client.api.ContentProvider;
+import org.eclipse.jetty.client.api.Request;
 
 /**
  * A {@link ContentProvider} that notifies listeners that content is available.
+ *
+ * @deprecated no replacement, use {@link Request.Content} instead.
  */
+@Deprecated
 public interface AsyncContentProvider extends ContentProvider
 {
     /**

jetty-client/src/main/java/org/eclipse/jetty/client/AuthenticationProtocolHandler.java

@@ -30,7 +30,6 @@ import java.util.regex.Pattern;
 import org.eclipse.jetty.client.api.Authentication;
 import org.eclipse.jetty.client.api.Authentication.HeaderInfo;
 import org.eclipse.jetty.client.api.Connection;
-import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
@@ -187,8 +186,8 @@ public abstract class AuthenticationProtocolHandler implements ProtocolHandler
                 return;
             }
 
-            ContentProvider requestContent = request.getContent();
-            if (requestContent != null && !requestContent.isReproducible())
+            Request.Content requestContent = request.getBody();
+            if (!requestContent.isReproducible())
             {
                 if (LOG.isDebugEnabled())
                     LOG.debug("Request content not reproducible for {}", request);

jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java

@@ -53,7 +53,7 @@ import org.eclipse.jetty.client.api.Destination;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
-import org.eclipse.jetty.client.util.FormContentProvider;
+import org.eclipse.jetty.client.util.FormRequestContent;
 import org.eclipse.jetty.http.HttpCompliance;
 import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpHeader;
@@ -380,7 +380,7 @@ public class HttpClient extends ContainerLifeCycle
      */
     public ContentResponse FORM(URI uri, Fields fields) throws InterruptedException, ExecutionException, TimeoutException
     {
-        return POST(uri).content(new FormContentProvider(fields)).send();
+        return POST(uri).body(new FormRequestContent(fields)).send();
     }
 
     /**
@@ -447,7 +447,7 @@ public class HttpClient extends ContainerLifeCycle
         Request newRequest = newHttpRequest(oldRequest.getConversation(), newURI);
         newRequest.method(oldRequest.getMethod())
             .version(oldRequest.getVersion())
-            .content(oldRequest.getContent())
+            .body(oldRequest.getBody())
             .idleTimeout(oldRequest.getIdleTimeout(), TimeUnit.MILLISECONDS)
             .timeout(oldRequest.getTimeout(), TimeUnit.MILLISECONDS)
             .followRedirects(oldRequest.isFollowRedirects());

jetty-client/src/main/java/org/eclipse/jetty/client/HttpConnection.java

@@ -27,9 +27,9 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
 import org.eclipse.jetty.client.api.Authentication;
-import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.util.BytesRequestContent;
 import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpVersion;
@@ -129,11 +129,6 @@ public abstract class HttpConnection implements IConnection
         if (normalized)
             return;
 
-        HttpVersion version = request.getVersion();
-        HttpFields headers = request.getHeaders();
-        ContentProvider content = request.getContent();
-        ProxyConfiguration.Proxy proxy = destination.getProxy();
-
         // Make sure the path is there
         String path = request.getPath();
         if (path.trim().length() == 0)
@@ -144,6 +139,7 @@ public abstract class HttpConnection implements IConnection
 
         URI uri = request.getURI();
 
+        ProxyConfiguration.Proxy proxy = destination.getProxy();
         if (proxy instanceof HttpProxy && !HttpClient.isSchemeSecure(request.getScheme()) && uri != null)
         {
             path = uri.toString();
@@ -151,6 +147,8 @@ public abstract class HttpConnection implements IConnection
         }
 
         // If we are HTTP 1.1, add the Host header
+        HttpVersion version = request.getVersion();
+        HttpFields headers = request.getHeaders();
         if (version.getVersion() <= 11)
         {
             if (!headers.containsKey(HttpHeader.HOST.asString()))
@@ -158,13 +156,16 @@ public abstract class HttpConnection implements IConnection
         }
 
         // Add content headers
-        if (content != null)
+        Request.Content content = request.getBody();
+        if (content == null)
+        {
+            request.body(new BytesRequestContent());
+        }
+        else
         {
             if (!headers.containsKey(HttpHeader.CONTENT_TYPE.asString()))
             {
-                String contentType = null;
-                if (content instanceof ContentProvider.Typed)
-                    contentType = ((ContentProvider.Typed)content).getContentType();
+                String contentType = content.getContentType();
                 if (contentType != null)
                 {
                     headers.put(HttpHeader.CONTENT_TYPE, contentType);

jetty-client/src/main/java/org/eclipse/jetty/client/HttpContent.java

@@ -1,237 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-package org.eclipse.jetty.client;
-
-import java.io.Closeable;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.Iterator;
-
-import org.eclipse.jetty.client.api.ContentProvider;
-import org.eclipse.jetty.util.BufferUtil;
-import org.eclipse.jetty.util.Callback;
-import org.eclipse.jetty.util.IO;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * {@link HttpContent} is a stateful, linear representation of the request content provided
- * by a {@link ContentProvider} that can be traversed one-way to obtain content buffers to
- * send to an HTTP server.
- * <p>
- * {@link HttpContent} offers the notion of a one-way cursor to traverse the content.
- * The cursor starts in a virtual "before" position and can be advanced using {@link #advance()}
- * until it reaches a virtual "after" position where the content is fully consumed.
- * <pre>
- *      +---+  +---+  +---+  +---+  +---+
- *      |   |  |   |  |   |  |   |  |   |
- *      +---+  +---+  +---+  +---+  +---+
- *   ^           ^                    ^    ^
- *   |           | --&gt; advance()      |    |
- *   |           |                  last   |
- *   |           |                         |
- * before        |                        after
- *               |
- *            current
- * </pre>
- * At each valid (non-before and non-after) cursor position, {@link HttpContent} provides the following state:
- * <ul>
- * <li>the buffer containing the content to send, via {@link #getByteBuffer()}</li>
- * <li>a copy of the content buffer that can be used for notifications, via {@link #getContent()}</li>
- * <li>whether the buffer to write is the last one, via {@link #isLast()}</li>
- * </ul>
- * {@link HttpContent} may not have content, if the related {@link ContentProvider} is {@code null}, and this
- * is reflected by {@link #hasContent()}.
- * <p>
- * {@link HttpContent} may have {@link AsyncContentProvider deferred content}, in which case {@link #advance()}
- * moves the cursor to a position that provides {@code null} {@link #getByteBuffer() buffer} and
- * {@link #getContent() content}. When the deferred content is available, a further call to {@link #advance()}
- * will move the cursor to a position that provides non {@code null} buffer and content.
- */
-public class HttpContent implements Callback, Closeable
-{
-    private static final Logger LOG = LoggerFactory.getLogger(HttpContent.class);
-    private static final ByteBuffer AFTER = ByteBuffer.allocate(0);
-    private static final ByteBuffer CLOSE = ByteBuffer.allocate(0);
-
-    private final ContentProvider provider;
-    private final Iterator<ByteBuffer> iterator;
-    private ByteBuffer buffer;
-    private ByteBuffer content;
-    private boolean last;
-
-    public HttpContent(ContentProvider provider)
-    {
-        this.provider = provider;
-        this.iterator = provider == null ? Collections.<ByteBuffer>emptyIterator() : provider.iterator();
-    }
-
-    /**
-     * @return true if the buffer is the sentinel instance {@link CLOSE}
-     */
-    private static boolean isTheCloseBuffer(ByteBuffer buffer)
-    {
-        @SuppressWarnings("ReferenceEquality")
-        boolean isTheCloseBuffer = (buffer == CLOSE);
-        return isTheCloseBuffer;
-    }
-
-    /**
-     * @return whether there is any content at all
-     */
-    public boolean hasContent()
-    {
-        return provider != null;
-    }
-
-    /**
-     * @return whether the cursor points to the last content
-     */
-    public boolean isLast()
-    {
-        return last;
-    }
-
-    /**
-     * @return the {@link ByteBuffer} containing the content at the cursor's position
-     */
-    public ByteBuffer getByteBuffer()
-    {
-        return buffer;
-    }
-
-    /**
-     * @return a {@link ByteBuffer#slice()} of {@link #getByteBuffer()} at the cursor's position
-     */
-    public ByteBuffer getContent()
-    {
-        return content;
-    }
-
-    /**
-     * Advances the cursor to the next block of content.
-     * <p>
-     * The next block of content may be valid (which yields a non-null buffer
-     * returned by {@link #getByteBuffer()}), but may also be deferred
-     * (which yields a null buffer returned by {@link #getByteBuffer()}).
-     * <p>
-     * If the block of content pointed by the new cursor position is valid, this method returns true.
-     *
-     * @return true if there is content at the new cursor's position, false otherwise.
-     */
-    public boolean advance()
-    {
-        if (iterator instanceof Synchronizable)
-        {
-            synchronized (((Synchronizable)iterator).getLock())
-            {
-                return advance(iterator);
-            }
-        }
-        else
-        {
-            return advance(iterator);
-        }
-    }
-
-    private boolean advance(Iterator<ByteBuffer> iterator)
-    {
-        boolean hasNext = iterator.hasNext();
-        ByteBuffer bytes = hasNext ? iterator.next() : null;
-        boolean hasMore = hasNext && iterator.hasNext();
-        boolean wasLast = last;
-        last = !hasMore;
-
-        if (hasNext)
-        {
-            buffer = bytes;
-            content = bytes == null ? null : bytes.slice();
-            if (LOG.isDebugEnabled())
-                LOG.debug("Advanced content to {} chunk {}", hasMore ? "next" : "last", String.valueOf(bytes));
-            return bytes != null;
-        }
-        else
-        {
-            // No more content, but distinguish between last and consumed.
-            if (wasLast)
-            {
-                buffer = content = AFTER;
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Advanced content past last chunk");
-            }
-            else
-            {
-                buffer = content = CLOSE;
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Advanced content to last chunk");
-            }
-            return false;
-        }
-    }
-
-    /**
-     * @return whether the cursor has been advanced past the {@link #isLast() last} position.
-     */
-    @SuppressWarnings("ReferenceEquality")
-    public boolean isConsumed()
-    {
-        return buffer == AFTER;
-    }
-
-    @Override
-    public void succeeded()
-    {
-        if (isConsumed())
-            return;
-        if (isTheCloseBuffer(buffer))
-            return;
-        if (iterator instanceof Callback)
-            ((Callback)iterator).succeeded();
-    }
-
-    @Override
-    public void failed(Throwable x)
-    {
-        if (isConsumed())
-            return;
-        if (isTheCloseBuffer(buffer))
-            return;
-        if (iterator instanceof Callback)
-            ((Callback)iterator).failed(x);
-    }
-
-    @Override
-    public void close()
-    {
-        if (iterator instanceof Closeable)
-            IO.close((Closeable)iterator);
-    }
-
-    @Override
-    public String toString()
-    {
-        return String.format("%s@%x - has=%b,last=%b,consumed=%b,buffer=%s",
-            getClass().getSimpleName(),
-            hashCode(),
-            hasContent(),
-            isLast(),
-            isConsumed(),
-            BufferUtil.toDetailString(getContent()));
-    }
-}

jetty-client/src/main/java/org/eclipse/jetty/client/HttpExchange.java

@@ -20,6 +20,7 @@ package org.eclipse.jetty.client;
 
 import java.util.List;
 
+import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
 import org.slf4j.Logger;
@@ -237,6 +238,12 @@ public class HttpExchange
 
         // We failed this exchange, deal with it.
 
+        // Applications could be blocked providing
+        // request content, notify them of the failure.
+        Request.Content body = request.getBody();
+        if (abortRequest && body != null)
+            body.fail(failure);
+
         // Case #1: exchange was in the destination queue.
         if (destination.remove(this))
         {

jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java

@@ -528,7 +528,7 @@ public abstract class HttpReceiver
         HttpResponse response = exchange.getResponse();
 
         if (LOG.isDebugEnabled())
-            LOG.debug("Response complete {}", response);
+            LOG.debug("Response complete {}, result: {}", response, result);
 
         if (result != null)
         {

jetty-client/src/main/java/org/eclipse/jetty/client/HttpRequest.java

@@ -49,8 +49,9 @@ import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
+import org.eclipse.jetty.client.internal.RequestContentAdapter;
 import org.eclipse.jetty.client.util.FutureResponseListener;
-import org.eclipse.jetty.client.util.PathContentProvider;
+import org.eclipse.jetty.client.util.PathRequestContent;
 import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
@@ -81,7 +82,7 @@ public class HttpRequest implements Request
     private long idleTimeout = -1;
     private long timeout;
     private long timeoutAt;
-    private ContentProvider content;
+    private Content content;
     private boolean followRedirects;
     private List<HttpCookie> cookies;
     private Map<String, Object> attributes;
@@ -647,7 +648,9 @@ public class HttpRequest implements Request
     @Override
     public ContentProvider getContent()
     {
-        return content;
+        if (content instanceof RequestContentAdapter)
+            return ((RequestContentAdapter)content).getContentProvider();
+        return null;
     }
 
     @Override
@@ -661,6 +664,18 @@ public class HttpRequest implements Request
     {
         if (contentType != null)
             header(HttpHeader.CONTENT_TYPE, contentType);
+        return body(ContentProvider.toRequestContent(content));
+    }
+
+    @Override
+    public Content getBody()
+    {
+        return content;
+    }
+
+    @Override
+    public Request body(Content content)
+    {
         this.content = content;
         return this;
     }
@@ -674,7 +689,7 @@ public class HttpRequest implements Request
     @Override
     public Request file(Path file, String contentType) throws IOException
     {
-        return content(new PathContentProvider(contentType, file));
+        return body(new PathRequestContent(contentType, file));
     }
 
     @Override
@@ -809,11 +824,7 @@ public class HttpRequest implements Request
     public boolean abort(Throwable cause)
     {
         if (aborted.compareAndSet(null, Objects.requireNonNull(cause)))
-        {
-            if (content instanceof Callback)
-                ((Callback)content).failed(cause);
             return conversation.abort(cause);
-        }
         return false;
     }
 

jetty-client/src/main/java/org/eclipse/jetty/client/HttpSender.java

@@ -23,52 +23,38 @@ import java.util.concurrent.Executor;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
 
-import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpHeaderValue;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.Callback;
-import org.eclipse.jetty.util.IteratingCallback;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * {@link HttpSender} abstracts the algorithm to send HTTP requests, so that subclasses only implement
- * the transport-specific code to send requests over the wire, implementing
- * {@link #sendHeaders(HttpExchange, HttpContent, Callback)} and
- * {@link #sendContent(HttpExchange, HttpContent, Callback)}.
- * <p>
- * {@link HttpSender} governs two state machines.
- * <p>
- * The request state machine is updated by {@link HttpSender} as the various steps of sending a request
- * are executed, see {@code RequestState}.
- * At any point in time, a user thread may abort the request, which may (if the request has not been
- * completely sent yet) move the request state machine to {@code RequestState#FAILURE}.
- * The request state machine guarantees that the request steps are executed (by I/O threads) only if
- * the request has not been failed already.
- * <p>
- * The sender state machine is updated by {@link HttpSender} from three sources: deferred content notifications
- * (via {@link #onContent()}), 100-continue notifications (via {@link #proceed(HttpExchange, Throwable)})
- * and normal request send (via {@link #sendContent(HttpExchange, HttpContent, Callback)}).
- * This state machine must guarantee that the request sending is never executed concurrently: only one of
- * those sources may trigger the call to {@link #sendContent(HttpExchange, HttpContent, Callback)}.
+ * <p>HttpSender abstracts the algorithm to send HTTP requests, so that subclasses only
+ * implement the transport-specific code to send requests over the wire, implementing
+ * {@link #sendHeaders(HttpExchange, ByteBuffer, boolean, Callback)} and
+ * {@link #sendContent(HttpExchange, ByteBuffer, boolean, Callback)}.</p>
+ * <p>HttpSender governs the request state machines, which is updated as the various
+ * steps of sending a request are executed, see {@code RequestState}.
+ * At any point in time, a user thread may abort the request, which may (if the request
+ * has not been completely sent yet) move the request state machine to {@code RequestState#FAILURE}.
+ * The request state machine guarantees that the request steps are executed (by I/O threads)
+ * only if the request has not been failed already.</p>
  *
  * @see HttpReceiver
  */
-public abstract class HttpSender implements AsyncContentProvider.Listener
+public abstract class HttpSender
 {
     private static final Logger LOG = LoggerFactory.getLogger(HttpSender.class);
 
+    private final ContentConsumer consumer = new ContentConsumer();
     private final AtomicReference<RequestState> requestState = new AtomicReference<>(RequestState.QUEUED);
-    private final AtomicReference<SenderState> senderState = new AtomicReference<>(SenderState.IDLE);
-    private final Callback commitCallback = new CommitCallback();
-    private final IteratingCallback contentCallback = new ContentCallback();
-    private final Callback lastCallback = new LastCallback();
+    private final AtomicReference<Throwable> failure = new AtomicReference<>();
     private final HttpChannel channel;
-    private HttpContent content;
-    private Throwable failure;
+    private Request.Content.Subscription subscription;
 
     protected HttpSender(HttpChannel channel)
     {
@@ -90,126 +76,15 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         return requestState.get() == RequestState.FAILURE;
     }
 
-    @Override
-    public void onContent()
-    {
-        HttpExchange exchange = getHttpExchange();
-        if (exchange == null)
-            return;
-
-        while (true)
-        {
-            SenderState current = senderState.get();
-            switch (current)
-            {
-                case IDLE:
-                {
-                    SenderState newSenderState = SenderState.SENDING;
-                    if (updateSenderState(current, newSenderState))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Deferred content available, {} -> {}", current, newSenderState);
-                        contentCallback.iterate();
-                        return;
-                    }
-                    break;
-                }
-                case SENDING:
-                {
-                    SenderState newSenderState = SenderState.SENDING_WITH_CONTENT;
-                    if (updateSenderState(current, newSenderState))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Deferred content available, {} -> {}", current, newSenderState);
-                        return;
-                    }
-                    break;
-                }
-                case EXPECTING:
-                {
-                    SenderState newSenderState = SenderState.EXPECTING_WITH_CONTENT;
-                    if (updateSenderState(current, newSenderState))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Deferred content available, {} -> {}", current, newSenderState);
-                        return;
-                    }
-                    break;
-                }
-                case PROCEEDING:
-                {
-                    SenderState newSenderState = SenderState.PROCEEDING_WITH_CONTENT;
-                    if (updateSenderState(current, newSenderState))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Deferred content available, {} -> {}", current, newSenderState);
-                        return;
-                    }
-                    break;
-                }
-                case SENDING_WITH_CONTENT:
-                case EXPECTING_WITH_CONTENT:
-                case PROCEEDING_WITH_CONTENT:
-                case WAITING:
-                case COMPLETED:
-                case FAILED:
-                {
-                    if (LOG.isDebugEnabled())
-                        LOG.debug("Deferred content available, {}", current);
-                    return;
-                }
-                default:
-                {
-                    illegalSenderState(current);
-                    return;
-                }
-            }
-        }
-    }
-
     public void send(HttpExchange exchange)
     {
         if (!queuedToBegin(exchange))
             return;
 
-        Request request = exchange.getRequest();
-        ContentProvider contentProvider = request.getContent();
-        HttpContent content = this.content = new HttpContent(contentProvider);
-
-        SenderState newSenderState = SenderState.SENDING;
-        if (expects100Continue(request))
-            newSenderState = content.hasContent() ? SenderState.EXPECTING_WITH_CONTENT : SenderState.EXPECTING;
-
-        out:
-        while (true)
-        {
-            SenderState current = senderState.get();
-            switch (current)
-            {
-                case IDLE:
-                case COMPLETED:
-                {
-                    if (updateSenderState(current, newSenderState))
-                        break out;
-                    break;
-                }
-                default:
-                {
-                    illegalSenderState(current);
-                    return;
-                }
-            }
-        }
-
-        // Setting the listener may trigger calls to onContent() by other
-        // threads so we must set it only after the sender state has been updated
-        if (contentProvider instanceof AsyncContentProvider)
-            ((AsyncContentProvider)contentProvider).setListener(this);
-
         if (!beginToHeaders(exchange))
             return;
 
-        sendHeaders(exchange, content, commitCallback);
+        demand();
     }
 
     protected boolean expects100Continue(Request request)
@@ -228,10 +103,16 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         RequestNotifier notifier = getHttpChannel().getHttpDestination().getRequestNotifier();
         notifier.notifyBegin(request);
 
+        Request.Content body = request.getBody();
+
+        consumer.exchange = exchange;
+        consumer.expect100 = expects100Continue(request);
+        subscription = body.subscribe(consumer, !consumer.expect100);
+
         if (updateRequestState(RequestState.TRANSIENT, RequestState.BEGIN))
             return true;
 
-        terminateRequest(exchange);
+        abortRequest(exchange);
         return false;
     }
 
@@ -249,7 +130,7 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         if (updateRequestState(RequestState.TRANSIENT, RequestState.HEADERS))
             return true;
 
-        terminateRequest(exchange);
+        abortRequest(exchange);
         return false;
     }
 
@@ -267,7 +148,7 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         if (updateRequestState(RequestState.TRANSIENT, RequestState.COMMIT))
             return true;
 
-        terminateRequest(exchange);
+        abortRequest(exchange);
         return false;
     }
 
@@ -291,7 +172,7 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
                 if (updateRequestState(RequestState.TRANSIENT, RequestState.CONTENT))
                     return true;
 
-                terminateRequest(exchange);
+                abortRequest(exchange);
                 return false;
             }
             default:
@@ -353,6 +234,20 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
             executeAbort(exchange, failure);
     }
 
+    private void demand()
+    {
+        try
+        {
+            subscription.demand();
+        }
+        catch (Throwable x)
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Failure invoking demand()", x);
+            anyToFailure(x);
+        }
+    }
+
     private void executeAbort(HttpExchange exchange, Throwable failure)
     {
         try
@@ -368,13 +263,23 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         }
     }
 
-    private void terminateRequest(HttpExchange exchange)
+    private void abortRequest(HttpExchange exchange)
     {
-        // In abort(), the state is updated before the failure is recorded
-        // to avoid to overwrite it, so here we may read a null failure.
-        Throwable failure = this.failure;
-        if (failure == null)
-            failure = new HttpRequestException("Concurrent failure", exchange.getRequest());
+        Throwable failure = this.failure.get();
+
+        if (subscription != null)
+            subscription.fail(failure);
+
+        dispose();
+
+        Request request = exchange.getRequest();
+        if (LOG.isDebugEnabled())
+            LOG.debug("Request abort {} {} on {}: {}", request, exchange, getHttpChannel(), failure);
+        HttpDestination destination = getHttpChannel().getHttpDestination();
+        destination.getRequestNotifier().notifyFailure(request, failure);
+
+        // Mark atomically the request as terminated, with
+        // respect to concurrency between request and response.
         Result result = exchange.terminateRequest();
         terminateRequest(exchange, failure, result);
     }
@@ -415,163 +320,73 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
     }
 
     /**
-     * Implementations should send the HTTP headers over the wire, possibly with some content,
-     * in a single write, and notify the given {@code callback} of the result of this operation.
-     * <p>
-     * If there is more content to send, then {@link #sendContent(HttpExchange, HttpContent, Callback)}
-     * will be invoked.
+     * <p>Implementations should send the HTTP headers over the wire, possibly with some content,
+     * in a single write, and notify the given {@code callback} of the result of this operation.</p>
+     * <p>If there is more content to send, then {@link #sendContent(HttpExchange, ByteBuffer, boolean, Callback)}
+     * will be invoked.</p>
      *
-     * @param exchange the exchange to send
-     * @param content the content to send
+     * @param exchange the exchange
+     * @param contentBuffer the content to send
+     * @param lastContent whether the content is the last content to send
      * @param callback the callback to notify
      */
-    protected abstract void sendHeaders(HttpExchange exchange, HttpContent content, Callback callback);
+    protected abstract void sendHeaders(HttpExchange exchange, ByteBuffer contentBuffer, boolean lastContent, Callback callback);
 
     /**
-     * Implementations should send the content at the {@link HttpContent} cursor position over the wire.
-     * <p>
-     * The {@link HttpContent} cursor is advanced by HttpSender at the right time, and if more
-     * content needs to be sent, this method is invoked again; subclasses need only to send the content
-     * at the {@link HttpContent} cursor position.
-     * <p>
-     * This method is invoked one last time when {@link HttpContent#isConsumed()} is true and therefore
-     * there is no actual content to send.
-     * This is done to allow subclasses to write "terminal" bytes (such as the terminal chunk when the
-     * transfer encoding is chunked) if their protocol needs to.
+     * <p>Implementations should send the given HTTP content over the wire.</p>
      *
-     * @param exchange the exchange to send
-     * @param content the content to send
+     * @param exchange the exchange
+     * @param contentBuffer the content to send
+     * @param lastContent whether the content is the last content to send
      * @param callback the callback to notify
      */
-    protected abstract void sendContent(HttpExchange exchange, HttpContent content, Callback callback);
+    protected abstract void sendContent(HttpExchange exchange, ByteBuffer contentBuffer, boolean lastContent, Callback callback);
 
     protected void reset()
     {
-        HttpContent content = this.content;
-        this.content = null;
-        content.close();
-        senderState.set(SenderState.COMPLETED);
+        consumer.reset();
     }
 
     protected void dispose()
     {
-        HttpContent content = this.content;
-        this.content = null;
-        if (content != null)
-            content.close();
-        senderState.set(SenderState.FAILED);
     }
 
     public void proceed(HttpExchange exchange, Throwable failure)
     {
-        if (!expects100Continue(exchange.getRequest()))
-            return;
-
-        if (failure != null)
-        {
+        consumer.expect100 = false;
+        if (failure == null)
+            demand();
+        else
             anyToFailure(failure);
-            return;
-        }
-
-        while (true)
-        {
-            SenderState current = senderState.get();
-            switch (current)
-            {
-                case EXPECTING:
-                {
-                    // We are still sending the headers, but we already got the 100 Continue.
-                    if (updateSenderState(current, SenderState.PROCEEDING))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Proceeding while expecting");
-                        return;
-                    }
-                    break;
-                }
-                case EXPECTING_WITH_CONTENT:
-                {
-                    // More deferred content was submitted to onContent(), we already
-                    // got the 100 Continue, but we may be still sending the headers
-                    // (for example, with SSL we may have sent the encrypted data,
-                    // received the 100 Continue but not yet updated the decrypted
-                    // WriteFlusher so sending more content now may result in a
-                    // WritePendingException).
-                    if (updateSenderState(current, SenderState.PROCEEDING_WITH_CONTENT))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Proceeding while scheduled");
-                        return;
-                    }
-                    break;
-                }
-                case WAITING:
-                {
-                    // We received the 100 Continue, now send the content if any.
-                    if (updateSenderState(current, SenderState.SENDING))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Proceeding while waiting");
-                        contentCallback.iterate();
-                        return;
-                    }
-                    break;
-                }
-                case FAILED:
-                {
-                    return;
-                }
-                default:
-                {
-                    illegalSenderState(current);
-                    return;
-                }
-            }
-        }
     }
 
     public boolean abort(HttpExchange exchange, Throwable failure)
     {
+        // Store only the first failure.
+        this.failure.compareAndSet(null, failure);
+
         // Update the state to avoid more request processing.
-        boolean terminate;
-        out:
+        boolean abort;
         while (true)
         {
             RequestState current = requestState.get();
-            switch (current)
+            if (current == RequestState.FAILURE)
             {
-                case FAILURE:
+                return false;
+            }
+            else
+            {
+                if (updateRequestState(current, RequestState.FAILURE))
                 {
-                    return false;
-                }
-                default:
-                {
-                    if (updateRequestState(current, RequestState.FAILURE))
-                    {
-                        terminate = current != RequestState.TRANSIENT;
-                        break out;
-                    }
+                    abort = current != RequestState.TRANSIENT;
                     break;
                 }
             }
         }
 
-        this.failure = failure;
-
-        dispose();
-
-        Request request = exchange.getRequest();
-        if (LOG.isDebugEnabled())
-            LOG.debug("Request abort {} {} on {}: {}", request, exchange, getHttpChannel(), failure);
-        HttpDestination destination = getHttpChannel().getHttpDestination();
-        destination.getRequestNotifier().notifyFailure(request, failure);
-
-        if (terminate)
+        if (abort)
         {
-            // Mark atomically the request as terminated, with
-            // respect to concurrency between request and response.
-            Result result = exchange.terminateRequest();
-            terminateRequest(exchange, failure, result);
+            abortRequest(exchange);
             return true;
         }
         else
@@ -590,27 +405,13 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         return updated;
     }
 
-    private boolean updateSenderState(SenderState from, SenderState to)
-    {
-        boolean updated = senderState.compareAndSet(from, to);
-        if (!updated && LOG.isDebugEnabled())
-            LOG.debug("SenderState update failed: {} -> {}: {}", from, to, senderState.get());
-        return updated;
-    }
-
-    private void illegalSenderState(SenderState current)
-    {
-        anyToFailure(new IllegalStateException("Expected " + current + " found " + senderState.get() + " instead"));
-    }
-
     @Override
     public String toString()
     {
-        return String.format("%s@%x(req=%s,snd=%s,failure=%s)",
+        return String.format("%s@%x(req=%s,failure=%s)",
             getClass().getSimpleName(),
             hashCode(),
             requestState,
-            senderState,
             failure);
     }
 
@@ -649,286 +450,98 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         FAILURE
     }
 
-    /**
-     * The sender states {@link HttpSender} goes through when sending a request.
-     */
-    private enum SenderState
+    private class ContentConsumer implements Request.Content.Consumer, Callback
     {
-        /**
-         * {@link HttpSender} is not sending request headers nor request content
-         */
-        IDLE,
-        /**
-         * {@link HttpSender} is sending the request header or request content
-         */
-        SENDING,
-        /**
-         * {@link HttpSender} is currently sending the request, and deferred content is available to be sent
-         */
-        SENDING_WITH_CONTENT,
-        /**
-         * {@link HttpSender} is sending the headers but will wait for 100 Continue before sending the content
-         */
-        EXPECTING,
-        /**
-         * {@link HttpSender} is currently sending the headers, will wait for 100 Continue, and deferred content is available to be sent
-         */
-        EXPECTING_WITH_CONTENT,
-        /**
-         * {@link HttpSender} has sent the headers and is waiting for 100 Continue
-         */
-        WAITING,
-        /**
-         * {@link HttpSender} is sending the headers, while 100 Continue has arrived
-         */
-        PROCEEDING,
-        /**
-         * {@link HttpSender} is sending the headers, while 100 Continue has arrived, and deferred content is available to be sent
-         */
-        PROCEEDING_WITH_CONTENT,
-        /**
-         * {@link HttpSender} has finished to send the request
-         */
-        COMPLETED,
-        /**
-         * {@link HttpSender} has failed to send the request
-         */
-        FAILED
-    }
+        private HttpExchange exchange;
+        private boolean expect100;
+        private ByteBuffer contentBuffer;
+        private boolean lastContent;
+        private Callback callback;
+        private boolean committed;
+
+        private void reset()
+        {
+            exchange = null;
+            contentBuffer = null;
+            lastContent = false;
+            callback = null;
+            committed = false;
+        }
+
+        @Override
+        public void onContent(ByteBuffer buffer, boolean last, Callback callback)
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Content {} last={} for {}", BufferUtil.toDetailString(buffer), last, exchange.getRequest());
+            this.contentBuffer = buffer.slice();
+            this.lastContent = last;
+            this.callback = callback;
+            if (committed)
+                sendContent(exchange, buffer, last, this);
+            else
+                sendHeaders(exchange, buffer, last, this);
+        }
+
+        @Override
+        public void onFailure(Throwable failure)
+        {
+            failed(failure);
+        }
 
-    private class CommitCallback implements Callback
-    {
         @Override
         public void succeeded()
         {
-            try
+            boolean proceed = false;
+            if (committed)
             {
-                HttpContent content = HttpSender.this.content;
-                if (content == null)
-                    return;
-                content.succeeded();
-                process();
-            }
-            catch (Throwable x)
-            {
-                anyToFailure(x);
-            }
-        }
-
-        @Override
-        public void failed(Throwable failure)
-        {
-            HttpContent content = HttpSender.this.content;
-            if (content == null)
-                return;
-            content.failed(failure);
-            anyToFailure(failure);
-        }
-
-        private void process() throws Exception
-        {
-            HttpExchange exchange = getHttpExchange();
-            if (exchange == null)
-                return;
-
-            if (!headersToCommit(exchange))
-                return;
-
-            HttpContent content = HttpSender.this.content;
-            if (content == null)
-                return;
-
-            if (!content.hasContent())
-            {
-                // No content to send, we are done.
-                someToSuccess(exchange);
+                proceed = someToContent(exchange, contentBuffer);
             }
             else
             {
-                // Was any content sent while committing?
-                ByteBuffer contentBuffer = content.getContent();
-                if (contentBuffer != null)
+                committed = true;
+                if (headersToCommit(exchange))
                 {
-                    if (!someToContent(exchange, contentBuffer))
-                        return;
-                }
-
-                while (true)
-                {
-                    SenderState current = senderState.get();
-                    switch (current)
-                    {
-                        case SENDING:
-                        {
-                            contentCallback.iterate();
-                            return;
-                        }
-                        case SENDING_WITH_CONTENT:
-                        {
-                            // We have deferred content to send.
-                            updateSenderState(current, SenderState.SENDING);
-                            break;
-                        }
-                        case EXPECTING:
-                        {
-                            // We sent the headers, wait for the 100 Continue response.
-                            if (updateSenderState(current, SenderState.WAITING))
-                                return;
-                            break;
-                        }
-                        case EXPECTING_WITH_CONTENT:
-                        {
-                            // We sent the headers, we have deferred content to send,
-                            // wait for the 100 Continue response.
-                            if (updateSenderState(current, SenderState.WAITING))
-                                return;
-                            break;
-                        }
-                        case PROCEEDING:
-                        {
-                            // We sent the headers, we have the 100 Continue response,
-                            // we have no content to send.
-                            if (updateSenderState(current, SenderState.IDLE))
-                                return;
-                            break;
-                        }
-                        case PROCEEDING_WITH_CONTENT:
-                        {
-                            // We sent the headers, we have the 100 Continue response,
-                            // we have deferred content to send.
-                            updateSenderState(current, SenderState.SENDING);
-                            break;
-                        }
-                        case FAILED:
-                        {
-                            return;
-                        }
-                        default:
-                        {
-                            illegalSenderState(current);
-                            return;
-                        }
-                    }
+                    proceed = true;
+                    // Was any content sent while committing?
+                    if (contentBuffer.hasRemaining())
+                        proceed = someToContent(exchange, contentBuffer);
                 }
             }
-        }
-    }
 
-    private class ContentCallback extends IteratingCallback
-    {
-        @Override
-        protected Action process() throws Exception
-        {
-            HttpExchange exchange = getHttpExchange();
-            if (exchange == null)
-                return Action.IDLE;
+            // Succeed the content callback only after emitting the request content event.
+            callback.succeeded();
 
-            HttpContent content = HttpSender.this.content;
-            if (content == null)
-                return Action.IDLE;
+            // There was some concurrent error?
+            if (!proceed)
+                return;
 
-            while (true)
+            if (lastContent)
+            {
+                someToSuccess(exchange);
+            }
+            else if (expect100)
             {
-                boolean advanced = content.advance();
-                boolean lastContent = content.isLast();
                 if (LOG.isDebugEnabled())
-                    LOG.debug("Content present {}, last {}, consumed {} for {}", advanced, lastContent, content.isConsumed(), exchange.getRequest());
-
-                if (advanced)
-                {
-                    sendContent(exchange, content, this);
-                    return Action.SCHEDULED;
-                }
-
-                if (lastContent)
-                {
-                    sendContent(exchange, content, lastCallback);
-                    return Action.IDLE;
-                }
-
-                SenderState current = senderState.get();
-                switch (current)
-                {
-                    case SENDING:
-                    {
-                        if (updateSenderState(current, SenderState.IDLE))
-                        {
-                            if (LOG.isDebugEnabled())
-                                LOG.debug("Content is deferred for {}", exchange.getRequest());
-                            return Action.IDLE;
-                        }
-                        break;
-                    }
-                    case SENDING_WITH_CONTENT:
-                    {
-                        updateSenderState(current, SenderState.SENDING);
-                        break;
-                    }
-                    default:
-                    {
-                        illegalSenderState(current);
-                        return Action.IDLE;
-                    }
-                }
+                    LOG.debug("Expecting 100 Continue for {}", exchange.getRequest());
+            }
+            else
+            {
+                demand();
             }
         }
 
         @Override
-        public void succeeded()
+        public void failed(Throwable x)
         {
-            HttpExchange exchange = getHttpExchange();
-            if (exchange == null)
-                return;
-            HttpContent content = HttpSender.this.content;
-            if (content == null)
-                return;
-            content.succeeded();
-            ByteBuffer buffer = content.getContent();
-            someToContent(exchange, buffer);
-            super.succeeded();
+            if (callback != null)
+                callback.failed(x);
+            anyToFailure(x);
         }
 
         @Override
-        public void onCompleteFailure(Throwable failure)
+        public InvocationType getInvocationType()
         {
-            HttpContent content = HttpSender.this.content;
-            if (content == null)
-                return;
-            content.failed(failure);
-            anyToFailure(failure);
-        }
-
-        @Override
-        protected void onCompleteSuccess()
-        {
-            // Nothing to do, since we always return IDLE from process().
-            // Termination is obtained via LastCallback.
-        }
-    }
-
-    private class LastCallback implements Callback
-    {
-        @Override
-        public void succeeded()
-        {
-            HttpExchange exchange = getHttpExchange();
-            if (exchange == null)
-                return;
-            HttpContent content = HttpSender.this.content;
-            if (content == null)
-                return;
-            content.succeeded();
-            someToSuccess(exchange);
-        }
-
-        @Override
-        public void failed(Throwable failure)
-        {
-            HttpContent content = HttpSender.this.content;
-            if (content == null)
-                return;
-            content.failed(failure);
-            anyToFailure(failure);
+            return InvocationType.NON_BLOCKING;
         }
     }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/RequestNotifier.java

@@ -158,10 +158,10 @@ public class RequestNotifier
 
     public void notifyContent(Request request, ByteBuffer content)
     {
-        // Slice the buffer to avoid that listeners peek into data they should not look at.
-        content = content.slice();
         if (!content.hasRemaining())
             return;
+        // Slice the buffer to avoid that listeners peek into data they should not look at.
+        content = content.slice();
         // Optimized to avoid allocations of iterator instances.
         List<Request.RequestListener> requestListeners = request.getRequestListeners(null);
         for (int i = 0; i < requestListeners.size(); ++i)

jetty-client/src/main/java/org/eclipse/jetty/client/api/ContentProvider.java

@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
 import java.util.Iterator;
 
 import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.internal.RequestContentAdapter;
 import org.eclipse.jetty.client.util.ByteBufferContentProvider;
 import org.eclipse.jetty.client.util.PathContentProvider;
 
@@ -41,9 +42,24 @@ import org.eclipse.jetty.client.util.PathContentProvider;
  * header set by applications; if the length is negative, it typically removes
  * any {@code Content-Length} header set by applications, resulting in chunked
  * content (i.e. {@code Transfer-Encoding: chunked}) being sent to the server.</p>
+ *
+ * @deprecated use {@link Request.Content} instead, or {@link #toRequestContent(ContentProvider)}
+ * to convert ContentProvider to {@link Request.Content}.
  */
+@Deprecated
 public interface ContentProvider extends Iterable<ByteBuffer>
 {
+    /**
+     * <p>Converts a ContentProvider to a {@link Request.Content}.</p>
+     *
+     * @param provider the ContentProvider to convert
+     * @return a {@link Request.Content} that wraps the ContentProvider
+     */
+    public static Request.Content toRequestContent(ContentProvider provider)
+    {
+        return new RequestContentAdapter(provider);
+    }
+
     /**
      * @return the content length, if known, or -1 if the content length is unknown
      */
@@ -68,7 +84,10 @@ public interface ContentProvider extends Iterable<ByteBuffer>
     /**
      * An extension of {@link ContentProvider} that provides a content type string
      * to be used as a {@code Content-Type} HTTP header in requests.
+     *
+     * @deprecated use {@link Request.Content} instead
      */
+    @Deprecated
     public interface Typed extends ContentProvider
     {
         /**

jetty-client/src/main/java/org/eclipse/jetty/client/api/Request.java

@@ -19,6 +19,7 @@
 package org.eclipse.jetty.client.api;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.HttpCookie;
 import java.net.URI;
 import java.net.URLEncoder;
@@ -37,6 +38,7 @@ import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.Fields;
 
 /**
@@ -216,22 +218,39 @@ public interface Request
 
     /**
      * @return the content provider of this request
+     * @deprecated use {@link #getBody()} instead
      */
+    @Deprecated
     ContentProvider getContent();
 
     /**
      * @param content the content provider of this request
      * @return this request object
+     * @deprecated use {@link #body(Content)} instead
      */
+    @Deprecated
     Request content(ContentProvider content);
 
     /**
      * @param content the content provider of this request
      * @param contentType the content type
      * @return this request object
+     * @deprecated use {@link #body(Content)} instead
      */
+    @Deprecated
     Request content(ContentProvider content, String contentType);
 
+    /**
+     * @return the request content of this request
+     */
+    Content getBody();
+
+    /**
+     * @param content the request content of this request
+     * @return this request object
+     */
+    Request body(Content content);
+
     /**
      * Shortcut method to specify a file as a content for this request, with the default content type of
      * "application/octect-stream".
@@ -615,4 +634,158 @@ public interface Request
         {
         }
     }
+
+    /**
+     * <p>A reactive model to produce request content, similar to {@link java.util.concurrent.Flow.Publisher}.</p>
+     * <p>Implementations receive the content consumer via {@link #subscribe(Consumer, boolean)},
+     * and return a {@link Subscription} as the link between producer and consumer.</p>
+     * <p>Content producers must notify content to the consumer only if there is demand.</p>
+     * <p>Content consumers can generate demand for content by invoking {@link Subscription#demand()}.</p>
+     * <p>Content production must follow this algorithm:</p>
+     * <ul>
+     *   <li>the first time content is demanded
+     *   <ul>
+     *     <li>when the content is not available =&gt; produce an empty content</li>
+     *     <li>when the content is available:
+     *       <ul>
+     *         <li>when {@code emitInitialContent == false} =&gt; produce an empty content</li>
+     *         <li>when {@code emitInitialContent == true} =&gt; produce the content</li>
+     *       </ul>
+     *     </li>
+     *   </ul>
+     *   </li>
+     *   <li>the second and subsequent times content is demanded
+     *     <ul>
+     *       <li>when the content is not available =&gt; do not produce content</li>
+     *       <li>when the content is available =&gt; produce the content</li>
+     *     </ul>
+     *   </li>
+     * </ul>
+     *
+     * @see #subscribe(Consumer, boolean)
+     */
+    public interface Content
+    {
+        /**
+         * @return the content type string such as "application/octet-stream" or
+         * "application/json;charset=UTF8", or null if no content type must be set
+         */
+        public default String getContentType()
+        {
+            return "application/octet-stream";
+        }
+
+        /**
+         * @return the content length, if known, or -1 if the content length is unknown
+         */
+        public default long getLength()
+        {
+            return -1;
+        }
+
+        /**
+         * <p>Whether this content producer can produce exactly the same content more
+         * than once.</p>
+         * <p>Implementations should return {@code true} only if the content can be
+         * produced more than once, which means that {@link #subscribe(Consumer, boolean)}
+         * may be called again.</p>
+         * <p>The {@link HttpClient} implementation may use this method in particular
+         * cases where it detects that it is safe to retry a request that failed.</p>
+         *
+         * @return whether the content can be produced more than once
+         */
+        public default boolean isReproducible()
+        {
+            return false;
+        }
+
+        /**
+         * <p>Initializes this content producer with the content consumer, and with
+         * the indication of whether initial content, if present, must be emitted
+         * upon the initial demand of content (to support delaying the send of the
+         * request content in case of {@code Expect: 100-Continue} when
+         * {@code emitInitialContent} is {@code false}).</p>
+         *
+         * @param consumer the content consumer to invoke when there is demand for content
+         * @param emitInitialContent whether to emit initial content, if present
+         * @return the Subscription that links this producer to the consumer
+         */
+        public Subscription subscribe(Consumer consumer, boolean emitInitialContent);
+
+        /**
+         * <p>Fails this request content, possibly failing and discarding accumulated
+         * content that was not demanded.</p>
+         * <p>The failure may be notified to the consumer at a later time, when the
+         * consumer demands for content.</p>
+         * <p>Typical failure: the request being aborted by user code, or idle timeouts.</p>
+         *
+         * @param failure the reason of the failure
+         */
+        public default void fail(Throwable failure)
+        {
+        }
+
+        /**
+         * <p>A reactive model to consume request content, similar to {@link java.util.concurrent.Flow.Subscriber}.</p>
+         * <p>Callback methods {@link #onContent(ByteBuffer, boolean, Callback)} and {@link #onFailure(Throwable)}
+         * are invoked in strict sequential order and never concurrently, although possibly by different threads.</p>
+         */
+        public interface Consumer
+        {
+            /**
+             * <p>Callback method invoked by the producer when there is content available
+             * <em>and</em> there is demand for content.</p>
+             * <p>The {@code callback} is associated with the {@code buffer} to
+             * signal when the content buffer has been consumed.</p>
+             * <p>Failing the {@code callback} does not have any effect on content
+             * production. To stop the content production, the consumer must call
+             * {@link Subscription#fail(Throwable)}.</p>
+             * <p>In case an exception is thrown by this method, it is equivalent to
+             * a call to {@link Subscription#fail(Throwable)}.</p>
+             *
+             * @param buffer the content buffer to consume
+             * @param last whether it's the last content
+             * @param callback a callback to invoke when the content buffer is consumed
+             */
+            public void onContent(ByteBuffer buffer, boolean last, Callback callback);
+
+            /**
+             * <p>Callback method invoked by the producer when it failed to produce content.</p>
+             * <p>Typical failure: a producer getting an exception while reading from an
+             * {@link InputStream} to produce content.</p>
+             *
+             * @param failure the reason of the failure
+             */
+            public default void onFailure(Throwable failure)
+            {
+            }
+        }
+
+        /**
+         * <p>The link between a content producer and a content consumer.</p>
+         * <p>Content consumers can demand more content via {@link #demand()},
+         * or ask the content producer to stop producing content via
+         * {@link #fail(Throwable)}.</p>
+         */
+        public interface Subscription
+        {
+            /**
+             * <p>Demands more content, which eventually results in
+             * {@link Consumer#onContent(ByteBuffer, boolean, Callback)} to be invoked.</p>
+             */
+            public void demand();
+
+            /**
+             * <p>Fails the subscription, notifying the content producer to stop producing
+             * content.</p>
+             * <p>Typical failure: a proxy consumer waiting for more content (or waiting
+             * to demand content) that is failed by an error response from the server.</p>
+             *
+             * @param failure the reason of the failure
+             */
+            public default void fail(Throwable failure)
+            {
+            }
+        }
+    }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/dynamic/HttpClientTransportDynamic.java

@@ -19,12 +19,14 @@
 package org.eclipse.jetty.client.dynamic;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.eclipse.jetty.alpn.client.ALPNClientConnection;
 import org.eclipse.jetty.alpn.client.ALPNClientConnectionFactory;
@@ -37,6 +39,7 @@ import org.eclipse.jetty.client.MultiplexConnectionPool;
 import org.eclipse.jetty.client.MultiplexHttpDestination;
 import org.eclipse.jetty.client.Origin;
 import org.eclipse.jetty.client.http.HttpClientConnectionFactory;
+import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.io.ClientConnectionFactory;
 import org.eclipse.jetty.io.ClientConnector;
@@ -105,7 +108,7 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
             factoryInfos = new Info[]{HttpClientConnectionFactory.HTTP11};
         this.factoryInfos = Arrays.asList(factoryInfos);
         this.protocols = Arrays.stream(factoryInfos)
-                .flatMap(info -> info.getProtocols().stream())
+                .flatMap(info -> Stream.concat(info.getProtocols(false).stream(), info.getProtocols(true).stream()))
                 .distinct()
                 .map(p -> p.toLowerCase(Locale.ENGLISH))
                 .collect(Collectors.toList());
@@ -117,9 +120,9 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
     @Override
     public Origin newOrigin(HttpRequest request)
     {
-        boolean ssl = HttpClient.isSchemeSecure(request.getScheme());
+        boolean secure = HttpClient.isSchemeSecure(request.getScheme());
         String http1 = "http/1.1";
-        String http2 = ssl ? "h2" : "h2c";
+        String http2 = secure ? "h2" : "h2c";
         List<String> protocols = List.of();
         if (request.isVersionExplicit())
         {
@@ -130,16 +133,23 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
         }
         else
         {
-            if (ssl)
+            if (secure)
             {
                 // There may be protocol negotiation, so preserve the order
                 // of protocols chosen by the application.
                 // We need to keep multiple protocols in case the protocol
                 // is negotiated: e.g. [http/1.1, h2] negotiates [h2], but
                 // here we don't know yet what will be negotiated.
+                List<String> http = List.of("http/1.1", "h2c", "h2");
                 protocols = this.protocols.stream()
-                    .filter(p -> p.equals(http1) || p.equals(http2))
-                    .collect(Collectors.toList());
+                    .filter(http::contains)
+                    .collect(Collectors.toCollection(ArrayList::new));
+
+                // The http/1.1 upgrade to http/2 over TLS implicitly
+                // "negotiates" [h2c], so we need to remove [h2]
+                // because we don't want to negotiate using ALPN.
+                if (request.getHeaders().contains(HttpHeader.UPGRADE, "h2c"))
+                    protocols.remove("h2");
             }
             else
             {
@@ -149,7 +159,7 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
         }
         Origin.Protocol protocol = null;
         if (!protocols.isEmpty())
-            protocol = new Origin.Protocol(protocols, ssl && protocols.contains(http2));
+            protocol = new Origin.Protocol(protocols, secure && protocols.contains(http2));
         return getHttpClient().createOrigin(request, protocol);
     }
 
@@ -164,32 +174,33 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
     {
         HttpDestination destination = (HttpDestination)context.get(HTTP_DESTINATION_CONTEXT_KEY);
         Origin.Protocol protocol = destination.getOrigin().getProtocol();
-        ClientConnectionFactory.Info factoryInfo;
+        ClientConnectionFactory factory;
         if (protocol == null)
         {
             // Use the default ClientConnectionFactory.
-            factoryInfo = factoryInfos.get(0);
+            factory = factoryInfos.get(0).getClientConnectionFactory();
         }
         else
         {
             if (destination.isSecure() && protocol.isNegotiate())
             {
-                factoryInfo = new ALPNClientConnectionFactory.ALPN(getClientConnector().getExecutor(), this::newNegotiatedConnection, protocol.getProtocols());
+                factory = new ALPNClientConnectionFactory(getClientConnector().getExecutor(), this::newNegotiatedConnection, protocol.getProtocols());
             }
             else
             {
-                factoryInfo = findClientConnectionFactoryInfo(protocol.getProtocols())
-                        .orElseThrow(() -> new IOException("Cannot find " + ClientConnectionFactory.class.getSimpleName() + " for " + protocol));
+                factory = findClientConnectionFactoryInfo(protocol.getProtocols(), destination.isSecure())
+                    .orElseThrow(() -> new IOException("Cannot find " + ClientConnectionFactory.class.getSimpleName() + " for " + protocol))
+                    .getClientConnectionFactory();
             }
         }
-        return factoryInfo.getClientConnectionFactory().newConnection(endPoint, context);
+        return factory.newConnection(endPoint, context);
     }
 
     public void upgrade(EndPoint endPoint, Map<String, Object> context)
     {
         HttpDestination destination = (HttpDestination)context.get(HTTP_DESTINATION_CONTEXT_KEY);
         Origin.Protocol protocol = destination.getOrigin().getProtocol();
-        Info info = findClientConnectionFactoryInfo(protocol.getProtocols())
+        Info info = findClientConnectionFactoryInfo(protocol.getProtocols(), destination.isSecure())
             .orElseThrow(() -> new IllegalStateException("Cannot find " + ClientConnectionFactory.class.getSimpleName() + " to upgrade to " + protocol));
         info.upgrade(endPoint, context);
     }
@@ -200,13 +211,22 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
         {
             ALPNClientConnection alpnConnection = (ALPNClientConnection)endPoint.getConnection();
             String protocol = alpnConnection.getProtocol();
-            if (LOG.isDebugEnabled())
-                LOG.debug("ALPN negotiated {} among {}", protocol, alpnConnection.getProtocols());
-            if (protocol == null)
-                throw new IOException("Could not negotiate protocol among " + alpnConnection.getProtocols());
-            List<String> protocols = List.of(protocol);
-            Info factoryInfo = findClientConnectionFactoryInfo(protocols)
+            Info factoryInfo;
+            if (protocol != null)
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("ALPN negotiated {} among {}", protocol, alpnConnection.getProtocols());
+                List<String> protocols = List.of(protocol);
+                factoryInfo = findClientConnectionFactoryInfo(protocols, true)
                     .orElseThrow(() -> new IOException("Cannot find " + ClientConnectionFactory.class.getSimpleName() + " for negotiated protocol " + protocol));
+            }
+            else
+            {
+                // Server does not support ALPN, let's try the first protocol.
+                factoryInfo = factoryInfos.get(0);
+                if (LOG.isDebugEnabled())
+                    LOG.debug("No ALPN protocol, using {}", factoryInfo);
+            }
             return factoryInfo.getClientConnectionFactory().newConnection(endPoint, context);
         }
         catch (Throwable failure)
@@ -216,10 +236,10 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
         }
     }
 
-    private Optional<Info> findClientConnectionFactoryInfo(List<String> protocols)
+    private Optional<Info> findClientConnectionFactoryInfo(List<String> protocols, boolean secure)
     {
         return factoryInfos.stream()
-                .filter(info -> info.matches(protocols))
+                .filter(info -> info.matches(protocols, secure))
                 .findFirst();
     }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpClientConnectionFactory.java

@@ -21,12 +21,16 @@ package org.eclipse.jetty.client.http;
 import java.util.List;
 import java.util.Map;
 
+import org.eclipse.jetty.client.dynamic.HttpClientTransportDynamic;
 import org.eclipse.jetty.io.ClientConnectionFactory;
 import org.eclipse.jetty.io.EndPoint;
 
 public class HttpClientConnectionFactory implements ClientConnectionFactory
 {
-    public static final Info HTTP11 = new Info(List.of("http/1.1"), new HttpClientConnectionFactory());
+    /**
+     * <p>Representation of the {@code HTTP/1.1} application protocol used by {@link HttpClientTransportDynamic}.</p>
+     */
+    public static final Info HTTP11 = new HTTP11(new HttpClientConnectionFactory());
 
     @Override
     public org.eclipse.jetty.io.Connection newConnection(EndPoint endPoint, Map<String, Object> context)
@@ -34,4 +38,26 @@ public class HttpClientConnectionFactory implements ClientConnectionFactory
         HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, context);
         return customize(connection, context);
     }
+
+    private static class HTTP11 extends Info
+    {
+        private static final List<String> protocols = List.of("http/1.1");
+
+        private HTTP11(ClientConnectionFactory factory)
+        {
+            super(factory);
+        }
+
+        @Override
+        public List<String> getProtocols(boolean secure)
+        {
+            return protocols;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s@%x%s", getClass().getSimpleName(), hashCode(), protocols);
+        }
+    }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpSenderOverHTTP.java

@@ -21,12 +21,11 @@ package org.eclipse.jetty.client.http;
 import java.nio.ByteBuffer;
 
 import org.eclipse.jetty.client.HttpClient;
-import org.eclipse.jetty.client.HttpContent;
 import org.eclipse.jetty.client.HttpExchange;
 import org.eclipse.jetty.client.HttpRequest;
 import org.eclipse.jetty.client.HttpRequestException;
 import org.eclipse.jetty.client.HttpSender;
-import org.eclipse.jetty.client.api.ContentProvider;
+import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.http.HttpGenerator;
 import org.eclipse.jetty.http.HttpURI;
 import org.eclipse.jetty.http.MetaData;
@@ -42,7 +41,14 @@ public class HttpSenderOverHTTP extends HttpSender
 {
     private static final Logger LOG = LoggerFactory.getLogger(HttpSenderOverHTTP.class);
 
+    private final IteratingCallback headersCallback = new HeadersCallback();
+    private final IteratingCallback contentCallback = new ContentCallback();
     private final HttpGenerator generator = new HttpGenerator();
+    private HttpExchange exchange;
+    private MetaData.Request metaData;
+    private ByteBuffer contentBuffer;
+    private boolean lastContent;
+    private Callback callback;
     private boolean shutdown;
 
     public HttpSenderOverHTTP(HttpChannelOverHTTP channel)
@@ -57,11 +63,26 @@ public class HttpSenderOverHTTP extends HttpSender
     }
 
     @Override
-    protected void sendHeaders(HttpExchange exchange, HttpContent content, Callback callback)
+    protected void sendHeaders(HttpExchange exchange, ByteBuffer contentBuffer, boolean lastContent, Callback callback)
     {
         try
         {
-            new HeadersCallback(exchange, content, callback).iterate();
+            this.exchange = exchange;
+            this.contentBuffer = contentBuffer;
+            this.lastContent = lastContent;
+            this.callback = callback;
+            HttpRequest request = exchange.getRequest();
+            Request.Content requestContent = request.getBody();
+            long contentLength = requestContent == null ? -1 : requestContent.getLength();
+            String path = request.getPath();
+            String query = request.getQuery();
+            if (query != null)
+                path += "?" + query;
+            metaData = new MetaData.Request(request.getMethod(), new HttpURI(path), request.getVersion(), request.getHeaders(), contentLength);
+            metaData.setTrailerSupplier(request.getTrailers());
+            if (LOG.isDebugEnabled())
+                LOG.debug("Sending headers with content {} last={} for {}", BufferUtil.toDetailString(contentBuffer), lastContent, exchange.getRequest());
+            headersCallback.iterate();
         }
         catch (Throwable x)
         {
@@ -72,67 +93,17 @@ public class HttpSenderOverHTTP extends HttpSender
     }
 
     @Override
-    protected void sendContent(HttpExchange exchange, HttpContent content, Callback callback)
+    protected void sendContent(HttpExchange exchange, ByteBuffer contentBuffer, boolean lastContent, Callback callback)
     {
         try
         {
-            HttpClient httpClient = getHttpChannel().getHttpDestination().getHttpClient();
-            ByteBufferPool bufferPool = httpClient.getByteBufferPool();
-            boolean useDirectByteBuffers = httpClient.isUseOutputDirectByteBuffers();
-            ByteBuffer chunk = null;
-            while (true)
-            {
-                ByteBuffer contentBuffer = content.getByteBuffer();
-                boolean lastContent = content.isLast();
-                HttpGenerator.Result result = generator.generateRequest(null, null, chunk, contentBuffer, lastContent);
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Generated content ({} bytes) - {}/{}",
-                        contentBuffer == null ? -1 : contentBuffer.remaining(),
-                        result, generator);
-                switch (result)
-                {
-                    case NEED_CHUNK:
-                    {
-                        chunk = bufferPool.acquire(HttpGenerator.CHUNK_SIZE, useDirectByteBuffers);
-                        break;
-                    }
-                    case NEED_CHUNK_TRAILER:
-                    {
-                        chunk = bufferPool.acquire(httpClient.getRequestBufferSize(), useDirectByteBuffers);
-                        break;
-                    }
-                    case FLUSH:
-                    {
-                        EndPoint endPoint = getHttpChannel().getHttpConnection().getEndPoint();
-                        if (chunk != null)
-                            endPoint.write(new ByteBufferRecyclerCallback(callback, bufferPool, chunk), chunk, contentBuffer);
-                        else
-                            endPoint.write(callback, contentBuffer);
-                        return;
-                    }
-                    case SHUTDOWN_OUT:
-                    {
-                        shutdownOutput();
-                        break;
-                    }
-                    case CONTINUE:
-                    {
-                        if (lastContent)
-                            break;
-                        callback.succeeded();
-                        return;
-                    }
-                    case DONE:
-                    {
-                        callback.succeeded();
-                        return;
-                    }
-                    default:
-                    {
-                        throw new IllegalStateException(result.toString());
-                    }
-                }
-            }
+            this.exchange = exchange;
+            this.contentBuffer = contentBuffer;
+            this.lastContent = lastContent;
+            this.callback = callback;
+            if (LOG.isDebugEnabled())
+                LOG.debug("Sending content {} last={} for {}", BufferUtil.toDetailString(contentBuffer), lastContent, exchange.getRequest());
+            contentCallback.iterate();
         }
         catch (Throwable x)
         {
@@ -145,6 +116,8 @@ public class HttpSenderOverHTTP extends HttpSender
     @Override
     protected void reset()
     {
+        headersCallback.reset();
+        contentCallback.reset();
         generator.reset();
         super.reset();
     }
@@ -177,54 +150,30 @@ public class HttpSenderOverHTTP extends HttpSender
 
     private class HeadersCallback extends IteratingCallback
     {
-        private final HttpExchange exchange;
-        private final Callback callback;
-        private final MetaData.Request metaData;
         private ByteBuffer headerBuffer;
         private ByteBuffer chunkBuffer;
-        private ByteBuffer contentBuffer;
-        private boolean lastContent;
         private boolean generated;
 
-        public HeadersCallback(HttpExchange exchange, HttpContent content, Callback callback)
+        private HeadersCallback()
         {
             super(false);
-            this.exchange = exchange;
-            this.callback = callback;
-
-            HttpRequest request = exchange.getRequest();
-            ContentProvider requestContent = request.getContent();
-            long contentLength = requestContent == null ? -1 : requestContent.getLength();
-            String path = request.getPath();
-            String query = request.getQuery();
-            if (query != null)
-                path += "?" + query;
-            metaData = new MetaData.Request(request.getMethod(), new HttpURI(path), request.getVersion(), request.getHeaders(), contentLength);
-            metaData.setTrailerSupplier(request.getTrailers());
-
-            if (!expects100Continue(request))
-            {
-                content.advance();
-                contentBuffer = content.getByteBuffer();
-                lastContent = content.isLast();
-            }
         }
 
         @Override
         protected Action process() throws Exception
         {
+            HttpClient httpClient = getHttpChannel().getHttpDestination().getHttpClient();
+            ByteBufferPool byteBufferPool = httpClient.getByteBufferPool();
+            boolean useDirectByteBuffers = httpClient.isUseOutputDirectByteBuffers();
             while (true)
             {
                 HttpGenerator.Result result = generator.generateRequest(metaData, headerBuffer, chunkBuffer, contentBuffer, lastContent);
                 if (LOG.isDebugEnabled())
-                    LOG.debug("Generated headers ({} bytes), chunk ({} bytes), content ({} bytes) - {}/{}",
+                    LOG.debug("Generated headers ({} bytes), chunk ({} bytes), content ({} bytes) - {}/{} for {}",
                         headerBuffer == null ? -1 : headerBuffer.remaining(),
                         chunkBuffer == null ? -1 : chunkBuffer.remaining(),
                         contentBuffer == null ? -1 : contentBuffer.remaining(),
-                        result, generator);
-                HttpClient httpClient = getHttpChannel().getHttpDestination().getHttpClient();
-                ByteBufferPool byteBufferPool = httpClient.getByteBufferPool();
-                boolean useDirectByteBuffers = httpClient.isUseOutputDirectByteBuffers();
+                        result, generator, exchange.getRequest());
                 switch (result)
                 {
                     case NEED_HEADER:
@@ -332,37 +281,86 @@ public class HttpSenderOverHTTP extends HttpSender
         }
     }
 
-    private class ByteBufferRecyclerCallback extends Callback.Nested
+    private class ContentCallback extends IteratingCallback
     {
-        private final ByteBufferPool pool;
-        private final ByteBuffer[] buffers;
+        private ByteBuffer chunkBuffer;
 
-        private ByteBufferRecyclerCallback(Callback callback, ByteBufferPool pool, ByteBuffer... buffers)
+        public ContentCallback()
         {
-            super(callback);
-            this.pool = pool;
-            this.buffers = buffers;
+            super(false);
         }
 
         @Override
-        public void succeeded()
+        protected Action process() throws Exception
         {
-            for (ByteBuffer buffer : buffers)
+            HttpClient httpClient = getHttpChannel().getHttpDestination().getHttpClient();
+            ByteBufferPool bufferPool = httpClient.getByteBufferPool();
+            boolean useDirectByteBuffers = httpClient.isUseOutputDirectByteBuffers();
+            while (true)
             {
-                assert !buffer.hasRemaining();
-                pool.release(buffer);
+                HttpGenerator.Result result = generator.generateRequest(null, null, chunkBuffer, contentBuffer, lastContent);
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Generated content ({} bytes, last={}) - {}/{}",
+                        contentBuffer == null ? -1 : contentBuffer.remaining(),
+                        lastContent, result, generator);
+                switch (result)
+                {
+                    case NEED_CHUNK:
+                    {
+                        chunkBuffer = bufferPool.acquire(HttpGenerator.CHUNK_SIZE, useDirectByteBuffers);
+                        break;
+                    }
+                    case NEED_CHUNK_TRAILER:
+                    {
+                        chunkBuffer = bufferPool.acquire(httpClient.getRequestBufferSize(), useDirectByteBuffers);
+                        break;
+                    }
+                    case FLUSH:
+                    {
+                        EndPoint endPoint = getHttpChannel().getHttpConnection().getEndPoint();
+                        if (chunkBuffer != null)
+                            endPoint.write(this, chunkBuffer, contentBuffer);
+                        else
+                            endPoint.write(this, contentBuffer);
+                        return Action.SCHEDULED;
+                    }
+                    case SHUTDOWN_OUT:
+                    {
+                        shutdownOutput();
+                        break;
+                    }
+                    case CONTINUE:
+                    {
+                        break;
+                    }
+                    case DONE:
+                    {
+                        release();
+                        callback.succeeded();
+                        return Action.IDLE;
+                    }
+                    default:
+                    {
+                        throw new IllegalStateException(result.toString());
+                    }
+                }
             }
-            super.succeeded();
         }
 
         @Override
-        public void failed(Throwable x)
+        protected void onCompleteFailure(Throwable cause)
         {
-            for (ByteBuffer buffer : buffers)
-            {
-                pool.release(buffer);
-            }
-            super.failed(x);
+            release();
+            callback.failed(cause);
+        }
+
+        private void release()
+        {
+            HttpClient httpClient = getHttpChannel().getHttpDestination().getHttpClient();
+            ByteBufferPool bufferPool = httpClient.getByteBufferPool();
+            bufferPool.release(chunkBuffer);
+            chunkBuffer = null;
+            contentBuffer = null;
         }
     }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/internal/RequestContentAdapter.java

@@ -0,0 +1,329 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.internal;
+
+import java.io.Closeable;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+import org.eclipse.jetty.client.AsyncContentProvider;
+import org.eclipse.jetty.client.Synchronizable;
+import org.eclipse.jetty.client.api.ContentProvider;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.IO;
+import org.eclipse.jetty.util.thread.AutoLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>Implements the conversion from {@link ContentProvider} to {@link Request.Content}.</p>
+ */
+public class RequestContentAdapter implements Request.Content, Request.Content.Subscription, AsyncContentProvider.Listener, Callback
+{
+    private static final Logger LOG = LoggerFactory.getLogger(RequestContentAdapter.class);
+
+    private final AutoLock lock = new AutoLock();
+    private final ContentProvider provider;
+    private Iterator<ByteBuffer> iterator;
+    private Consumer consumer;
+    private boolean emitInitialContent;
+    private boolean lastContent;
+    private boolean committed;
+    private int demand;
+    private boolean stalled;
+    private boolean hasContent;
+    private Throwable failure;
+
+    public RequestContentAdapter(ContentProvider provider)
+    {
+        this.provider = provider;
+        if (provider instanceof AsyncContentProvider)
+            ((AsyncContentProvider)provider).setListener(this);
+    }
+
+    public ContentProvider getContentProvider()
+    {
+        return provider;
+    }
+
+    @Override
+    public String getContentType()
+    {
+        return provider instanceof ContentProvider.Typed ? ((ContentProvider.Typed)provider).getContentType() : null;
+    }
+
+    @Override
+    public long getLength()
+    {
+        return provider.getLength();
+    }
+
+    @Override
+    public boolean isReproducible()
+    {
+        return provider.isReproducible();
+    }
+
+    @Override
+    public Subscription subscribe(Consumer consumer, boolean emitInitialContent)
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            if (this.consumer != null && !isReproducible())
+                throw new IllegalStateException("Multiple subscriptions not supported on " + this);
+            this.iterator = provider.iterator();
+            this.consumer = consumer;
+            this.emitInitialContent = emitInitialContent;
+            this.lastContent = false;
+            this.committed = false;
+            this.demand = 0;
+            this.stalled = true;
+            this.hasContent = false;
+        }
+        return this;
+    }
+
+    @Override
+    public void demand()
+    {
+        boolean produce;
+        try (AutoLock ignored = lock.lock())
+        {
+            ++demand;
+            produce = stalled;
+            if (stalled)
+                stalled = false;
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("Content demand, producing {} for {}", produce, this);
+        if (produce)
+            produce();
+    }
+
+    @Override
+    public void fail(Throwable failure)
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            if (this.failure == null)
+                this.failure = failure;
+        }
+        failed(failure);
+    }
+
+    @Override
+    public void onContent()
+    {
+        boolean produce = false;
+        try (AutoLock ignored = lock.lock())
+        {
+            hasContent = true;
+            if (demand > 0)
+            {
+                produce = stalled;
+                if (stalled)
+                    stalled = false;
+            }
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("Content event, processing {} for {}", produce, this);
+        if (produce)
+            produce();
+    }
+
+    @Override
+    public void succeeded()
+    {
+        if (iterator instanceof Callback)
+            ((Callback)iterator).succeeded();
+        if (lastContent && iterator instanceof Closeable)
+            IO.close((Closeable)iterator);
+    }
+
+    @Override
+    public void failed(Throwable x)
+    {
+        if (iterator == null)
+            failed(provider, x);
+        else
+            failed(iterator, x);
+    }
+
+    private void failed(Object object, Throwable failure)
+    {
+        if (object instanceof Callback)
+            ((Callback)object).failed(failure);
+        if (object instanceof Closeable)
+            IO.close((Closeable)object);
+    }
+
+    @Override
+    public InvocationType getInvocationType()
+    {
+        return InvocationType.NON_BLOCKING;
+    }
+
+    private void produce()
+    {
+        while (true)
+        {
+            Throwable failure;
+            try (AutoLock ignored = lock.lock())
+            {
+                failure = this.failure;
+            }
+            if (failure != null)
+            {
+                notifyFailure(failure);
+                return;
+            }
+
+            if (committed)
+            {
+                ByteBuffer content = advance();
+                if (content != null)
+                {
+                    notifyContent(content, lastContent);
+                }
+                else
+                {
+                    try (AutoLock ignored = lock.lock())
+                    {
+                        // Call to advance() said there was no content,
+                        // but some content may have arrived meanwhile.
+                        if (hasContent)
+                        {
+                            hasContent = false;
+                            continue;
+                        }
+                        else
+                        {
+                            stalled = true;
+                        }
+                    }
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("No content, processing stalled for {}", this);
+                    return;
+                }
+            }
+            else
+            {
+                committed = true;
+                if (emitInitialContent)
+                {
+                    ByteBuffer content = advance();
+                    if (content != null)
+                        notifyContent(content, lastContent);
+                    else
+                        notifyContent(BufferUtil.EMPTY_BUFFER, false);
+                }
+                else
+                {
+                    notifyContent(BufferUtil.EMPTY_BUFFER, false);
+                }
+            }
+            boolean noDemand;
+            try (AutoLock ignored = lock.lock())
+            {
+                noDemand = demand == 0;
+                if (noDemand)
+                    stalled = true;
+            }
+            if (noDemand)
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("No demand, processing stalled for {}", this);
+                return;
+            }
+        }
+    }
+
+    private ByteBuffer advance()
+    {
+        if (iterator instanceof Synchronizable)
+        {
+            synchronized (((Synchronizable)iterator).getLock())
+            {
+                return next();
+            }
+        }
+        else
+        {
+            return next();
+        }
+    }
+
+    private ByteBuffer next()
+    {
+        boolean hasNext = iterator.hasNext();
+        ByteBuffer bytes = hasNext ? iterator.next() : null;
+        boolean hasMore = hasNext && iterator.hasNext();
+        lastContent = !hasMore;
+        return hasNext ? bytes : BufferUtil.EMPTY_BUFFER;
+    }
+
+    private void notifyContent(ByteBuffer buffer, boolean last)
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            --demand;
+            hasContent = false;
+        }
+
+        try
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Notifying content last={} {} for {}", last, BufferUtil.toDetailString(buffer), this);
+            consumer.onContent(buffer, last, this);
+        }
+        catch (Throwable x)
+        {
+            fail(x);
+        }
+    }
+
+    private void notifyFailure(Throwable failure)
+    {
+        try
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Notifying failure for {}", this, failure);
+            consumer.onFailure(failure);
+        }
+        catch (Exception x)
+        {
+            LOG.trace("Failure while notifying content failure {}", failure, x);
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        int demand;
+        boolean stalled;
+        try (AutoLock ignored = lock.lock())
+        {
+            demand = this.demand;
+            stalled = this.stalled;
+        }
+        return String.format("%s@%x[demand=%d,stalled=%b]", getClass().getSimpleName(), hashCode(), demand, stalled);
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/AbstractRequestContent.java

@@ -0,0 +1,257 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.nio.ByteBuffer;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.thread.AutoLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>Partial implementation of {@link Request.Content}.</p>
+ */
+public abstract class AbstractRequestContent implements Request.Content
+{
+    private static final Logger LOG = LoggerFactory.getLogger(AbstractRequestContent.class);
+
+    private final AutoLock lock = new AutoLock();
+    private final String contentType;
+
+    protected AbstractRequestContent(String contentType)
+    {
+        this.contentType = contentType;
+    }
+
+    @Override
+    public String getContentType()
+    {
+        return contentType;
+    }
+
+    @Override
+    public Subscription subscribe(Consumer consumer, boolean emitInitialContent)
+    {
+        Subscription subscription = newSubscription(consumer, emitInitialContent);
+        if (LOG.isDebugEnabled())
+            LOG.debug("Content subscription for {}: {}", subscription, consumer);
+        return subscription;
+    }
+
+    protected abstract Subscription newSubscription(Consumer consumer, boolean emitInitialContent);
+
+    /**
+     * <p>Partial implementation of {@code Subscription}.</p>
+     * <p>Implements the algorithm described in {@link Request.Content}.</p>
+     */
+    public abstract class AbstractSubscription implements Subscription
+    {
+        private final Consumer consumer;
+        private final boolean emitInitialContent;
+        private Throwable failure;
+        private int demand;
+        // Whether content production was stalled because there was no demand.
+        private boolean stalled;
+        // Whether the first content has been produced.
+        private boolean committed;
+
+        public AbstractSubscription(Consumer consumer, boolean emitInitialContent)
+        {
+            this.consumer = consumer;
+            this.emitInitialContent = emitInitialContent;
+            this.stalled = true;
+        }
+
+        @Override
+        public void demand()
+        {
+            boolean produce;
+            try (AutoLock ignored = lock.lock())
+            {
+                ++demand;
+                produce = stalled;
+                if (stalled)
+                    stalled = false;
+            }
+            if (LOG.isDebugEnabled())
+                LOG.debug("Content demand, producing {} for {}", produce, this);
+            if (produce)
+                produce();
+        }
+
+        private void produce()
+        {
+            while (true)
+            {
+                Throwable failure;
+                boolean committed;
+                try (AutoLock ignored = lock.lock())
+                {
+                    failure = this.failure;
+                    committed = this.committed;
+                }
+                if (failure != null)
+                {
+                    notifyFailure(failure);
+                    return;
+                }
+
+                if (committed || emitInitialContent)
+                {
+                    try
+                    {
+                        if (!produceContent(this::processContent))
+                            return;
+                    }
+                    catch (Throwable x)
+                    {
+                        // Fail and loop around to notify the failure.
+                        fail(x);
+                    }
+                }
+                else
+                {
+                    if (!processContent(BufferUtil.EMPTY_BUFFER, false, Callback.NOOP))
+                        return;
+                }
+            }
+        }
+
+        /**
+         * <p>Subclasses implement this method to produce content,
+         * without worrying about demand or exception handling.</p>
+         * <p>Typical implementation (pseudo code):</p>
+         * <pre>
+         * protected boolean produceContent(Producer producer) throws Exception
+         * {
+         *     // Step 1: try to produce content, exceptions may be thrown during production
+         *     //  (for example, producing content reading from an InputStream may throw).
+         *
+         *     // Step 2A: content could be produced.
+         *     ByteBuffer buffer = ...;
+         *     boolean last = ...;
+         *     Callback callback = ...;
+         *     return producer.produce(buffer, last, callback);
+         *
+         *     // Step 2B: content could not be produced.
+         *     //  (for example it is not available yet)
+         *     return false;
+         * }
+         * </pre>
+         *
+         * @param producer the producer to notify when content can be produced
+         * @return whether content production should continue
+         * @throws Exception when content production fails
+         */
+        protected abstract boolean produceContent(Producer producer) throws Exception;
+
+        @Override
+        public void fail(Throwable failure)
+        {
+            try (AutoLock ignored = lock.lock())
+            {
+                if (this.failure == null)
+                    this.failure = failure;
+            }
+        }
+
+        private boolean processContent(ByteBuffer content, boolean last, Callback callback)
+        {
+            try (AutoLock ignored = lock.lock())
+            {
+                committed = true;
+                --demand;
+            }
+
+            if (content != null)
+                notifyContent(content, last, callback);
+            else
+                callback.succeeded();
+
+            boolean noDemand;
+            try (AutoLock ignored = lock.lock())
+            {
+                noDemand = demand == 0;
+                if (noDemand)
+                    stalled = true;
+            }
+            if (noDemand)
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("No demand, processing stalled for {}", this);
+                return false;
+            }
+            return true;
+        }
+
+        protected void notifyContent(ByteBuffer buffer, boolean last, Callback callback)
+        {
+            try
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Notifying content last={} {} for {}", last, BufferUtil.toDetailString(buffer), this);
+                consumer.onContent(buffer, last, callback);
+            }
+            catch (Throwable x)
+            {
+                callback.failed(x);
+                fail(x);
+            }
+        }
+
+        private void notifyFailure(Throwable failure)
+        {
+            try
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Notifying failure for {}", this, failure);
+                consumer.onFailure(failure);
+            }
+            catch (Exception x)
+            {
+                LOG.trace("Failure while notifying content failure {}", failure, x);
+            }
+        }
+
+        @Override
+        public String toString()
+        {
+            int demand;
+            boolean stalled;
+            boolean committed;
+            try (AutoLock ignored = lock.lock())
+            {
+                demand = this.demand;
+                stalled = this.stalled;
+                committed = this.committed;
+            }
+            return String.format("%s.%s@%x[demand=%d,stalled=%b,committed=%b,emitInitial=%b]",
+                getClass().getEnclosingClass().getSimpleName(),
+                getClass().getSimpleName(), hashCode(), demand, stalled, committed, emitInitialContent);
+        }
+    }
+
+    public interface Producer
+    {
+        boolean produce(ByteBuffer content, boolean lastContent, Callback callback);
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/AbstractTypedContentProvider.java

@@ -20,6 +20,10 @@ package org.eclipse.jetty.client.util;
 
 import org.eclipse.jetty.client.api.ContentProvider;
 
+/**
+ * @deprecated use {@link AbstractRequestContent} instead.
+ */
+@Deprecated
 public abstract class AbstractTypedContentProvider implements ContentProvider.Typed
 {
     private final String contentType;

jetty-client/src/main/java/org/eclipse/jetty/client/util/AsyncRequestContent.java

@@ -0,0 +1,385 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.Closeable;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.locks.Condition;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.thread.AutoLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AsyncRequestContent implements Request.Content, Request.Content.Subscription, Closeable
+{
+    private static final Logger LOG = LoggerFactory.getLogger(AsyncRequestContent.class);
+
+    private final AutoLock lock = new AutoLock();
+    private final Condition flush = lock.newCondition();
+    private final Deque<Chunk> chunks = new ArrayDeque<>();
+    private final String contentType;
+    private long length = -1;
+    private Consumer consumer;
+    private boolean emitInitialContent;
+    private int demand;
+    private boolean stalled;
+    private boolean committed;
+    private boolean closed;
+    private boolean terminated;
+    private Throwable failure;
+
+    public AsyncRequestContent(ByteBuffer... buffers)
+    {
+        this("application/octet-stream", buffers);
+    }
+
+    public AsyncRequestContent(String contentType, ByteBuffer... buffers)
+    {
+        this.contentType = contentType;
+        Stream.of(buffers).forEach(this::offer);
+    }
+
+    @Override
+    public String getContentType()
+    {
+        return contentType;
+    }
+
+    @Override
+    public long getLength()
+    {
+        return length;
+    }
+
+    @Override
+    public Subscription subscribe(Consumer consumer, boolean emitInitialContent)
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            if (this.consumer != null)
+                throw new IllegalStateException("Multiple subscriptions not supported on " + this);
+            this.consumer = consumer;
+            this.emitInitialContent = emitInitialContent;
+            this.stalled = true;
+            if (closed)
+                length = chunks.stream().mapToLong(chunk -> chunk.buffer.remaining()).sum();
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("Content subscription for {}: {}", this, consumer);
+        return this;
+    }
+
+    @Override
+    public void demand()
+    {
+        boolean produce;
+        try (AutoLock ignored = lock.lock())
+        {
+            ++demand;
+            produce = stalled;
+            if (stalled)
+                stalled = false;
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("Content demand, producing {} for {}", produce, this);
+        if (produce)
+            produce();
+    }
+
+    @Override
+    public void fail(Throwable failure)
+    {
+        List<Callback> toFail = List.of();
+        try (AutoLock ignored = lock.lock())
+        {
+            if (this.failure == null)
+            {
+                this.failure = failure;
+                // Transfer all chunks to fail them all.
+                toFail = chunks.stream()
+                    .map(chunk -> chunk.callback)
+                    .collect(Collectors.toList());
+                chunks.clear();
+                flush.signal();
+            }
+        }
+        toFail.forEach(c -> c.failed(failure));
+    }
+
+    public boolean offer(ByteBuffer buffer)
+    {
+        return offer(buffer, Callback.NOOP);
+    }
+
+    public boolean offer(ByteBuffer buffer, Callback callback)
+    {
+        return offer(new Chunk(buffer, callback));
+    }
+
+    private boolean offer(Chunk chunk)
+    {
+        boolean produce = false;
+        Throwable failure;
+        try (AutoLock ignored = lock.lock())
+        {
+            failure = this.failure;
+            if (failure == null)
+            {
+                if (closed)
+                {
+                    failure = new IOException("closed");
+                }
+                else
+                {
+                    chunks.offer(chunk);
+                    if (demand > 0)
+                    {
+                        if (stalled)
+                        {
+                            stalled = false;
+                            produce = true;
+                        }
+                    }
+                }
+            }
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("Content offer {}, producing {} for {}", failure == null ? "succeeded" : "failed", produce, this, failure);
+        if (failure != null)
+        {
+            chunk.callback.failed(failure);
+            return false;
+        }
+        else if (produce)
+        {
+            produce();
+        }
+        return true;
+    }
+
+    private void produce()
+    {
+        while (true)
+        {
+            Throwable failure;
+            try (AutoLock ignored = lock.lock())
+            {
+                failure = this.failure;
+            }
+            if (failure != null)
+            {
+                notifyFailure(consumer, failure);
+                return;
+            }
+
+            try
+            {
+                Consumer consumer;
+                Chunk chunk = Chunk.EMPTY;
+                boolean lastContent = false;
+                try (AutoLock ignored = lock.lock())
+                {
+                    if (terminated)
+                        throw new EOFException("Demand after last content");
+                    consumer = this.consumer;
+                    if (committed || emitInitialContent)
+                    {
+                        chunk = chunks.poll();
+                        lastContent = closed && chunks.isEmpty();
+                        if (lastContent)
+                            terminated = true;
+                    }
+                    if (chunk == null && (lastContent || !committed))
+                        chunk = Chunk.EMPTY;
+                    if (chunk == null)
+                    {
+                        stalled = true;
+                    }
+                    else
+                    {
+                        --demand;
+                        committed = true;
+                    }
+                }
+                if (chunk == null)
+                {
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("No content, processing stalled for {}", this);
+                    return;
+                }
+
+                notifyContent(consumer, chunk.buffer, lastContent, Callback.from(this::notifyFlush, chunk.callback));
+
+                boolean noDemand;
+                try (AutoLock ignored = lock.lock())
+                {
+                    noDemand = demand == 0;
+                    if (noDemand)
+                        stalled = true;
+                }
+                if (noDemand)
+                {
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("No demand, processing stalled for {}", this);
+                    return;
+                }
+            }
+            catch (Throwable x)
+            {
+                // Fail and loop around to notify the failure.
+                fail(x);
+            }
+        }
+    }
+
+    private void notifyContent(Consumer consumer, ByteBuffer buffer, boolean last, Callback callback)
+    {
+        try
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Notifying content last={} {} for {}", last, BufferUtil.toDetailString(buffer), this);
+            consumer.onContent(buffer, last, callback);
+        }
+        catch (Throwable x)
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Failure while notifying content", x);
+            callback.failed(x);
+            fail(x);
+        }
+    }
+
+    private void notifyFailure(Consumer consumer, Throwable failure)
+    {
+        try
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Notifying failure for {}", this, failure);
+            consumer.onFailure(failure);
+        }
+        catch (Throwable x)
+        {
+            LOG.trace("Failure while notifying content failure {}", failure, x);
+        }
+    }
+
+    private void notifyFlush()
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            flush.signal();
+        }
+    }
+
+    public void flush() throws IOException
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            try
+            {
+                while (true)
+                {
+                    // Always wrap the exception to make sure
+                    // the stack trace comes from flush().
+                    if (failure != null)
+                        throw new IOException(failure);
+                    if (chunks.isEmpty())
+                        return;
+                    flush.await();
+                }
+            }
+            catch (InterruptedException x)
+            {
+                throw new InterruptedIOException();
+            }
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        boolean produce = false;
+        try (AutoLock ignored = lock.lock())
+        {
+            if (closed)
+                return;
+            closed = true;
+            if (demand > 0)
+            {
+                if (stalled)
+                {
+                    stalled = false;
+                    produce = true;
+                }
+            }
+            flush.signal();
+        }
+        if (produce)
+            produce();
+    }
+
+    public boolean isClosed()
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            return closed;
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        int demand;
+        boolean stalled;
+        int chunks;
+        try (AutoLock ignored = lock.lock())
+        {
+            demand = this.demand;
+            stalled = this.stalled;
+            chunks = this.chunks.size();
+        }
+        return String.format("%s@%x[demand=%d,stalled=%b,chunks=%d]", getClass().getSimpleName(), hashCode(), demand, stalled, chunks);
+    }
+
+    private static class Chunk
+    {
+        private static final Chunk EMPTY = new Chunk(BufferUtil.EMPTY_BUFFER, Callback.NOOP);
+
+        private final ByteBuffer buffer;
+        private final Callback callback;
+
+        private Chunk(ByteBuffer buffer, Callback callback)
+        {
+            this.buffer = Objects.requireNonNull(buffer);
+            this.callback = Objects.requireNonNull(callback);
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/ByteBufferContentProvider.java

@@ -30,7 +30,10 @@ import org.eclipse.jetty.client.api.ContentProvider;
  * The position and limit of the {@link ByteBuffer}s passed to the constructor are not modified,
  * and each invocation of the {@link #iterator()} method returns a {@link ByteBuffer#slice() slice}
  * of the original {@link ByteBuffer}.
+ *
+ * @deprecated use {@link ByteBufferRequestContent} instead.
  */
+@Deprecated
 public class ByteBufferContentProvider extends AbstractTypedContentProvider
 {
     private final ByteBuffer[] buffers;

jetty-client/src/main/java/org/eclipse/jetty/client/util/ByteBufferRequestContent.java

@@ -0,0 +1,94 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+
+/**
+ * <p>A {@link Request.Content} for {@link ByteBuffer}s.</p>
+ * <p>The position and limit of the {@link ByteBuffer}s passed to the constructor are not modified;
+ * content production returns a {@link ByteBuffer#slice() slice} of the original {@link ByteBuffer}.
+ */
+public class ByteBufferRequestContent extends AbstractRequestContent
+{
+    private final ByteBuffer[] buffers;
+    private final long length;
+
+    public ByteBufferRequestContent(ByteBuffer... buffers)
+    {
+        this("application/octet-stream", buffers);
+    }
+
+    public ByteBufferRequestContent(String contentType, ByteBuffer... buffers)
+    {
+        super(contentType);
+        this.buffers = buffers;
+        this.length = Arrays.stream(buffers).mapToLong(Buffer::remaining).sum();
+    }
+
+    @Override
+    public long getLength()
+    {
+        return length;
+    }
+
+    @Override
+    public boolean isReproducible()
+    {
+        return true;
+    }
+
+    @Override
+    protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
+    {
+        return new SubscriptionImpl(consumer, emitInitialContent);
+    }
+
+    private class SubscriptionImpl extends AbstractSubscription
+    {
+        private int index;
+
+        private SubscriptionImpl(Consumer consumer, boolean emitInitialContent)
+        {
+            super(consumer, emitInitialContent);
+        }
+
+        @Override
+        protected boolean produceContent(Producer producer) throws IOException
+        {
+            if (index < 0)
+                throw new EOFException("Demand after last content");
+            ByteBuffer buffer = BufferUtil.EMPTY_BUFFER;
+            if (index < buffers.length)
+                buffer = buffers[index++];
+            boolean lastContent = index == buffers.length;
+            if (lastContent)
+                index = -1;
+            return producer.produce(buffer.slice(), lastContent, Callback.NOOP);
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/BytesContentProvider.java

@@ -26,7 +26,10 @@ import org.eclipse.jetty.client.api.ContentProvider;
 
 /**
  * A {@link ContentProvider} for byte arrays.
+ *
+ * @deprecated use {@link BytesRequestContent} instead.
  */
+@Deprecated
 public class BytesContentProvider extends AbstractTypedContentProvider
 {
     private final byte[][] bytes;

jetty-client/src/main/java/org/eclipse/jetty/client/util/BytesRequestContent.java

@@ -0,0 +1,91 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+
+/**
+ * A {@link Request.Content} for byte arrays.
+ */
+public class BytesRequestContent extends AbstractRequestContent
+{
+    private final byte[][] bytes;
+    private final long length;
+
+    public BytesRequestContent(byte[]... bytes)
+    {
+        this("application/octet-stream", bytes);
+    }
+
+    public BytesRequestContent(String contentType, byte[]... bytes)
+    {
+        super(contentType);
+        this.bytes = bytes;
+        this.length = Arrays.stream(bytes).mapToLong(a -> a.length).sum();
+    }
+
+    @Override
+    public long getLength()
+    {
+        return length;
+    }
+
+    @Override
+    public boolean isReproducible()
+    {
+        return true;
+    }
+
+    @Override
+    protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
+    {
+        return new SubscriptionImpl(consumer, emitInitialContent);
+    }
+
+    private class SubscriptionImpl extends AbstractSubscription
+    {
+        private int index;
+
+        private SubscriptionImpl(Consumer consumer, boolean emitInitialContent)
+        {
+            super(consumer, emitInitialContent);
+        }
+
+        @Override
+        protected boolean produceContent(Producer producer) throws IOException
+        {
+            if (index < 0)
+                throw new EOFException("Demand after last content");
+            ByteBuffer buffer = BufferUtil.EMPTY_BUFFER;
+            if (index < bytes.length)
+                buffer = ByteBuffer.wrap(bytes[index++]);
+            boolean lastContent = index == bytes.length;
+            if (lastContent)
+                index = -1;
+            return producer.produce(buffer, lastContent, Callback.NOOP);
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/DeferredContentProvider.java

@@ -85,7 +85,10 @@ import org.eclipse.jetty.util.Callback;
  *     content.offer(ByteBuffer.wrap("some content".getBytes()));
  * }
  * </pre>
+ *
+ * @deprecated use {@link AsyncRequestContent} instead.
  */
+@Deprecated
 public class DeferredContentProvider implements AsyncContentProvider, Callback, Closeable
 {
     private static final Chunk CLOSE = new Chunk(BufferUtil.EMPTY_BUFFER, Callback.NOOP);
@@ -285,6 +288,7 @@ public class DeferredContentProvider implements AsyncContentProvider, Callback,
             synchronized (lock)
             {
                 chunk = current;
+                current = null;
                 if (chunk != null)
                 {
                     --size;

jetty-client/src/main/java/org/eclipse/jetty/client/util/FormContentProvider.java

@@ -30,7 +30,10 @@ import org.eclipse.jetty.util.Fields;
 /**
  * A {@link ContentProvider} for form uploads with the
  * "application/x-www-form-urlencoded" content type.
+ *
+ * @deprecated use {@link FormRequestContent} instead.
  */
+@Deprecated
 public class FormContentProvider extends StringContentProvider
 {
     public FormContentProvider(Fields fields)

jetty-client/src/main/java/org/eclipse/jetty/client/util/FormRequestContent.java

@@ -0,0 +1,78 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.charset.UnsupportedCharsetException;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.Fields;
+
+/**
+ * <p>A {@link Request.Content} for form uploads with the
+ * "application/x-www-form-urlencoded" content type.</p>
+ */
+public class FormRequestContent extends StringRequestContent
+{
+    public FormRequestContent(Fields fields)
+    {
+        this(fields, StandardCharsets.UTF_8);
+    }
+
+    public FormRequestContent(Fields fields, Charset charset)
+    {
+        super("application/x-www-form-urlencoded", convert(fields, charset), charset);
+    }
+
+    public static String convert(Fields fields)
+    {
+        return convert(fields, StandardCharsets.UTF_8);
+    }
+
+    public static String convert(Fields fields, Charset charset)
+    {
+        // Assume 32 chars between name and value.
+        StringBuilder builder = new StringBuilder(fields.getSize() * 32);
+        for (Fields.Field field : fields)
+        {
+            for (String value : field.getValues())
+            {
+                if (builder.length() > 0)
+                    builder.append("&");
+                builder.append(encode(field.getName(), charset)).append("=").append(encode(value, charset));
+            }
+        }
+        return builder.toString();
+    }
+
+    private static String encode(String value, Charset charset)
+    {
+        try
+        {
+            return URLEncoder.encode(value, charset.name());
+        }
+        catch (UnsupportedEncodingException x)
+        {
+            throw new UnsupportedCharsetException(charset.name());
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/InputStreamContentProvider.java

@@ -50,7 +50,10 @@ import org.slf4j.LoggerFactory;
  * The {@link InputStream} passed to the constructor is by default closed when is it fully
  * consumed (or when an exception is thrown while reading it), unless otherwise specified
  * to the {@link #InputStreamContentProvider(java.io.InputStream, int, boolean) constructor}.
+ *
+ * @deprecated use {@link InputStreamRequestContent} instead
  */
+@Deprecated
 public class InputStreamContentProvider implements ContentProvider, Callback, Closeable
 {
     private static final Logger LOG = LoggerFactory.getLogger(InputStreamContentProvider.class);

jetty-client/src/main/java/org/eclipse/jetty/client/util/InputStreamRequestContent.java

@@ -0,0 +1,149 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.IO;
+
+/**
+ * <p>A {@link Request.Content} that produces content from an {@link InputStream}.</p>
+ * <p>The input stream is read once and therefore fully consumed.</p>
+ * <p>It is possible to specify, at the constructor, a buffer size used to read
+ * content from the stream, by default 1024 bytes.</p>
+ * <p>The {@link InputStream} passed to the constructor is by default closed
+ * when is it fully consumed.</p>
+ */
+public class InputStreamRequestContent extends AbstractRequestContent
+{
+    private static final int DEFAULT_BUFFER_SIZE = 4096;
+
+    private final InputStream stream;
+    private final int bufferSize;
+    private Subscription subscription;
+
+    public InputStreamRequestContent(InputStream stream)
+    {
+        this(stream, DEFAULT_BUFFER_SIZE);
+    }
+
+    public InputStreamRequestContent(String contentType, InputStream stream)
+    {
+        this(contentType, stream, DEFAULT_BUFFER_SIZE);
+    }
+
+    public InputStreamRequestContent(InputStream stream, int bufferSize)
+    {
+        this("application/octet-stream", stream, bufferSize);
+    }
+
+    public InputStreamRequestContent(String contentType, InputStream stream, int bufferSize)
+    {
+        super(contentType);
+        this.stream = stream;
+        this.bufferSize = bufferSize;
+    }
+
+    @Override
+    protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
+    {
+        if (subscription != null)
+            throw new IllegalStateException("Multiple subscriptions not supported on " + this);
+        return subscription = new SubscriptionImpl(consumer, emitInitialContent);
+    }
+
+    @Override
+    public void fail(Throwable failure)
+    {
+        super.fail(failure);
+        close();
+    }
+
+    protected ByteBuffer onRead(byte[] buffer, int offset, int length)
+    {
+        return ByteBuffer.wrap(buffer, offset, length);
+    }
+
+    protected void onReadFailure(Throwable failure)
+    {
+    }
+
+    private void close()
+    {
+        IO.close(stream);
+    }
+
+    private class SubscriptionImpl extends AbstractSubscription
+    {
+        private boolean terminated;
+
+        private SubscriptionImpl(Consumer consumer, boolean emitInitialContent)
+        {
+            super(consumer, emitInitialContent);
+        }
+
+        @Override
+        protected boolean produceContent(Producer producer) throws IOException
+        {
+            if (terminated)
+                throw new EOFException("Demand after last content");
+            byte[] bytes = new byte[bufferSize];
+            int read = read(bytes);
+            ByteBuffer buffer = BufferUtil.EMPTY_BUFFER;
+            boolean last = true;
+            if (read < 0)
+            {
+                close();
+                terminated = true;
+            }
+            else
+            {
+                buffer = onRead(bytes, 0, read);
+                last = false;
+            }
+            return producer.produce(buffer, last, Callback.NOOP);
+        }
+
+        private int read(byte[] bytes) throws IOException
+        {
+            try
+            {
+                return stream.read(bytes);
+            }
+            catch (Throwable x)
+            {
+                onReadFailure(x);
+                throw x;
+            }
+        }
+
+        @Override
+        public void fail(Throwable failure)
+        {
+            super.fail(failure);
+            close();
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/InputStreamResponseListener.java

@@ -27,6 +27,7 @@ import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.Queue;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
@@ -76,12 +77,13 @@ import org.slf4j.LoggerFactory;
 public class InputStreamResponseListener extends Listener.Adapter
 {
     private static final Logger LOG = LoggerFactory.getLogger(InputStreamResponseListener.class);
-    private static final DeferredContentProvider.Chunk EOF = new DeferredContentProvider.Chunk(BufferUtil.EMPTY_BUFFER, Callback.NOOP);
+    private static final Chunk EOF = new Chunk(BufferUtil.EMPTY_BUFFER, Callback.NOOP);
+
     private final Object lock = this;
     private final CountDownLatch responseLatch = new CountDownLatch(1);
     private final CountDownLatch resultLatch = new CountDownLatch(1);
     private final AtomicReference<InputStream> stream = new AtomicReference<>();
-    private final Queue<DeferredContentProvider.Chunk> chunks = new ArrayDeque<>();
+    private final Queue<Chunk> chunks = new ArrayDeque<>();
     private Response response;
     private Result result;
     private Throwable failure;
@@ -120,7 +122,7 @@ public class InputStreamResponseListener extends Listener.Adapter
             {
                 if (LOG.isDebugEnabled())
                     LOG.debug("Queueing content {}", content);
-                chunks.add(new DeferredContentProvider.Chunk(content, callback));
+                chunks.add(new Chunk(content, callback));
                 lock.notifyAll();
             }
         }
@@ -268,7 +270,7 @@ public class InputStreamResponseListener extends Listener.Adapter
         {
             while (true)
             {
-                DeferredContentProvider.Chunk chunk = chunks.peek();
+                Chunk chunk = chunks.peek();
                 if (chunk == null || chunk == EOF)
                     break;
                 callbacks.add(chunk.callback);
@@ -299,7 +301,7 @@ public class InputStreamResponseListener extends Listener.Adapter
                 Callback callback = null;
                 synchronized (lock)
                 {
-                    DeferredContentProvider.Chunk chunk;
+                    Chunk chunk;
                     while (true)
                     {
                         chunk = chunks.peek();
@@ -367,4 +369,16 @@ public class InputStreamResponseListener extends Listener.Adapter
             super.close();
         }
     }
+
+    private static class Chunk
+    {
+        private final ByteBuffer buffer;
+        private final Callback callback;
+
+        private Chunk(ByteBuffer buffer, Callback callback)
+        {
+            this.buffer = Objects.requireNonNull(buffer);
+            this.callback = Objects.requireNonNull(callback);
+        }
+    }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/util/MultiPartContentProvider.java

@@ -63,7 +63,10 @@ import org.slf4j.LoggerFactory;
  *     <input type="file" name="icon" />
  * </form>
  * </pre>
+ *
+ * @deprecated use {@link MultiPartRequestContent} instead.
  */
+@Deprecated
 public class MultiPartContentProvider extends AbstractTypedContentProvider implements AsyncContentProvider, Closeable
 {
     private static final Logger LOG = LoggerFactory.getLogger(MultiPartContentProvider.class);

jetty-client/src/main/java/org/eclipse/jetty/client/util/MultiPartRequestContent.java

@@ -0,0 +1,392 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.http.HttpField;
+import org.eclipse.jetty.http.HttpFields;
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.io.RuntimeIOException;
+import org.eclipse.jetty.util.Callback;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>A {@link Request.Content} for form uploads with the {@code "multipart/form-data"}
+ * content type.</p>
+ * <p>Example usage:</p>
+ * <pre>
+ * MultiPartRequestContent multiPart = new MultiPartRequestContent();
+ * multiPart.addFieldPart("field", new StringRequestContent("foo"), null);
+ * multiPart.addFilePart("icon", "img.png", new PathRequestContent(Paths.get("/tmp/img.png")), null);
+ * multiPart.close();
+ * ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
+ *         .method(HttpMethod.POST)
+ *         .content(multiPart)
+ *         .send();
+ * </pre>
+ * <p>The above example would be the equivalent of submitting this form:</p>
+ * <pre>
+ * &lt;form method="POST" enctype="multipart/form-data"  accept-charset="UTF-8"&gt;
+ *     &lt;input type="text" name="field" value="foo" /&gt;
+ *     &lt;input type="file" name="icon" /&gt;
+ * &lt;/form&gt;
+ * </pre>
+ */
+public class MultiPartRequestContent extends AbstractRequestContent implements Closeable
+{
+    private static final Logger LOG = LoggerFactory.getLogger(MultiPartRequestContent.class);
+    private static final byte[] COLON_SPACE_BYTES = new byte[]{':', ' '};
+    private static final byte[] CR_LF_BYTES = new byte[]{'\r', '\n'};
+
+    private static String makeBoundary()
+    {
+        Random random = new Random();
+        StringBuilder builder = new StringBuilder("JettyHttpClientBoundary");
+        int length = builder.length();
+        while (builder.length() < length + 16)
+        {
+            long rnd = random.nextLong();
+            builder.append(Long.toString(rnd < 0 ? -rnd : rnd, 36));
+        }
+        builder.setLength(length + 16);
+        return builder.toString();
+    }
+
+    private final List<Part> parts = new ArrayList<>();
+    private final ByteBuffer firstBoundary;
+    private final ByteBuffer middleBoundary;
+    private final ByteBuffer onlyBoundary;
+    private final ByteBuffer lastBoundary;
+    private long length;
+    private boolean closed;
+    private Subscription subscription;
+
+    public MultiPartRequestContent()
+    {
+        this(makeBoundary());
+    }
+
+    public MultiPartRequestContent(String boundary)
+    {
+        super("multipart/form-data; boundary=" + boundary);
+        String firstBoundaryLine = "--" + boundary + "\r\n";
+        this.firstBoundary = ByteBuffer.wrap(firstBoundaryLine.getBytes(StandardCharsets.US_ASCII));
+        String middleBoundaryLine = "\r\n" + firstBoundaryLine;
+        this.middleBoundary = ByteBuffer.wrap(middleBoundaryLine.getBytes(StandardCharsets.US_ASCII));
+        String onlyBoundaryLine = "--" + boundary + "--\r\n";
+        this.onlyBoundary = ByteBuffer.wrap(onlyBoundaryLine.getBytes(StandardCharsets.US_ASCII));
+        String lastBoundaryLine = "\r\n" + onlyBoundaryLine;
+        this.lastBoundary = ByteBuffer.wrap(lastBoundaryLine.getBytes(StandardCharsets.US_ASCII));
+        this.length = -1;
+    }
+
+    @Override
+    public long getLength()
+    {
+        return length;
+    }
+
+    @Override
+    protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
+    {
+        if (!closed)
+            throw new IllegalStateException("MultiPartRequestContent must be closed before sending the request");
+        if (subscription != null)
+            throw new IllegalStateException("Multiple subscriptions not supported on " + this);
+        length = calculateLength();
+        return subscription = new SubscriptionImpl(consumer, emitInitialContent);
+    }
+
+    @Override
+    public void fail(Throwable failure)
+    {
+        parts.stream()
+            .map(part -> part.content)
+            .forEach(content -> content.fail(failure));
+    }
+
+    /**
+     * <p>Adds a field part with the given {@code name} as field name, and the given
+     * {@code content} as part content.</p>
+     * <p>The {@code Content-Type} of this part will be obtained from:</p>
+     * <ul>
+     * <li>the {@code Content-Type} header in the {@code fields} parameter; otherwise</li>
+     * <li>the {@link Request.Content#getContentType()}</li>
+     * </ul>
+     *
+     * @param name the part name
+     * @param content the part content
+     * @param fields the headers associated with this part
+     */
+    public void addFieldPart(String name, Request.Content content, HttpFields fields)
+    {
+        addPart(new Part(name, null, content, fields));
+    }
+
+    /**
+     * <p>Adds a file part with the given {@code name} as field name, the given
+     * {@code fileName} as file name, and the given {@code content} as part content.</p>
+     * <p>The {@code Content-Type} of this part will be obtained from:</p>
+     * <ul>
+     * <li>the {@code Content-Type} header in the {@code fields} parameter; otherwise</li>
+     * <li>the {@link Request.Content#getContentType()}</li>
+     * </ul>
+     *
+     * @param name the part name
+     * @param fileName the file name associated to this part
+     * @param content the part content
+     * @param fields the headers associated with this part
+     */
+    public void addFilePart(String name, String fileName, Request.Content content, HttpFields fields)
+    {
+        addPart(new Part(name, fileName, content, fields));
+    }
+
+    private void addPart(Part part)
+    {
+        parts.add(part);
+        if (LOG.isDebugEnabled())
+            LOG.debug("Added {}", part);
+    }
+
+    @Override
+    public void close()
+    {
+        closed = true;
+    }
+
+    private long calculateLength()
+    {
+        // Compute the length, if possible.
+        if (parts.isEmpty())
+        {
+            return onlyBoundary.remaining();
+        }
+        else
+        {
+            long result = 0;
+            for (int i = 0; i < parts.size(); ++i)
+            {
+                result += (i == 0) ? firstBoundary.remaining() : middleBoundary.remaining();
+                Part part = parts.get(i);
+                long partLength = part.length;
+                result += partLength;
+                if (partLength < 0)
+                {
+                    result = -1;
+                    break;
+                }
+            }
+            if (result > 0)
+                result += lastBoundary.remaining();
+            return result;
+        }
+    }
+
+    private static class Part
+    {
+        private final String name;
+        private final String fileName;
+        private final Request.Content content;
+        private final HttpFields fields;
+        private final ByteBuffer headers;
+        private final long length;
+
+        private Part(String name, String fileName, Request.Content content, HttpFields fields)
+        {
+            this.name = name;
+            this.fileName = fileName;
+            this.content = content;
+            this.fields = fields;
+            this.headers = headers();
+            this.length = content.getLength() < 0 ? -1 : headers.remaining() + content.getLength();
+        }
+
+        private ByteBuffer headers()
+        {
+            try
+            {
+                // Compute the Content-Disposition.
+                String contentDisposition = "Content-Disposition: form-data; name=\"" + name + "\"";
+                if (fileName != null)
+                    contentDisposition += "; filename=\"" + fileName + "\"";
+                contentDisposition += "\r\n";
+
+                // Compute the Content-Type.
+                String contentType = fields == null ? null : fields.get(HttpHeader.CONTENT_TYPE);
+                if (contentType == null)
+                    contentType = content.getContentType();
+                contentType = "Content-Type: " + contentType + "\r\n";
+
+                if (fields == null || fields.size() == 0)
+                {
+                    String headers = contentDisposition;
+                    headers += contentType;
+                    headers += "\r\n";
+                    return ByteBuffer.wrap(headers.getBytes(StandardCharsets.UTF_8));
+                }
+
+                ByteArrayOutputStream buffer = new ByteArrayOutputStream((fields.size() + 1) * contentDisposition.length());
+                buffer.write(contentDisposition.getBytes(StandardCharsets.UTF_8));
+                buffer.write(contentType.getBytes(StandardCharsets.UTF_8));
+                for (HttpField field : fields)
+                {
+                    if (HttpHeader.CONTENT_TYPE.equals(field.getHeader()))
+                        continue;
+                    buffer.write(field.getName().getBytes(StandardCharsets.US_ASCII));
+                    buffer.write(COLON_SPACE_BYTES);
+                    String value = field.getValue();
+                    if (value != null)
+                        buffer.write(value.getBytes(StandardCharsets.UTF_8));
+                    buffer.write(CR_LF_BYTES);
+                }
+                buffer.write(CR_LF_BYTES);
+                return ByteBuffer.wrap(buffer.toByteArray());
+            }
+            catch (IOException x)
+            {
+                throw new RuntimeIOException(x);
+            }
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s@%x[name=%s,fileName=%s,length=%d,headers=%s]",
+                getClass().getSimpleName(),
+                hashCode(),
+                name,
+                fileName,
+                content.getLength(),
+                fields);
+        }
+    }
+
+    private class SubscriptionImpl extends AbstractSubscription implements Consumer
+    {
+        private State state = State.FIRST_BOUNDARY;
+        private int index;
+        private Subscription subscription;
+
+        private SubscriptionImpl(Consumer consumer, boolean emitInitialContent)
+        {
+            super(consumer, emitInitialContent);
+        }
+
+        @Override
+        protected boolean produceContent(Producer producer) throws IOException
+        {
+            ByteBuffer buffer;
+            boolean last = false;
+            switch (state)
+            {
+                case FIRST_BOUNDARY:
+                {
+                    if (parts.isEmpty())
+                    {
+                        state = State.COMPLETE;
+                        buffer = onlyBoundary.slice();
+                        last = true;
+                        break;
+                    }
+                    else
+                    {
+                        state = State.HEADERS;
+                        buffer = firstBoundary.slice();
+                        break;
+                    }
+                }
+                case HEADERS:
+                {
+                    Part part = parts.get(index);
+                    Request.Content content = part.content;
+                    subscription = content.subscribe(this, true);
+                    state = State.CONTENT;
+                    buffer = part.headers.slice();
+                    break;
+                }
+                case CONTENT:
+                {
+                    buffer = null;
+                    subscription.demand();
+                    break;
+                }
+                case MIDDLE_BOUNDARY:
+                {
+                    state = State.HEADERS;
+                    buffer = middleBoundary.slice();
+                    break;
+                }
+                case LAST_BOUNDARY:
+                {
+                    state = State.COMPLETE;
+                    buffer = lastBoundary.slice();
+                    last = true;
+                    break;
+                }
+                case COMPLETE:
+                {
+                    throw new EOFException("Demand after last content");
+                }
+                default:
+                {
+                    throw new IllegalStateException("Invalid state " + state);
+                }
+            }
+            return producer.produce(buffer, last, Callback.NOOP);
+        }
+
+        @Override
+        public void onContent(ByteBuffer buffer, boolean last, Callback callback)
+        {
+            if (last)
+            {
+                ++index;
+                if (index < parts.size())
+                    state = State.MIDDLE_BOUNDARY;
+                else
+                    state = State.LAST_BOUNDARY;
+            }
+            notifyContent(buffer, false, callback);
+        }
+
+        @Override
+        public void onFailure(Throwable failure)
+        {
+            if (subscription != null)
+                subscription.fail(failure);
+        }
+    }
+
+    private enum State
+    {
+        FIRST_BOUNDARY, HEADERS, CONTENT, MIDDLE_BOUNDARY, LAST_BOUNDARY, COMPLETE
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/OutputStreamContentProvider.java

@@ -72,7 +72,10 @@ import org.eclipse.jetty.util.Callback;
  *     output.write("some content".getBytes());
  * }
  * </pre>
+ *
+ * @deprecated use {@link OutputStreamRequestContent} instead
  */
+@Deprecated
 public class OutputStreamContentProvider implements AsyncContentProvider, Callback, Closeable
 {
     private final DeferredContentProvider deferred = new DeferredContentProvider();

jetty-client/src/main/java/org/eclipse/jetty/client/util/OutputStreamRequestContent.java

@@ -0,0 +1,125 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.concurrent.ExecutionException;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.util.FutureCallback;
+
+/**
+ * <p>A {@link Request.Content} that provides content asynchronously through an {@link OutputStream}
+ * similar to {@link AsyncRequestContent}.</p>
+ * <p>{@link OutputStreamRequestContent} can only be used in conjunction with
+ * {@link Request#send(Response.CompleteListener)} (and not with its blocking counterpart
+ * {@link Request#send()}) because it provides content asynchronously.</p>
+ * <p>Content must be provided by writing to the {@link #getOutputStream() output stream}
+ * that must be {@link OutputStream#close() closed} when all content has been provided.</p>
+ * <p>Example usage:</p>
+ * <pre>
+ * HttpClient httpClient = ...;
+ *
+ * // Use try-with-resources to autoclose the output stream.
+ * OutputStreamRequestContent content = new OutputStreamRequestContent();
+ * try (OutputStream output = content.getOutputStream())
+ * {
+ *     httpClient.newRequest("localhost", 8080)
+ *             .content(content)
+ *             .send(new Response.CompleteListener()
+ *             {
+ *                 &#64;Override
+ *                 public void onComplete(Result result)
+ *                 {
+ *                     // Your logic here
+ *                 }
+ *             });
+ *
+ *     // At a later time...
+ *     output.write("some content".getBytes());
+ *
+ *     // Even later...
+ *     output.write("more content".getBytes());
+ * } // Implicit call to output.close().
+ * </pre>
+ */
+public class OutputStreamRequestContent extends AsyncRequestContent
+{
+    private final AsyncOutputStream output;
+
+    public OutputStreamRequestContent()
+    {
+        this("application/octet-stream");
+    }
+
+    public OutputStreamRequestContent(String contentType)
+    {
+        super(contentType);
+        this.output = new AsyncOutputStream();
+    }
+
+    public OutputStream getOutputStream()
+    {
+        return output;
+    }
+
+    private class AsyncOutputStream extends OutputStream
+    {
+        @Override
+        public void write(int b) throws IOException
+        {
+            write(new byte[]{(byte)b}, 0, 1);
+        }
+
+        @Override
+        public void write(byte[] b, int off, int len) throws IOException
+        {
+            try
+            {
+                FutureCallback callback = new FutureCallback();
+                offer(ByteBuffer.wrap(b, off, len), callback);
+                callback.get();
+            }
+            catch (InterruptedException x)
+            {
+                throw new InterruptedIOException();
+            }
+            catch (ExecutionException x)
+            {
+                throw new IOException(x.getCause());
+            }
+        }
+
+        @Override
+        public void flush() throws IOException
+        {
+            OutputStreamRequestContent.this.flush();
+        }
+
+        @Override
+        public void close()
+        {
+            OutputStreamRequestContent.this.close();
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/PathContentProvider.java

@@ -43,7 +43,10 @@ import org.slf4j.LoggerFactory;
  * If a {@link ByteBufferPool} is provided via {@link #setByteBufferPool(ByteBufferPool)},
  * the buffer will be allocated from that pool, otherwise one buffer will be
  * allocated and used to read the file.</p>
+ *
+ * @deprecated use {@link PathRequestContent} instead.
  */
+@Deprecated
 public class PathContentProvider extends AbstractTypedContentProvider
 {
     private static final Logger LOG = LoggerFactory.getLogger(PathContentProvider.class);

jetty-client/src/main/java/org/eclipse/jetty/client/util/PathRequestContent.java

@@ -0,0 +1,177 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.io.ByteBufferPool;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.IO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>A {@link Request.Content} for files using JDK 7's {@code java.nio.file} APIs.</p>
+ * <p>It is possible to specify, at the constructor, a buffer size used to read
+ * content from the stream, by default 4096 bytes.
+ * If a {@link ByteBufferPool} is provided via {@link #setByteBufferPool(ByteBufferPool)},
+ * the buffer will be allocated from that pool, otherwise one buffer will be
+ * allocated and used to read the file.</p>
+ */
+public class PathRequestContent extends AbstractRequestContent
+{
+    private static final Logger LOG = LoggerFactory.getLogger(PathRequestContent.class);
+
+    private final Path filePath;
+    private final long fileSize;
+    private final int bufferSize;
+    private ByteBufferPool bufferPool;
+    private boolean useDirectByteBuffers = true;
+
+    public PathRequestContent(Path filePath) throws IOException
+    {
+        this(filePath, 4096);
+    }
+
+    public PathRequestContent(Path filePath, int bufferSize) throws IOException
+    {
+        this("application/octet-stream", filePath, bufferSize);
+    }
+
+    public PathRequestContent(String contentType, Path filePath) throws IOException
+    {
+        this(contentType, filePath, 4096);
+    }
+
+    public PathRequestContent(String contentType, Path filePath, int bufferSize) throws IOException
+    {
+        super(contentType);
+        if (!Files.isRegularFile(filePath))
+            throw new NoSuchFileException(filePath.toString());
+        if (!Files.isReadable(filePath))
+            throw new AccessDeniedException(filePath.toString());
+        this.filePath = filePath;
+        this.fileSize = Files.size(filePath);
+        this.bufferSize = bufferSize;
+    }
+
+    @Override
+    public long getLength()
+    {
+        return fileSize;
+    }
+
+    @Override
+    public boolean isReproducible()
+    {
+        return true;
+    }
+
+    public ByteBufferPool getByteBufferPool()
+    {
+        return bufferPool;
+    }
+
+    public void setByteBufferPool(ByteBufferPool byteBufferPool)
+    {
+        this.bufferPool = byteBufferPool;
+    }
+
+    public boolean isUseDirectByteBuffers()
+    {
+        return useDirectByteBuffers;
+    }
+
+    public void setUseDirectByteBuffers(boolean useDirectByteBuffers)
+    {
+        this.useDirectByteBuffers = useDirectByteBuffers;
+    }
+
+    @Override
+    protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
+    {
+        return new SubscriptionImpl(consumer, emitInitialContent);
+    }
+
+    private class SubscriptionImpl extends AbstractSubscription
+    {
+        private ReadableByteChannel channel;
+        private long readTotal;
+
+        private SubscriptionImpl(Consumer consumer, boolean emitInitialContent)
+        {
+            super(consumer, emitInitialContent);
+        }
+
+        @Override
+        protected boolean produceContent(Producer producer) throws IOException
+        {
+            ByteBuffer buffer;
+            boolean last;
+            if (channel == null)
+            {
+                channel = Files.newByteChannel(filePath, StandardOpenOption.READ);
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Opened file {}", filePath);
+            }
+
+            buffer = bufferPool == null
+                ? BufferUtil.allocate(bufferSize, isUseDirectByteBuffers())
+                : bufferPool.acquire(bufferSize, isUseDirectByteBuffers());
+
+            BufferUtil.clearToFill(buffer);
+            int read = channel.read(buffer);
+            BufferUtil.flipToFlush(buffer, 0);
+            if (LOG.isDebugEnabled())
+                LOG.debug("Read {} bytes from {}", read, filePath);
+            if (!channel.isOpen() && read < 0)
+                throw new EOFException("EOF reached for " + filePath);
+
+            if (read > 0)
+                readTotal += read;
+            last = readTotal == fileSize;
+            if (last)
+                IO.close(channel);
+            return producer.produce(buffer, last, Callback.from(() -> release(buffer)));
+        }
+
+        private void release(ByteBuffer buffer)
+        {
+            if (bufferPool != null)
+                bufferPool.release(buffer);
+        }
+
+        @Override
+        public void fail(Throwable failure)
+        {
+            super.fail(failure);
+            IO.close(channel);
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/StringContentProvider.java

@@ -28,7 +28,10 @@ import org.eclipse.jetty.client.api.ContentProvider;
  * <p>
  * It is possible to specify, at the constructor, an encoding used to convert
  * the string into bytes, by default UTF-8.
+ *
+ * @deprecated use {@link StringRequestContent} instead.
  */
+@Deprecated
 public class StringContentProvider extends BytesContentProvider
 {
     public StringContentProvider(String content)

jetty-client/src/main/java/org/eclipse/jetty/client/util/StringRequestContent.java

@@ -0,0 +1,52 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+import org.eclipse.jetty.client.api.Request;
+
+/**
+ * <p>A {@link Request.Content} for strings.</p>
+ * <p>It is possible to specify, at the constructor, an encoding used to convert
+ * the string into bytes, by default UTF-8.</p>
+ */
+public class StringRequestContent extends BytesRequestContent
+{
+    public StringRequestContent(String content)
+    {
+        this("text/plain;charset=UTF-8", content);
+    }
+
+    public StringRequestContent(String content, Charset encoding)
+    {
+        this("text/plain;charset=" + encoding.name(), content, encoding);
+    }
+
+    public StringRequestContent(String contentType, String content)
+    {
+        this(contentType, content, StandardCharsets.UTF_8);
+    }
+
+    public StringRequestContent(String contentType, String content, Charset encoding)
+    {
+        super(contentType, content.getBytes(encoding));
+    }
+}

jetty-client/src/test/java/org/eclipse/jetty/client/ClientConnectionCloseTest.java

@@ -29,8 +29,8 @@ import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.http.HttpConnectionOverHTTP;
-import org.eclipse.jetty.client.util.DeferredContentProvider;
-import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
+import org.eclipse.jetty.client.util.StringRequestContent;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpHeaderValue;
 import org.eclipse.jetty.http.HttpStatus;
@@ -87,7 +87,7 @@ public class ClientConnectionCloseTest extends AbstractHttpClientServerTest
         var request = client.newRequest(host, port)
             .scheme(scenario.getScheme())
             .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString())
-            .content(new StringContentProvider("0"))
+            .body(new StringRequestContent("0"))
             .onRequestSuccess(r ->
             {
                 HttpDestination destination = (HttpDestination)client.resolveDestination(r);
@@ -184,12 +184,12 @@ public class ClientConnectionCloseTest extends AbstractHttpClientServerTest
         String host = "localhost";
         int port = connector.getLocalPort();
 
-        DeferredContentProvider content = new DeferredContentProvider(ByteBuffer.allocate(8));
+        AsyncRequestContent content = new AsyncRequestContent(ByteBuffer.allocate(8));
         CountDownLatch resultLatch = new CountDownLatch(1);
         var request = client.newRequest(host, port)
             .scheme(scenario.getScheme())
             .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString())
-            .content(content)
+            .body(content)
             .idleTimeout(idleTimeout, TimeUnit.MILLISECONDS)
             .onRequestSuccess(r ->
             {

jetty-client/src/test/java/org/eclipse/jetty/client/ConnectionPoolTest.java

@@ -33,7 +33,7 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
-import org.eclipse.jetty.client.util.BytesContentProvider;
+import org.eclipse.jetty.client.util.BytesRequestContent;
 import org.eclipse.jetty.client.util.FutureResponseListener;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpMethod;
@@ -213,7 +213,7 @@ public class ConnectionPoolTest
                 break;
             case POST:
                 request.header(HttpHeader.CONTENT_LENGTH, String.valueOf(contentLength));
-                request.content(new BytesContentProvider(new byte[contentLength]));
+                request.body(new BytesRequestContent(new byte[contentLength]));
                 break;
             default:
                 throw new IllegalStateException();

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientAuthenticationTest.java

@@ -22,9 +22,7 @@ import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.nio.ByteBuffer;
-import java.util.Iterator;
 import java.util.List;
-import java.util.NoSuchElementException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -37,15 +35,15 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jetty.client.api.Authentication;
 import org.eclipse.jetty.client.api.Authentication.HeaderInfo;
 import org.eclipse.jetty.client.api.AuthenticationStore;
-import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Response.Listener;
 import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.client.util.AbstractAuthentication;
+import org.eclipse.jetty.client.util.AbstractRequestContent;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
 import org.eclipse.jetty.client.util.BasicAuthentication;
-import org.eclipse.jetty.client.util.DeferredContentProvider;
 import org.eclipse.jetty.client.util.DigestAuthentication;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpStatus;
@@ -60,6 +58,8 @@ import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
 import org.eclipse.jetty.util.Attributes;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.IO;
 import org.eclipse.jetty.util.URIUtil;
 import org.eclipse.jetty.util.security.Constraint;
@@ -460,7 +460,7 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
 
         CountDownLatch resultLatch = new CountDownLatch(1);
         byte[] data = new byte[]{'h', 'e', 'l', 'l', 'o'};
-        DeferredContentProvider content = new DeferredContentProvider(ByteBuffer.wrap(data))
+        AsyncRequestContent content = new AsyncRequestContent(ByteBuffer.wrap(data))
         {
             @Override
             public boolean isReproducible()
@@ -470,7 +470,7 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
         };
         Request request = client.newRequest(uri)
             .path("/secure")
-            .content(content);
+            .body(content);
         request.send(result ->
         {
             if (result.isSucceeded() && result.getResponse().getStatus() == HttpStatus.UNAUTHORIZED_401)
@@ -527,7 +527,7 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
         authenticationStore.addAuthentication(authentication);
 
         AtomicBoolean fail = new AtomicBoolean(true);
-        GeneratingContentProvider content = new GeneratingContentProvider(index ->
+        GeneratingRequestContent content = new GeneratingRequestContent(index ->
         {
             switch (index)
             {
@@ -546,9 +546,8 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
                         catch (InterruptedException ignored)
                         {
                         }
-
                         // Trigger request failure.
-                        throw new RuntimeException();
+                        throw new RuntimeException("explicitly_thrown_by_test");
                     }
                     else
                     {
@@ -563,7 +562,7 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
         client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .path("/secure")
-            .content(content)
+            .body(content)
             .onResponseSuccess(r -> authLatch.countDown())
             .send(result ->
             {
@@ -803,23 +802,16 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
         assertEquals(headerInfo.getParameter("nonce"), "1523430383=");
     }
 
-    private static class GeneratingContentProvider implements ContentProvider
+    private static class GeneratingRequestContent extends AbstractRequestContent
     {
-        private static final ByteBuffer DONE = ByteBuffer.allocate(0);
-
         private final IntFunction<ByteBuffer> generator;
 
-        private GeneratingContentProvider(IntFunction<ByteBuffer> generator)
+        private GeneratingRequestContent(IntFunction<ByteBuffer> generator)
         {
+            super("application/octet-stream");
             this.generator = generator;
         }
 
-        @Override
-        public long getLength()
-        {
-            return -1;
-        }
-
         @Override
         public boolean isReproducible()
         {
@@ -827,36 +819,32 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
         }
 
         @Override
-        public Iterator<ByteBuffer> iterator()
+        protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
         {
-            return new Iterator<ByteBuffer>()
+            return new SubscriptionImpl(consumer, emitInitialContent);
+        }
+
+        private class SubscriptionImpl extends AbstractSubscription
+        {
+            private int index;
+
+            public SubscriptionImpl(Consumer consumer, boolean emitInitialContent)
             {
-                private int index;
-                public ByteBuffer current;
+                super(consumer, emitInitialContent);
+            }
 
-                @Override
-                @SuppressWarnings("ReferenceEquality")
-                public boolean hasNext()
+            @Override
+            protected boolean produceContent(Producer producer)
+            {
+                ByteBuffer buffer = generator.apply(index++);
+                boolean last = false;
+                if (buffer == null)
                 {
-                    if (current == null)
-                    {
-                        current = generator.apply(index++);
-                        if (current == null)
-                            current = DONE;
-                    }
-                    return current != DONE;
+                    buffer = BufferUtil.EMPTY_BUFFER;
+                    last = true;
                 }
-
-                @Override
-                public ByteBuffer next()
-                {
-                    ByteBuffer result = current;
-                    current = null;
-                    if (result == null)
-                        throw new NoSuchElementException();
-                    return result;
-                }
-            };
+                return producer.produce(buffer, last, Callback.NOOP);
+            }
         }
     }
 }

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientFailureTest.java

@@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
 import org.eclipse.jetty.client.http.HttpConnectionOverHTTP;
-import org.eclipse.jetty.client.util.DeferredContentProvider;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.Server;
@@ -116,16 +116,16 @@ public class HttpClientFailureTest
         });
         client.start();
 
-        final CountDownLatch commitLatch = new CountDownLatch(1);
-        final CountDownLatch completeLatch = new CountDownLatch(1);
-        DeferredContentProvider content = new DeferredContentProvider();
+        CountDownLatch commitLatch = new CountDownLatch(1);
+        CountDownLatch completeLatch = new CountDownLatch(1);
+        AsyncRequestContent content = new AsyncRequestContent();
         client.newRequest("localhost", connector.getLocalPort())
             .onRequestCommit(request ->
             {
                 connectionRef.get().getEndPoint().close();
                 commitLatch.countDown();
             })
-            .content(content)
+            .body(content)
             .idleTimeout(2, TimeUnit.SECONDS)
             .send(result ->
             {

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientRedirectTest.java

@@ -36,7 +36,7 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
-import org.eclipse.jetty.client.util.ByteBufferContentProvider;
+import org.eclipse.jetty.client.util.ByteBufferRequestContent;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpStatus;
@@ -153,7 +153,7 @@ public class HttpClientRedirectTest extends AbstractHttpClientServerTest
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
             .path("/307/localhost/done")
-            .content(new ByteBufferContentProvider(ByteBuffer.wrap(data)))
+            .body(new ByteBufferRequestContent(ByteBuffer.wrap(data)))
             .timeout(5, TimeUnit.SECONDS)
             .send();
         assertNotNull(response);

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientSynchronizationTest.java

@@ -47,20 +47,22 @@ public class HttpClientSynchronizationTest extends AbstractHttpClientServerTest
         server.stop();
 
         int count = 10;
-        final CountDownLatch latch = new CountDownLatch(count);
+        CountDownLatch latch = new CountDownLatch(count);
         for (int i = 0; i < count; ++i)
         {
             Request request = client.newRequest("localhost", port)
-                .scheme(scenario.getScheme());
+                .scheme(scenario.getScheme())
+                .path("/" + i);
 
-            synchronized (this)
+            Object lock = this;
+            synchronized (lock)
             {
                 request.send(new Response.Listener.Adapter()
                 {
                     @Override
                     public void onFailure(Response response, Throwable failure)
                     {
-                        synchronized (HttpClientSynchronizationTest.this)
+                        synchronized (lock)
                         {
                             assertThat(failure, Matchers.instanceOf(ConnectException.class));
                             latch.countDown();
@@ -80,20 +82,22 @@ public class HttpClientSynchronizationTest extends AbstractHttpClientServerTest
         start(scenario, new EmptyServerHandler());
 
         int count = 10;
-        final CountDownLatch latch = new CountDownLatch(count);
+        CountDownLatch latch = new CountDownLatch(count);
         for (int i = 0; i < count; ++i)
         {
             Request request = client.newRequest("localhost", connector.getLocalPort())
-                .scheme(scenario.getScheme());
+                .scheme(scenario.getScheme())
+                .path("/" + i);
 
-            synchronized (this)
+            Object lock = this;
+            synchronized (lock)
             {
                 request.send(new Response.Listener.Adapter()
                 {
                     @Override
                     public void onComplete(Result result)
                     {
-                        synchronized (HttpClientSynchronizationTest.this)
+                        synchronized (lock)
                         {
                             assertFalse(result.isFailed());
                             latch.countDown();

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java

@@ -38,10 +38,8 @@ import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.NoSuchElementException;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Exchanger;
@@ -59,7 +57,6 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.client.api.Connection;
-import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Destination;
 import org.eclipse.jetty.client.api.Request;
@@ -67,11 +64,12 @@ import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
 import org.eclipse.jetty.client.http.HttpConnectionOverHTTP;
+import org.eclipse.jetty.client.util.AbstractRequestContent;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
 import org.eclipse.jetty.client.util.BufferingResponseListener;
-import org.eclipse.jetty.client.util.BytesContentProvider;
-import org.eclipse.jetty.client.util.DeferredContentProvider;
+import org.eclipse.jetty.client.util.BytesRequestContent;
 import org.eclipse.jetty.client.util.FutureResponseListener;
-import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.client.util.StringRequestContent;
 import org.eclipse.jetty.http.BadMessageException;
 import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpHeader;
@@ -231,7 +229,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         });
 
         String value1 = "\u20AC";
-        String paramValue1 = URLEncoder.encode(value1, "UTF-8");
+        String paramValue1 = URLEncoder.encode(value1, StandardCharsets.UTF_8);
         String query = paramName1 + "=" + paramValue1 + "&" + paramName2;
         ContentResponse response = client.GET(scenario.getScheme() + "://localhost:" + connector.getLocalPort() + "/?" + query);
 
@@ -268,9 +266,9 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         String value11 = "\u20AC";
         String value12 = "\u20AA";
         String value2 = "&";
-        String paramValue11 = URLEncoder.encode(value11, "UTF-8");
-        String paramValue12 = URLEncoder.encode(value12, "UTF-8");
-        String paramValue2 = URLEncoder.encode(value2, "UTF-8");
+        String paramValue11 = URLEncoder.encode(value11, StandardCharsets.UTF_8);
+        String paramValue12 = URLEncoder.encode(value12, StandardCharsets.UTF_8);
+        String paramValue2 = URLEncoder.encode(value2, StandardCharsets.UTF_8);
         String query = paramName1 + "=" + paramValue11 + "&" + paramName1 + "=" + paramValue12 + "&" + paramName2 + "=" + paramValue2;
         ContentResponse response = client.GET(scenario.getScheme() + "://localhost:" + connector.getLocalPort() + "/?" + query);
 
@@ -318,7 +316,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
     {
         String paramName = "a";
         String paramValue = "\u20AC";
-        String encodedParamValue = URLEncoder.encode(paramValue, "UTF-8");
+        String encodedParamValue = URLEncoder.encode(paramValue, StandardCharsets.UTF_8);
         start(scenario, new AbstractHandler()
         {
             @Override
@@ -372,7 +370,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
 
         ContentResponse response = client.POST(scenario.getScheme() + "://localhost:" + connector.getLocalPort() + "/?b=1")
             .param(paramName, paramValue)
-            .content(new BytesContentProvider(content))
+            .body(new BytesRequestContent(content))
             .timeout(5, TimeUnit.SECONDS)
             .send();
 
@@ -404,7 +402,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
                 if (!Arrays.equals(content, bytes))
                     request.abort(new Exception());
             })
-            .content(new BytesContentProvider(content))
+            .body(new BytesRequestContent(content))
             .timeout(5, TimeUnit.SECONDS)
             .send();
 
@@ -435,7 +433,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
                 buffer.get(bytes);
                 assertEquals(bytes[0], progress.getAndIncrement());
             })
-            .content(new BytesContentProvider(new byte[]{0}, new byte[]{1}, new byte[]{2}, new byte[]{3}, new byte[]{4}))
+            .body(new BytesRequestContent(new byte[]{0}, new byte[]{1}, new byte[]{2}, new byte[]{3}, new byte[]{4}))
             .timeout(5, TimeUnit.SECONDS)
             .send();
 
@@ -511,7 +509,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
 
         client.setMaxConnectionsPerDestination(1);
 
-        try (StacklessLogging stackless = new StacklessLogging(org.eclipse.jetty.server.HttpChannel.class))
+        try (StacklessLogging ignored = new StacklessLogging(org.eclipse.jetty.server.HttpChannel.class))
         {
             CountDownLatch latch = new CountDownLatch(2);
             client.newRequest("localhost", connector.getLocalPort())
@@ -630,36 +628,23 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         CountDownLatch latch = new CountDownLatch(1);
         client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
-            // The second ByteBuffer set to null will throw an exception
-            .content(new ContentProvider()
+            .body(new AbstractRequestContent("application/octet-stream")
             {
                 @Override
-                public long getLength()
+                protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
                 {
-                    return -1;
-                }
-
-                @Override
-                public Iterator<ByteBuffer> iterator()
-                {
-                    return new Iterator<>()
+                    return new AbstractSubscription(consumer, emitInitialContent)
                     {
-                        @Override
-                        public boolean hasNext()
-                        {
-                            return true;
-                        }
+                        private int count;
 
                         @Override
-                        public ByteBuffer next()
+                        protected boolean produceContent(Producer producer) throws Exception
                         {
-                            throw new NoSuchElementException("explicitly_thrown_by_test");
-                        }
-
-                        @Override
-                        public void remove()
-                        {
-                            throw new UnsupportedOperationException();
+                            if (count == 2)
+                                throw new IOException("explicitly_thrown_by_test");
+                            ByteBuffer buffer = BufferUtil.allocate(512);
+                            ++count;
+                            return producer.produce(buffer, false, Callback.NOOP);
                         }
                     };
                 }
@@ -1244,7 +1229,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
             public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException
             {
                 // Send the headers at this point, then write the content
-                byte[] content = "TEST".getBytes("UTF-8");
+                byte[] content = "TEST".getBytes(StandardCharsets.UTF_8);
                 response.setContentLength(content.length);
                 response.flushBuffer();
                 response.getOutputStream().write(content);
@@ -1413,11 +1398,11 @@ public class HttpClientTest extends AbstractHttpClientServerTest
             }
         });
 
-        DeferredContentProvider content = new DeferredContentProvider(ByteBuffer.wrap(new byte[]{0}));
+        AsyncRequestContent content = new AsyncRequestContent(ByteBuffer.wrap(new byte[]{0}));
         Request request = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .version(version)
-            .content(content);
+            .body(content);
         FutureResponseListener listener = new FutureResponseListener(request);
         request.send(listener);
         // Wait some time to simulate a slow request.
@@ -1530,7 +1515,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         client = new HttpClient(new HttpClientTransportOverHTTP(clientConnector)
         {
             @Override
-            public org.eclipse.jetty.io.Connection newConnection(EndPoint endPoint, Map<String, Object> context) throws IOException
+            public org.eclipse.jetty.io.Connection newConnection(EndPoint endPoint, Map<String, Object> context)
             {
                 return new HttpConnectionOverHTTP(endPoint, context)
                 {
@@ -1658,7 +1643,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         assertCopyRequest(client.newRequest("http://example.com/some/url")
             .method(HttpMethod.HEAD)
             .version(HttpVersion.HTTP_2)
-            .content(new StringContentProvider("some string"))
+            .body(new StringRequestContent("some string"))
             .timeout(321, TimeUnit.SECONDS)
             .idleTimeout(2221, TimeUnit.SECONDS)
             .followRedirects(true)
@@ -1668,7 +1653,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         assertCopyRequest(client.newRequest("https://example.com")
             .method(HttpMethod.POST)
             .version(HttpVersion.HTTP_1_0)
-            .content(new StringContentProvider("some other string"))
+            .body(new StringRequestContent("some other string"))
             .timeout(123231, TimeUnit.SECONDS)
             .idleTimeout(232342, TimeUnit.SECONDS)
             .followRedirects(false)
@@ -1797,7 +1782,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         start(scenario, new AbstractHandler()
         {
             @Override
-            public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException
             {
                 baseRequest.setHandled(true);
                 ServletOutputStream output = response.getOutputStream();
@@ -1845,7 +1830,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         assertEquals(original.getURI(), copy.getURI());
         assertEquals(original.getMethod(), copy.getMethod());
         assertEquals(original.getVersion(), copy.getVersion());
-        assertEquals(original.getContent(), copy.getContent());
+        assertEquals(original.getBody(), copy.getBody());
         assertEquals(original.getIdleTimeout(), copy.getIdleTimeout());
         assertEquals(original.getTimeout(), copy.getTimeout());
         assertEquals(original.isFollowRedirects(), copy.isFollowRedirects());
@@ -1910,7 +1895,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
                 .scheme(scheme)
                 .method("POST")
                 .param("attempt", String.valueOf(retries))
-                .content(new StringContentProvider("0123456789ABCDEF"))
+                .body(new StringRequestContent("0123456789ABCDEF"))
                 .send(this);
         }
     }

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientUploadDuringServerShutdownTest.java

@@ -32,7 +32,7 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jetty.client.http.HttpChannelOverHTTP;
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
 import org.eclipse.jetty.client.http.HttpConnectionOverHTTP;
-import org.eclipse.jetty.client.util.BytesContentProvider;
+import org.eclipse.jetty.client.util.BytesRequestContent;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.Server;
@@ -116,7 +116,7 @@ public class HttpClientUploadDuringServerShutdownTest
                 {
                     int length = 16 * 1024 * 1024 + random.nextInt(16 * 1024 * 1024);
                     client.newRequest("localhost", 8888)
-                            .content(new BytesContentProvider(new byte[length]))
+                            .body(new BytesRequestContent(new byte[length]))
                             .send(result -> latch.countDown());
                     long sleep = 1 + random.nextInt(10);
                     TimeUnit.MILLISECONDS.sleep(sleep);

jetty-client/src/test/java/org/eclipse/jetty/client/HttpConnectionLifecycleTest.java

@@ -32,7 +32,7 @@ import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
-import org.eclipse.jetty.client.util.ByteBufferContentProvider;
+import org.eclipse.jetty.client.util.ByteBufferRequestContent;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.logging.StacklessLogging;
@@ -411,7 +411,7 @@ public class HttpConnectionLifecycleTest extends AbstractHttpClientServerTest
             CountDownLatch latch = new CountDownLatch(1);
             ByteBuffer buffer = ByteBuffer.allocate(16 * 1024 * 1024);
             Arrays.fill(buffer.array(), (byte)'x');
-            request.content(new ByteBufferContentProvider(buffer))
+            request.body(new ByteBufferRequestContent(buffer))
                 .send(new Response.Listener.Adapter()
                 {
                     @Override

jetty-client/src/test/java/org/eclipse/jetty/client/HttpRequestAbortTest.java

@@ -32,7 +32,7 @@ import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Result;
-import org.eclipse.jetty.client.util.ByteBufferContentProvider;
+import org.eclipse.jetty.client.util.ByteBufferRequestContent;
 import org.eclipse.jetty.logging.StacklessLogging;
 import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.eclipse.jetty.util.IO;
@@ -268,7 +268,7 @@ public class HttpRequestAbortTest extends AbstractHttpClientServerTest
             {
                 aborted.set(r.abort(cause));
                 latch.countDown();
-            }).content(new ByteBufferContentProvider(ByteBuffer.wrap(new byte[]{0}), ByteBuffer.wrap(new byte[]{1}))
+            }).body(new ByteBufferRequestContent(ByteBuffer.wrap(new byte[]{0}), ByteBuffer.wrap(new byte[]{1}))
             {
                 @Override
                 public long getLength()
@@ -323,7 +323,7 @@ public class HttpRequestAbortTest extends AbstractHttpClientServerTest
                 {
                     aborted.set(r.abort(cause));
                     latch.countDown();
-                }).content(new ByteBufferContentProvider(ByteBuffer.wrap(new byte[]{0}), ByteBuffer.wrap(new byte[]{1}))
+                }).body(new ByteBufferRequestContent(ByteBuffer.wrap(new byte[]{0}), ByteBuffer.wrap(new byte[]{1}))
                 {
                     @Override
                     public long getLength()

jetty-client/src/test/java/org/eclipse/jetty/client/HttpResponseAbortTest.java

@@ -24,11 +24,10 @@ import java.nio.ByteBuffer;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.eclipse.jetty.client.util.DeferredContentProvider;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -104,7 +103,7 @@ public class HttpResponseAbortTest extends AbstractHttpClientServerTest
         start(scenario, new AbstractHandler()
         {
             @Override
-            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
             {
                 try
                 {
@@ -141,7 +140,7 @@ public class HttpResponseAbortTest extends AbstractHttpClientServerTest
         start(scenario, new AbstractHandler()
         {
             @Override
-            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
             {
                 try
                 {
@@ -159,18 +158,18 @@ public class HttpResponseAbortTest extends AbstractHttpClientServerTest
             }
         });
 
-        final DeferredContentProvider contentProvider = new DeferredContentProvider(ByteBuffer.allocate(1));
-        final AtomicInteger completes = new AtomicInteger();
-        final CountDownLatch completeLatch = new CountDownLatch(1);
+        AsyncRequestContent requestContent = new AsyncRequestContent(ByteBuffer.allocate(1));
+        AtomicInteger completes = new AtomicInteger();
+        CountDownLatch completeLatch = new CountDownLatch(1);
         client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
-            .content(contentProvider)
+            .body(requestContent)
             .onResponseContent((response, content) ->
             {
                 try
                 {
                     response.abort(new Exception());
-                    contentProvider.close();
+                    requestContent.close();
                     // Delay to let the request side to finish its processing.
                     Thread.sleep(1000);
                 }

jetty-client/src/test/java/org/eclipse/jetty/client/NetworkTrafficListenerTest.java

@@ -0,0 +1,525 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.Socket;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectableChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
+import org.eclipse.jetty.client.util.FormRequestContent;
+import org.eclipse.jetty.client.util.StringRequestContent;
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpHeaderValue;
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.io.ClientConnector;
+import org.eclipse.jetty.io.EndPoint;
+import org.eclipse.jetty.io.ManagedSelector;
+import org.eclipse.jetty.io.NetworkTrafficListener;
+import org.eclipse.jetty.io.NetworkTrafficSocketChannelEndPoint;
+import org.eclipse.jetty.io.SelectorManager;
+import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.NetworkTrafficServerConnector;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Fields;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class NetworkTrafficListenerTest
+{
+    private static final String END_OF_CONTENT = "~";
+
+    private Server server;
+    private NetworkTrafficServerConnector connector;
+    private NetworkTrafficHttpClient client;
+
+    private void start(Handler handler) throws Exception
+    {
+        startServer(handler);
+        startClient();
+    }
+
+    private void startServer(Handler handler) throws Exception
+    {
+        server = new Server();
+        connector = new NetworkTrafficServerConnector(server);
+        connector.getConnectionFactory(HttpConfiguration.ConnectionFactory.class).getHttpConfiguration().setSendDateHeader(false);
+        connector.getConnectionFactory(HttpConfiguration.ConnectionFactory.class).getHttpConfiguration().setSendServerVersion(false);
+        server.addConnector(connector);
+        server.setHandler(handler);
+        server.start();
+    }
+
+    private void startClient() throws Exception
+    {
+        client = new NetworkTrafficHttpClient(new AtomicReference<>());
+        client.start();
+    }
+
+    @AfterEach
+    public void dispose() throws Exception
+    {
+        if (client != null)
+            client.stop();
+        if (server != null)
+            server.stop();
+    }
+
+    @Test
+    public void testOpenedClosedAreInvoked() throws Exception
+    {
+        startServer(null);
+
+        CountDownLatch openedLatch = new CountDownLatch(1);
+        CountDownLatch closedLatch = new CountDownLatch(1);
+        connector.setNetworkTrafficListener(new NetworkTrafficListener()
+        {
+            public volatile Socket socket;
+
+            @Override
+            public void opened(Socket socket)
+            {
+                this.socket = socket;
+                openedLatch.countDown();
+            }
+
+            @Override
+            public void closed(Socket socket)
+            {
+                if (this.socket == socket)
+                    closedLatch.countDown();
+            }
+        });
+        int port = connector.getLocalPort();
+
+        // Connect to the server
+        try (Socket ignored = new Socket("localhost", port))
+        {
+            assertTrue(openedLatch.await(10, TimeUnit.SECONDS));
+        }
+        assertTrue(closedLatch.await(10, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testTrafficWithNoResponseContentOnNonPersistentConnection() throws Exception
+    {
+        start(new AbstractHandler()
+        {
+            @Override
+            public void handle(String uri, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse)
+            {
+                request.setHandled(true);
+            }
+        });
+
+        AtomicReference<String> serverIncoming = new AtomicReference<>("");
+        CountDownLatch serverIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> serverOutgoing = new AtomicReference<>("");
+        CountDownLatch serverOutgoingLatch = new CountDownLatch(1);
+        connector.setNetworkTrafficListener(new NetworkTrafficListener()
+        {
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                serverIncoming.set(serverIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverIncomingLatch.countDown();
+            }
+
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                serverOutgoing.set(serverOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverOutgoingLatch.countDown();
+            }
+        });
+
+        AtomicReference<String> clientIncoming = new AtomicReference<>("");
+        CountDownLatch clientIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> clientOutgoing = new AtomicReference<>("");
+        CountDownLatch clientOutgoingLatch = new CountDownLatch(1);
+        client.listener.set(new NetworkTrafficListener()
+        {
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                clientOutgoing.set(clientOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientOutgoingLatch.countDown();
+            }
+
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                clientIncoming.set(clientIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientIncomingLatch.countDown();
+            }
+        });
+
+        ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
+            .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString())
+            .send();
+        assertEquals(HttpStatus.OK_200, response.getStatus());
+
+        assertTrue(clientOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(clientIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertEquals(clientOutgoing.get(), serverIncoming.get());
+        assertEquals(serverOutgoing.get(), clientIncoming.get());
+    }
+
+    @Test
+    public void testTrafficWithResponseContentOnPersistentConnection() throws Exception
+    {
+        String responseContent = "response_content" + END_OF_CONTENT;
+        start(new AbstractHandler()
+        {
+            @Override
+            public void handle(String uri, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException
+            {
+                request.setHandled(true);
+                ServletOutputStream output = servletResponse.getOutputStream();
+                output.write(responseContent.getBytes(StandardCharsets.UTF_8));
+            }
+        });
+
+        AtomicReference<String> serverIncoming = new AtomicReference<>("");
+        CountDownLatch serverIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> serverOutgoing = new AtomicReference<>("");
+        CountDownLatch serverOutgoingLatch = new CountDownLatch(1);
+        connector.setNetworkTrafficListener(new NetworkTrafficListener()
+        {
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                serverIncoming.set(serverIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverIncomingLatch.countDown();
+            }
+
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                serverOutgoing.set(serverOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverOutgoingLatch.countDown();
+            }
+        });
+
+        AtomicReference<String> clientIncoming = new AtomicReference<>("");
+        CountDownLatch clientIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> clientOutgoing = new AtomicReference<>("");
+        CountDownLatch clientOutgoingLatch = new CountDownLatch(1);
+        client.listener.set(new NetworkTrafficListener()
+        {
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                clientOutgoing.set(clientOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientOutgoingLatch.countDown();
+            }
+
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                clientIncoming.set(clientIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientIncomingLatch.countDown();
+            }
+        });
+
+        ContentResponse response = client.newRequest("localhost", connector.getLocalPort()).send();
+        assertEquals(HttpStatus.OK_200, response.getStatus());
+        assertEquals(responseContent, response.getContentAsString());
+
+        assertTrue(clientOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(clientIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertEquals(clientOutgoing.get(), serverIncoming.get());
+        assertEquals(serverOutgoing.get(), clientIncoming.get());
+    }
+
+    @Test
+    public void testTrafficWithResponseContentChunkedOnPersistentConnection() throws Exception
+    {
+        String responseContent = "response_content";
+        String responseChunk1 = responseContent.substring(0, responseContent.length() / 2);
+        String responseChunk2 = responseContent.substring(responseContent.length() / 2);
+        start(new AbstractHandler()
+        {
+            @Override
+            public void handle(String uri, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException
+            {
+                request.setHandled(true);
+                ServletOutputStream output = servletResponse.getOutputStream();
+                output.write(responseChunk1.getBytes(StandardCharsets.UTF_8));
+                output.flush();
+                output.write(responseChunk2.getBytes(StandardCharsets.UTF_8));
+                output.flush();
+            }
+        });
+
+        AtomicReference<String> serverIncoming = new AtomicReference<>("");
+        CountDownLatch serverIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> serverOutgoing = new AtomicReference<>("");
+        CountDownLatch serverOutgoingLatch = new CountDownLatch(1);
+        connector.setNetworkTrafficListener(new NetworkTrafficListener()
+        {
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                serverIncoming.set(serverIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverIncomingLatch.countDown();
+            }
+
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                serverOutgoing.set(serverOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                if (serverOutgoing.get().endsWith("\r\n0\r\n\r\n"))
+                    serverOutgoingLatch.countDown();
+            }
+        });
+
+        AtomicReference<String> clientIncoming = new AtomicReference<>("");
+        CountDownLatch clientIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> clientOutgoing = new AtomicReference<>("");
+        CountDownLatch clientOutgoingLatch = new CountDownLatch(1);
+        client.listener.set(new NetworkTrafficListener()
+        {
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                clientOutgoing.set(clientOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientOutgoingLatch.countDown();
+            }
+
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                clientIncoming.set(clientIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                if (clientIncoming.get().endsWith("\r\n0\r\n\r\n"))
+                    clientIncomingLatch.countDown();
+            }
+        });
+
+        ContentResponse response = client.newRequest("localhost", connector.getLocalPort()).send();
+        assertEquals(HttpStatus.OK_200, response.getStatus());
+
+        assertTrue(clientOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(clientIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertEquals(clientOutgoing.get(), serverIncoming.get());
+        assertEquals(serverOutgoing.get(), clientIncoming.get());
+    }
+
+    @Test
+    public void testTrafficWithRequestContentWithResponseRedirectOnPersistentConnection() throws Exception
+    {
+        String location = "/redirect";
+        start(new AbstractHandler()
+        {
+            @Override
+            public void handle(String uri, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException
+            {
+                request.setHandled(true);
+                servletResponse.sendRedirect(location);
+            }
+        });
+
+        AtomicReference<String> serverIncoming = new AtomicReference<>("");
+        CountDownLatch serverIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> serverOutgoing = new AtomicReference<>("");
+        CountDownLatch serverOutgoingLatch = new CountDownLatch(1);
+        connector.setNetworkTrafficListener(new NetworkTrafficListener()
+        {
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                serverIncoming.set(serverIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverIncomingLatch.countDown();
+            }
+
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                serverOutgoing.set(serverOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverOutgoingLatch.countDown();
+            }
+        });
+
+        AtomicReference<String> clientIncoming = new AtomicReference<>("");
+        CountDownLatch clientIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> clientOutgoing = new AtomicReference<>("");
+        CountDownLatch clientOutgoingLatch = new CountDownLatch(1);
+        client.listener.set(new NetworkTrafficListener()
+        {
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                clientOutgoing.set(clientOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientOutgoingLatch.countDown();
+            }
+
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                clientIncoming.set(clientIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientIncomingLatch.countDown();
+            }
+        });
+
+        client.setFollowRedirects(false);
+        Fields fields = new Fields();
+        fields.put("a", "1");
+        fields.put("b", "2");
+        ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
+            .body(new FormRequestContent(fields))
+            .send();
+        assertEquals(HttpStatus.FOUND_302, response.getStatus());
+
+        assertTrue(clientOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(clientIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertEquals(clientOutgoing.get(), serverIncoming.get());
+        assertEquals(serverOutgoing.get(), clientIncoming.get());
+    }
+
+    @Test
+    public void testTrafficWithBigRequestContentOnPersistentConnection() throws Exception
+    {
+        start(new AbstractHandler()
+        {
+            @Override
+            public void handle(String uri, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException
+            {
+                // Read and discard the request body to make the test more
+                // reliable, otherwise there is a race between request body
+                // upload and response download
+                InputStream input = servletRequest.getInputStream();
+                byte[] buffer = new byte[4096];
+                while (true)
+                {
+                    int read = input.read(buffer);
+                    if (read < 0)
+                        break;
+                }
+                request.setHandled(true);
+            }
+        });
+
+        AtomicReference<String> serverIncoming = new AtomicReference<>("");
+        AtomicReference<String> serverOutgoing = new AtomicReference<>("");
+        CountDownLatch serverOutgoingLatch = new CountDownLatch(1);
+        connector.setNetworkTrafficListener(new NetworkTrafficListener()
+        {
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                serverIncoming.set(serverIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+            }
+
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                serverOutgoing.set(serverOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverOutgoingLatch.countDown();
+            }
+        });
+
+        AtomicReference<String> clientIncoming = new AtomicReference<>("");
+        CountDownLatch clientIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> clientOutgoing = new AtomicReference<>("");
+        client.listener.set(new NetworkTrafficListener()
+        {
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                clientOutgoing.set(clientOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+            }
+
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                clientIncoming.set(clientIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientIncomingLatch.countDown();
+            }
+        });
+
+        // Generate a large request content.
+        String requestContent = "0123456789ABCDEF";
+        for (int i = 0; i < 16; ++i)
+        {
+            requestContent += requestContent;
+        }
+
+        ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
+            .body(new StringRequestContent(requestContent))
+            .send();
+        assertEquals(HttpStatus.OK_200, response.getStatus());
+
+        assertTrue(serverOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(clientIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertEquals(clientOutgoing.get(), serverIncoming.get());
+        assertTrue(clientOutgoing.get().length() > requestContent.length());
+        assertEquals(serverOutgoing.get(), clientIncoming.get());
+    }
+
+    private static class NetworkTrafficHttpClient extends HttpClient
+    {
+        private final AtomicReference<NetworkTrafficListener> listener;
+
+        private NetworkTrafficHttpClient(AtomicReference<NetworkTrafficListener> listener)
+        {
+            super(new HttpClientTransportOverHTTP(new ClientConnector()
+            {
+                @Override
+                protected SelectorManager newSelectorManager()
+                {
+                    return new ClientSelectorManager(getExecutor(), getScheduler(), getSelectors())
+                    {
+                        @Override
+                        protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey selectionKey)
+                        {
+                            return new NetworkTrafficSocketChannelEndPoint(channel, selector, selectionKey, getScheduler(), getIdleTimeout().toMillis(), listener.get());
+                        }
+                    };
+                }
+            }));
+            this.listener = listener;
+        }
+    }
+}

jetty-client/src/test/java/org/eclipse/jetty/client/api/Usage.java

@@ -32,12 +32,12 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
 import org.eclipse.jetty.client.util.BasicAuthentication;
-import org.eclipse.jetty.client.util.DeferredContentProvider;
 import org.eclipse.jetty.client.util.FutureResponseListener;
-import org.eclipse.jetty.client.util.InputStreamContentProvider;
+import org.eclipse.jetty.client.util.InputStreamRequestContent;
 import org.eclipse.jetty.client.util.InputStreamResponseListener;
-import org.eclipse.jetty.client.util.OutputStreamContentProvider;
+import org.eclipse.jetty.client.util.OutputStreamRequestContent;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.util.FuturePromise;
@@ -101,16 +101,12 @@ public class Usage
 
         client.newRequest("localhost", 8080)
             // Send asynchronously
-            .send(new Response.CompleteListener()
+            .send(result ->
             {
-                @Override
-                public void onComplete(Result result)
+                if (result.isSucceeded())
                 {
-                    if (result.isSucceeded())
-                    {
-                        responseRef.set(result.getResponse());
-                        latch.countDown();
-                    }
+                    responseRef.set(result.getResponse());
+                    latch.countDown();
                 }
             });
 
@@ -278,7 +274,7 @@ public class Usage
 
         ContentResponse response = client.newRequest("localhost", 8080)
             // Provide the content as InputStream
-            .content(new InputStreamContentProvider(input))
+            .body(new InputStreamRequestContent(input))
             .send();
 
         assertEquals(200, response.getStatus());
@@ -290,11 +286,11 @@ public class Usage
         HttpClient client = new HttpClient();
         client.start();
 
-        OutputStreamContentProvider content = new OutputStreamContentProvider();
+        OutputStreamRequestContent content = new OutputStreamRequestContent();
         try (OutputStream output = content.getOutputStream())
         {
             client.newRequest("localhost", 8080)
-                    .content(content)
+                    .body(content)
                     .send(result -> assertEquals(200, result.getResponse().getStatus()));
 
             output.write(new byte[1024]);
@@ -308,15 +304,15 @@ public class Usage
     public void testProxyUsage() throws Exception
     {
         // In proxies, we receive the headers but not the content, so we must be able to send the request with
-        // a lazy content provider that does not block request.send(...)
+        // a lazy request content that does not block request.send(...)
 
         HttpClient client = new HttpClient();
         client.start();
 
-        final AtomicBoolean sendContent = new AtomicBoolean(true);
-        DeferredContentProvider async = new DeferredContentProvider(ByteBuffer.wrap(new byte[]{0, 1, 2}));
+        AtomicBoolean sendContent = new AtomicBoolean(true);
+        AsyncRequestContent async = new AsyncRequestContent(ByteBuffer.wrap(new byte[]{0, 1, 2}));
         client.newRequest("localhost", 8080)
-            .content(async)
+            .body(async)
             .send(new Response.Listener.Adapter()
             {
                 @Override

jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpSenderOverHTTPTest.java

@@ -33,7 +33,7 @@ import org.eclipse.jetty.client.api.Connection;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
-import org.eclipse.jetty.client.util.ByteBufferContentProvider;
+import org.eclipse.jetty.client.util.ByteBufferRequestContent;
 import org.eclipse.jetty.io.ByteArrayEndPoint;
 import org.eclipse.jetty.util.Promise;
 import org.hamcrest.Matchers;
@@ -201,7 +201,7 @@ public class HttpSenderOverHTTPTest
         HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<Connection>());
         Request request = client.newRequest(URI.create("http://localhost/"));
         String content = "abcdef";
-        request.content(new ByteBufferContentProvider(ByteBuffer.wrap(content.getBytes(StandardCharsets.UTF_8))));
+        request.body(new ByteBufferRequestContent(ByteBuffer.wrap(content.getBytes(StandardCharsets.UTF_8))));
         final CountDownLatch headersLatch = new CountDownLatch(1);
         final CountDownLatch successLatch = new CountDownLatch(1);
         request.listener(new Request.Listener.Adapter()
@@ -237,7 +237,7 @@ public class HttpSenderOverHTTPTest
         Request request = client.newRequest(URI.create("http://localhost/"));
         String content1 = "0123456789";
         String content2 = "abcdef";
-        request.content(new ByteBufferContentProvider(ByteBuffer.wrap(content1.getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(content2.getBytes(StandardCharsets.UTF_8))));
+        request.body(new ByteBufferRequestContent(ByteBuffer.wrap(content1.getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(content2.getBytes(StandardCharsets.UTF_8))));
         final CountDownLatch headersLatch = new CountDownLatch(1);
         final CountDownLatch successLatch = new CountDownLatch(1);
         request.listener(new Request.Listener.Adapter()
@@ -273,7 +273,7 @@ public class HttpSenderOverHTTPTest
         Request request = client.newRequest(URI.create("http://localhost/"));
         String content1 = "0123456789";
         String content2 = "ABCDEF";
-        request.content(new ByteBufferContentProvider(ByteBuffer.wrap(content1.getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(content2.getBytes(StandardCharsets.UTF_8)))
+        request.body(new ByteBufferRequestContent(ByteBuffer.wrap(content1.getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(content2.getBytes(StandardCharsets.UTF_8)))
         {
             @Override
             public long getLength()

jetty-client/src/test/java/org/eclipse/jetty/client/util/AsyncRequestContentTest.java

@@ -0,0 +1,151 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.Callback;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class AsyncRequestContentTest
+{
+    private ExecutorService executor;
+
+    @BeforeEach
+    public void prepare()
+    {
+        executor = Executors.newCachedThreadPool();
+    }
+
+    @AfterEach
+    public void dispose()
+    {
+        executor.shutdownNow();
+    }
+
+    @Test
+    public void testWhenEmptyFlushDoesNotBlock() throws Exception
+    {
+        AsyncRequestContent content = new AsyncRequestContent();
+
+        Future<?> task = executor.submit(() ->
+        {
+            content.flush();
+            return null;
+        });
+
+        assertTrue(await(task, 5000));
+    }
+
+    @Test
+    public void testOfferFlushDemandBlocksUntilSucceeded() throws Exception
+    {
+        AsyncRequestContent content = new AsyncRequestContent();
+        content.offer(ByteBuffer.allocate(1));
+
+        Future<?> task = executor.submit(() ->
+        {
+            content.flush();
+            return null;
+        });
+
+        // Wait until flush() blocks.
+        assertFalse(await(task, 500));
+
+        AtomicReference<Callback> callbackRef = new AtomicReference<>();
+        content.subscribe((buffer, last, callback) -> callbackRef.set(callback), true).demand();
+
+        // Flush should block until the callback is succeeded.
+        assertFalse(await(task, 500));
+
+        callbackRef.get().succeeded();
+
+        // Flush should return.
+        assertTrue(await(task, 5000));
+    }
+
+    @Test
+    public void testCloseFlushDoesNotBlock() throws Exception
+    {
+        AsyncRequestContent content = new AsyncRequestContent();
+        content.close();
+
+        Future<?> task = executor.submit(() ->
+        {
+            content.flush();
+            return null;
+        });
+
+        assertTrue(await(task, 5000));
+    }
+
+    @Test
+    public void testStallThenCloseProduces() throws Exception
+    {
+        AsyncRequestContent content = new AsyncRequestContent();
+
+        CountDownLatch latch = new CountDownLatch(1);
+        Request.Content.Subscription subscription = content.subscribe((buffer, last, callback) ->
+        {
+            callback.succeeded();
+            if (last)
+                latch.countDown();
+        }, true);
+
+        // Demand the initial content.
+        subscription.demand();
+
+        // Content must not be the last one.
+        assertFalse(latch.await(1, TimeUnit.SECONDS));
+
+        // Demand more content, now we are stalled.
+        subscription.demand();
+
+        // Close, we must be notified.
+        content.close();
+
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
+
+    private boolean await(Future<?> task, long time) throws Exception
+    {
+        try
+        {
+            task.get(time, TimeUnit.MILLISECONDS);
+            return true;
+        }
+        catch (TimeoutException x)
+        {
+            return false;
+        }
+    }
+}

jetty-client/src/test/java/org/eclipse/jetty/client/util/DeferredContentProviderTest.java

@@ -1,151 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-package org.eclipse.jetty.client.util;
-
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import org.eclipse.jetty.util.Callback;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-public class DeferredContentProviderTest
-{
-    private ExecutorService executor;
-
-    @BeforeEach
-    public void prepare() throws Exception
-    {
-        executor = Executors.newCachedThreadPool();
-    }
-
-    @AfterEach
-    public void dispose() throws Exception
-    {
-        executor.shutdownNow();
-    }
-
-    @Test
-    public void testWhenEmptyFlushDoesNotBlock() throws Exception
-    {
-        final DeferredContentProvider provider = new DeferredContentProvider();
-
-        Future<?> task = executor.submit(new Callable<Object>()
-        {
-            @Override
-            public Object call() throws Exception
-            {
-                provider.flush();
-                return null;
-            }
-        });
-
-        assertTrue(await(task, 5, TimeUnit.SECONDS));
-    }
-
-    @Test
-    public void testOfferFlushBlocksUntilSucceeded() throws Exception
-    {
-        final DeferredContentProvider provider = new DeferredContentProvider();
-        Iterator<ByteBuffer> iterator = provider.iterator();
-
-        provider.offer(ByteBuffer.allocate(0));
-
-        Future<?> task = executor.submit(new Callable<Object>()
-        {
-            @Override
-            public Object call() throws Exception
-            {
-                provider.flush();
-                return null;
-            }
-        });
-
-        // Wait until flush() blocks.
-        assertFalse(await(task, 1, TimeUnit.SECONDS));
-
-        // Consume the content and succeed the callback.
-        iterator.next();
-        ((Callback)iterator).succeeded();
-
-        // Flush should return.
-        assertTrue(await(task, 5, TimeUnit.SECONDS));
-    }
-
-    @Test
-    public void testCloseFlushDoesNotBlock() throws Exception
-    {
-        final DeferredContentProvider provider = new DeferredContentProvider();
-
-        provider.close();
-
-        Future<?> task = executor.submit(new Callable<Object>()
-        {
-            @Override
-            public Object call() throws Exception
-            {
-                provider.flush();
-                return null;
-            }
-        });
-
-        // Wait until flush() blocks.
-        assertTrue(await(task, 5, TimeUnit.SECONDS));
-    }
-
-    @Test
-    public void testCloseNextHasNextReturnsFalse() throws Exception
-    {
-        DeferredContentProvider provider = new DeferredContentProvider();
-        Iterator<ByteBuffer> iterator = provider.iterator();
-
-        provider.close();
-
-        assertFalse(iterator.hasNext());
-
-        assertThrows(NoSuchElementException.class, () -> iterator.next());
-
-        assertFalse(iterator.hasNext());
-    }
-
-    private boolean await(Future<?> task, long time, TimeUnit unit) throws Exception
-    {
-        try
-        {
-            task.get(time, unit);
-            return true;
-        }
-        catch (TimeoutException x)
-        {
-            return false;
-        }
-    }
-}

jetty-client/src/test/java/org/eclipse/jetty/client/util/InputStreamContentProviderTest.java

@@ -1,161 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-package org.eclipse.jetty.client.util;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.junit.jupiter.api.Test;
-
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-public class InputStreamContentProviderTest
-{
-    @Test
-    public void testHasNextFalseThenNext()
-    {
-        final AtomicBoolean closed = new AtomicBoolean();
-        InputStream stream = new InputStream()
-        {
-            @Override
-            public int read() throws IOException
-            {
-                return -1;
-            }
-
-            @Override
-            public void close() throws IOException
-            {
-                super.close();
-                closed.compareAndSet(false, true);
-            }
-        };
-
-        InputStreamContentProvider provider = new InputStreamContentProvider(stream);
-        Iterator<ByteBuffer> iterator = provider.iterator();
-
-        assertNotNull(iterator);
-        assertFalse(iterator.hasNext());
-
-        assertThrows(NoSuchElementException.class, () -> iterator.next());
-
-        assertFalse(iterator.hasNext());
-        assertTrue(closed.get());
-    }
-
-    @Test
-    public void testStreamWithContentThenNextThenNext()
-    {
-        final AtomicBoolean closed = new AtomicBoolean();
-        ByteArrayInputStream stream = new ByteArrayInputStream(new byte[]{1})
-        {
-            @Override
-            public void close() throws IOException
-            {
-                super.close();
-                closed.compareAndSet(false, true);
-            }
-        };
-
-        InputStreamContentProvider provider = new InputStreamContentProvider(stream);
-        Iterator<ByteBuffer> iterator = provider.iterator();
-
-        assertNotNull(iterator);
-
-        ByteBuffer buffer = iterator.next();
-
-        assertNotNull(buffer);
-
-        assertThrows(NoSuchElementException.class, () -> iterator.next());
-
-        assertFalse(iterator.hasNext());
-        assertTrue(closed.get());
-    }
-
-    @Test
-    public void testStreamWithExceptionThenNext()
-    {
-        final AtomicBoolean closed = new AtomicBoolean();
-        InputStream stream = new InputStream()
-        {
-            @Override
-            public int read() throws IOException
-            {
-                throw new IOException();
-            }
-
-            @Override
-            public void close() throws IOException
-            {
-                super.close();
-                closed.compareAndSet(false, true);
-            }
-        };
-
-        InputStreamContentProvider provider = new InputStreamContentProvider(stream);
-        Iterator<ByteBuffer> iterator = provider.iterator();
-
-        assertNotNull(iterator);
-
-        assertThrows(NoSuchElementException.class, () -> iterator.next());
-
-        assertFalse(iterator.hasNext());
-        assertTrue(closed.get());
-    }
-
-    @Test
-    public void testHasNextWithExceptionThenNext()
-    {
-        final AtomicBoolean closed = new AtomicBoolean();
-        InputStream stream = new InputStream()
-        {
-            @Override
-            public int read() throws IOException
-            {
-                throw new IOException();
-            }
-
-            @Override
-            public void close() throws IOException
-            {
-                super.close();
-                closed.compareAndSet(false, true);
-            }
-        };
-
-        InputStreamContentProvider provider = new InputStreamContentProvider(stream);
-        Iterator<ByteBuffer> iterator = provider.iterator();
-
-        assertNotNull(iterator);
-        assertTrue(iterator.hasNext());
-
-        assertThrows(NoSuchElementException.class, () -> iterator.next());
-
-        assertFalse(iterator.hasNext());
-        assertTrue(closed.get());
-    }
-}

jetty-client/src/test/java/org/eclipse/jetty/client/util/InputStreamContentTest.java

@@ -0,0 +1,266 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.eclipse.jetty.client.EmptyServerHandler;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.util.IO;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class InputStreamContentTest
+{
+    private Server server;
+    private ServerConnector connector;
+    private HttpClient client;
+
+    private void start(Handler handler) throws Exception
+    {
+        startServer(handler);
+        startClient();
+    }
+
+    private void startServer(Handler handler) throws Exception
+    {
+        QueuedThreadPool serverThreads = new QueuedThreadPool();
+        serverThreads.setName("server");
+        server = new Server(serverThreads);
+        connector = new ServerConnector(server, 1, 1);
+        server.addConnector(connector);
+        server.setHandler(handler);
+        server.start();
+    }
+
+    private void startClient() throws Exception
+    {
+        QueuedThreadPool clientThreads = new QueuedThreadPool();
+        clientThreads.setName("client");
+        client = new HttpClient();
+        client.setExecutor(clientThreads);
+        client.start();
+    }
+
+    @AfterEach
+    public void dispose() throws Exception
+    {
+        if (client != null)
+            client.stop();
+        if (server != null)
+            server.stop();
+    }
+
+    private static List<BiConsumer<Request, InputStream>> content()
+    {
+        return List.of(
+            (request, stream) -> request.body(new InputStreamRequestContent(stream)),
+            (request, stream) -> request.body(new InputStreamRequestContent(stream))
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("content")
+    public void testInputStreamEmpty(BiConsumer<Request, InputStream> setContent) throws Exception
+    {
+        CountDownLatch serverLatch = new CountDownLatch(1);
+        start(new EmptyServerHandler()
+        {
+            @Override
+            protected void service(String target, org.eclipse.jetty.server.Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException
+            {
+                serverLatch.countDown();
+                if (request.getInputStream().read() >= 0)
+                    throw new IOException();
+            }
+        });
+
+        CountDownLatch closeLatch = new CountDownLatch(1);
+        InputStream stream = new InputStream()
+        {
+            @Override
+            public int read()
+            {
+                return -1;
+            }
+
+            @Override
+            public void close() throws IOException
+            {
+                super.close();
+                closeLatch.countDown();
+            }
+        };
+
+        Request request = client.newRequest("localhost", connector.getLocalPort())
+            .timeout(5, TimeUnit.SECONDS);
+        setContent.accept(request, stream);
+        ContentResponse response = request.send();
+
+        assertTrue(serverLatch.await(5, TimeUnit.SECONDS));
+        assertEquals(response.getStatus(), HttpStatus.OK_200);
+        assertTrue(closeLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @MethodSource("content")
+    public void testInputStreamThrowing(BiConsumer<Request, InputStream> setContent) throws Exception
+    {
+        start(new EmptyServerHandler());
+
+        CountDownLatch closeLatch = new CountDownLatch(1);
+        InputStream stream = new InputStream()
+        {
+            @Override
+            public int read() throws IOException
+            {
+                throw new IOException();
+            }
+
+            @Override
+            public void close() throws IOException
+            {
+                super.close();
+                closeLatch.countDown();
+            }
+        };
+
+        Request request = client.newRequest("localhost", connector.getLocalPort())
+            .timeout(5, TimeUnit.SECONDS);
+        setContent.accept(request, stream);
+
+        assertThrows(ExecutionException.class, request::send);
+        assertTrue(closeLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @MethodSource("content")
+    public void testInputStreamThrowingAfterFirstRead(BiConsumer<Request, InputStream> setContent) throws Exception
+    {
+        byte singleByteContent = 0;
+        CountDownLatch serverLatch = new CountDownLatch(1);
+        start(new EmptyServerHandler()
+        {
+            @Override
+            protected void service(String target, org.eclipse.jetty.server.Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            {
+                assertEquals(singleByteContent, request.getInputStream().read());
+                serverLatch.countDown();
+            }
+        });
+
+        CountDownLatch closeLatch = new CountDownLatch(1);
+        InputStream stream = new InputStream()
+        {
+            private int reads;
+
+            @Override
+            public int read() throws IOException
+            {
+                if (++reads == 1)
+                    return singleByteContent;
+                throw new IOException();
+            }
+
+            @Override
+            public void close() throws IOException
+            {
+                super.close();
+                closeLatch.countDown();
+            }
+        };
+
+        Request request = client.newRequest("localhost", connector.getLocalPort())
+            .timeout(5, TimeUnit.SECONDS);
+        setContent.accept(request, stream);
+
+        assertThrows(ExecutionException.class, request::send);
+        assertTrue(closeLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @MethodSource("content")
+    public void testInputStreamWithSmallContent(BiConsumer<Request, InputStream> setContent) throws Exception
+    {
+        testInputStreamWithContent(setContent, new byte[1024]);
+    }
+
+    @ParameterizedTest
+    @MethodSource("content")
+    public void testInputStreamWithLargeContent(BiConsumer<Request, InputStream> setContent) throws Exception
+    {
+        testInputStreamWithContent(setContent, new byte[64 * 1024 * 1024]);
+    }
+
+    private void testInputStreamWithContent(BiConsumer<Request, InputStream> setContent, byte[] content) throws Exception
+    {
+        CountDownLatch serverLatch = new CountDownLatch(1);
+        start(new EmptyServerHandler()
+        {
+            @Override
+            protected void service(String target, org.eclipse.jetty.server.Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            {
+                serverLatch.countDown();
+                IO.copy(request.getInputStream(), IO.getNullStream());
+            }
+        });
+
+        CountDownLatch closeLatch = new CountDownLatch(1);
+        ByteArrayInputStream stream = new ByteArrayInputStream(content)
+        {
+            @Override
+            public void close() throws IOException
+            {
+                super.close();
+                closeLatch.countDown();
+            }
+        };
+
+        Request request = client.newRequest("localhost", connector.getLocalPort())
+            .timeout(5, TimeUnit.SECONDS);
+        setContent.accept(request, stream);
+        ContentResponse response = request.send();
+
+        assertTrue(serverLatch.await(5, TimeUnit.SECONDS));
+        assertEquals(response.getStatus(), HttpStatus.OK_200);
+        assertTrue(closeLatch.await(5, TimeUnit.SECONDS));
+    }
+}

jetty-client/src/test/java/org/eclipse/jetty/client/util/MultiPartContentTest.java

@@ -20,7 +20,6 @@ package org.eclipse.jetty.client.util;
 
 import java.io.BufferedWriter;
 import java.io.ByteArrayInputStream;
-import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -32,12 +31,10 @@ import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
 import javax.servlet.MultipartConfigElement;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
@@ -63,7 +60,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 // @checkstyle-disable-check : AvoidEscapedUnicodeCharactersCheck
-public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
+public class MultiPartContentTest extends AbstractHttpClientServerTest
 {
     @ParameterizedTest
     @ArgumentsSource(ScenarioProvider.class)
@@ -79,12 +76,12 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
         multiPart.close();
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send();
 
         assertEquals(200, response.getStatus());
@@ -109,13 +106,13 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
-        multiPart.addFieldPart(name, new StringContentProvider(value), null);
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
+        multiPart.addFieldPart(name, new StringRequestContent(value), null);
         multiPart.close();
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send();
 
         assertEquals(200, response.getStatus());
@@ -123,7 +120,7 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
 
     @ParameterizedTest
     @ArgumentsSource(ScenarioProvider.class)
-    public void testFieldWithOverridenContentType(Scenario scenario) throws Exception
+    public void testFieldWithOverriddenContentType(Scenario scenario) throws Exception
     {
         String name = "field";
         String value = "\u00e8";
@@ -146,16 +143,16 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
         HttpFields fields = new HttpFields();
         fields.put(HttpHeader.CONTENT_TYPE, "text/plain;charset=" + encoding.name());
-        BytesContentProvider content = new BytesContentProvider(value.getBytes(encoding));
+        BytesRequestContent content = new BytesRequestContent(value.getBytes(encoding));
         multiPart.addFieldPart(name, content, fields);
         multiPart.close();
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send();
 
         assertEquals(200, response.getStatus());
@@ -181,15 +178,15 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
-        DeferredContentProvider content = new DeferredContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
+        AsyncRequestContent content = new AsyncRequestContent("text/plain");
         multiPart.addFieldPart(name, content, null);
         multiPart.close();
         CountDownLatch responseLatch = new CountDownLatch(1);
         client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send(result ->
             {
                 assertTrue(result.isSucceeded(), supply(result.getFailure()));
@@ -233,8 +230,8 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
         });
 
         CountDownLatch closeLatch = new CountDownLatch(1);
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
-        InputStreamContentProvider content = new InputStreamContentProvider(new ByteArrayInputStream(data)
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
+        InputStreamRequestContent content = new InputStreamRequestContent(new ByteArrayInputStream(data)
         {
             @Override
             public void close() throws IOException
@@ -250,7 +247,7 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send();
 
         assertTrue(closeLatch.await(5, TimeUnit.SECONDS));
@@ -289,8 +286,8 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
-        PathContentProvider content = new PathContentProvider(contentType, tmpPath);
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
+        PathRequestContent content = new PathRequestContent(contentType, tmpPath);
         content.setByteBufferPool(client.getByteBufferPool());
         content.setUseDirectByteBuffers(client.isUseOutputDirectByteBuffers());
         multiPart.addFilePart(name, tmpPath.getFileName().toString(), content, null);
@@ -298,7 +295,7 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send();
 
         assertEquals(200, response.getStatus());
@@ -356,16 +353,16 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
         HttpFields fields = new HttpFields();
         fields.put(headerName, headerValue);
-        multiPart.addFieldPart(field, new StringContentProvider(value, encoding), fields);
-        multiPart.addFilePart(fileField, tmpPath.getFileName().toString(), new PathContentProvider(tmpPath), null);
+        multiPart.addFieldPart(field, new StringRequestContent(value, encoding), fields);
+        multiPart.addFilePart(fileField, tmpPath.getFileName().toString(), new PathRequestContent(tmpPath), null);
         multiPart.close();
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send();
 
         assertEquals(200, response.getStatus());
@@ -406,16 +403,18 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
-        DeferredContentProvider fieldContent = new DeferredContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
+        AsyncRequestContent fieldContent = new AsyncRequestContent();
         multiPart.addFieldPart("field", fieldContent, null);
-        DeferredContentProvider fileContent = new DeferredContentProvider();
+        AsyncRequestContent fileContent = new AsyncRequestContent();
         multiPart.addFilePart("file", "fileName", fileContent, null);
+        multiPart.close();
+
         CountDownLatch responseLatch = new CountDownLatch(1);
         client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send(result ->
             {
                 assertTrue(result.isSucceeded(), supply(result.getFailure()));
@@ -435,51 +434,9 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
         fieldContent.offer(encoding.encode(value));
         fieldContent.close();
 
-        multiPart.close();
-
         assertTrue(responseLatch.await(5, TimeUnit.SECONDS));
     }
 
-    @ParameterizedTest
-    @ArgumentsSource(ScenarioProvider.class)
-    public void testEachPartIsClosed(Scenario scenario) throws Exception
-    {
-        String name1 = "field1";
-        String value1 = "value1";
-        String name2 = "field2";
-        String value2 = "value2";
-        start(scenario, new AbstractMultiPartHandler()
-        {
-            @Override
-            protected void handle(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
-            {
-                Collection<Part> parts = request.getParts();
-                assertEquals(2, parts.size());
-                Iterator<Part> iterator = parts.iterator();
-                Part part1 = iterator.next();
-                assertEquals(name1, part1.getName());
-                assertEquals(value1, IO.toString(part1.getInputStream()));
-                Part part2 = iterator.next();
-                assertEquals(name2, part2.getName());
-                assertEquals(value2, IO.toString(part2.getInputStream()));
-            }
-        });
-
-        AtomicInteger closeCount = new AtomicInteger();
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
-        multiPart.addFieldPart(name1, new CloseableStringContentProvider(value1, closeCount::incrementAndGet), null);
-        multiPart.addFieldPart(name2, new CloseableStringContentProvider(value2, closeCount::incrementAndGet), null);
-        multiPart.close();
-        ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
-            .scheme(scenario.getScheme())
-            .method(HttpMethod.POST)
-            .content(multiPart)
-            .send();
-
-        assertEquals(200, response.getStatus());
-        assertEquals(2, closeCount.get());
-    }
-
     private abstract static class AbstractMultiPartHandler extends AbstractHandler
     {
         @Override
@@ -493,49 +450,4 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
 
         protected abstract void handle(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException;
     }
-
-    private static class CloseableStringContentProvider extends StringContentProvider
-    {
-        private final Runnable closeFn;
-
-        private CloseableStringContentProvider(String content, Runnable closeFn)
-        {
-            super(content);
-            this.closeFn = closeFn;
-        }
-
-        @Override
-        public Iterator<ByteBuffer> iterator()
-        {
-            return new CloseableIterator<>(super.iterator());
-        }
-
-        private class CloseableIterator<T> implements Iterator<T>, Closeable
-        {
-            private final Iterator<T> iterator;
-
-            public CloseableIterator(Iterator<T> iterator)
-            {
-                this.iterator = iterator;
-            }
-
-            @Override
-            public boolean hasNext()
-            {
-                return iterator.hasNext();
-            }
-
-            @Override
-            public T next()
-            {
-                return iterator.next();
-            }
-
-            @Override
-            public void close()
-            {
-                closeFn.run();
-            }
-        }
-    }
 }

jetty-client/src/test/java/org/eclipse/jetty/client/util/RequestContentBehaviorTest.java

@@ -0,0 +1,339 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.Fields;
+import org.eclipse.jetty.util.IO;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public class RequestContentBehaviorTest
+{
+    private static Path emptyFile;
+    private static Path smallFile;
+
+    @BeforeAll
+    public static void prepare() throws IOException
+    {
+        Path testPath = MavenTestingUtils.getTargetTestingPath();
+        Files.createDirectories(testPath);
+        emptyFile = testPath.resolve("empty.txt");
+        Files.write(emptyFile, new byte[0]);
+        smallFile = testPath.resolve("small.txt");
+        byte[] bytes = new byte[64];
+        Arrays.fill(bytes, (byte)'#');
+        Files.write(smallFile, bytes);
+    }
+
+    @AfterAll
+    public static void dispose() throws IOException
+    {
+        if (smallFile != null)
+            Files.delete(smallFile);
+        if (emptyFile != null)
+            Files.delete(emptyFile);
+    }
+
+    public static List<Request.Content> emptyContents() throws IOException
+    {
+        return List.of(
+            new AsyncRequestContent()
+            {
+                {
+                    close();
+                }
+            },
+            new ByteBufferRequestContent(),
+            new BytesRequestContent(),
+            new FormRequestContent(new Fields()),
+            new InputStreamRequestContent(IO.getClosedStream()),
+            new MultiPartRequestContent()
+            {
+                {
+                    close();
+                }
+            },
+            new PathRequestContent(emptyFile),
+            new StringRequestContent("")
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("emptyContents")
+    public void testEmptyContentEmitInitialFirstDemand(Request.Content content) throws Exception
+    {
+        CountDownLatch latch = new CountDownLatch(1);
+        Request.Content.Subscription subscription = content.subscribe((buffer, last, callback) ->
+        {
+            if (last)
+                latch.countDown();
+        }, true);
+
+        subscription.demand();
+
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @MethodSource("emptyContents")
+    public void testEmptyContentDontEmitInitialFirstDemand(Request.Content content) throws Exception
+    {
+        AtomicBoolean initial = new AtomicBoolean(true);
+        AtomicReference<CountDownLatch> latch = new AtomicReference<>(new CountDownLatch(1));
+        Request.Content.Subscription subscription = content.subscribe((buffer, last, callback) ->
+        {
+            if (initial.get())
+            {
+                if (!last)
+                    latch.get().countDown();
+            }
+            else
+            {
+                if (last)
+                    latch.get().countDown();
+            }
+        }, false);
+
+        // Initial demand should have last=false.
+        subscription.demand();
+
+        assertTrue(latch.get().await(5, TimeUnit.SECONDS));
+
+        // More demand should have last=true.
+        initial.set(false);
+        latch.set(new CountDownLatch(1));
+        subscription.demand();
+
+        assertTrue(latch.get().await(5, TimeUnit.SECONDS));
+    }
+
+    public static List<Request.Content> smallContents() throws IOException
+    {
+        return List.of(
+            new AsyncRequestContent(ByteBuffer.allocate(64))
+            {
+                {
+                    close();
+                }
+            },
+            new ByteBufferRequestContent(ByteBuffer.allocate(64)),
+            new BytesRequestContent(new byte[64]),
+            new FormRequestContent(new Fields()
+            {
+                {
+                    add("foo", "bar");
+                }
+            }),
+            new InputStreamRequestContent(new ByteArrayInputStream(new byte[64])),
+            new MultiPartRequestContent()
+            {
+                {
+                    addFieldPart("field", new StringRequestContent("*".repeat(64)), null);
+                    close();
+                }
+            },
+            new PathRequestContent(smallFile),
+            new StringRequestContent("x".repeat(64))
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("smallContents")
+    public void testSmallContentEmitInitialFirstDemand(Request.Content content) throws Exception
+    {
+        AtomicBoolean initial = new AtomicBoolean(true);
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Request.Content.Subscription> subscriptionRef = new AtomicReference<>();
+        Request.Content.Subscription subscription = content.subscribe((buffer, last, callback) ->
+        {
+            if (initial.getAndSet(false))
+                assertTrue(buffer.hasRemaining());
+            if (last)
+                latch.countDown();
+            else
+                subscriptionRef.get().demand();
+        }, true);
+        subscriptionRef.set(subscription);
+
+        // Initial demand.
+        subscription.demand();
+
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @MethodSource("smallContents")
+    public void testSmallContentDontEmitInitialFirstDemand(Request.Content content) throws Exception
+    {
+        AtomicBoolean initial = new AtomicBoolean(true);
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Request.Content.Subscription> subscriptionRef = new AtomicReference<>();
+        Request.Content.Subscription subscription = content.subscribe((buffer, last, callback) ->
+        {
+            if (initial.getAndSet(false))
+            {
+                assertFalse(buffer.hasRemaining());
+                assertFalse(last);
+            }
+            if (last)
+                latch.countDown();
+            else
+                subscriptionRef.get().demand();
+        }, false);
+        subscriptionRef.set(subscription);
+
+        // Initial demand.
+        subscription.demand();
+
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @MethodSource("smallContents")
+    public void testSmallContentFailedAfterFirstDemand(Request.Content content)
+    {
+        Throwable testFailure = new Throwable("test_failure");
+
+        AtomicInteger notified = new AtomicInteger();
+        AtomicReference<Throwable> failureRef = new AtomicReference<>();
+        Request.Content.Subscription subscription = content.subscribe(new Request.Content.Consumer()
+        {
+            @Override
+            public void onContent(ByteBuffer buffer, boolean last, Callback callback)
+            {
+                notified.getAndIncrement();
+            }
+
+            @Override
+            public void onFailure(Throwable error)
+            {
+                testFailure.addSuppressed(new Throwable("suppressed"));
+                failureRef.compareAndSet(null, error);
+            }
+        }, false);
+
+        // Initial demand.
+        subscription.demand();
+
+        assertEquals(1, notified.get());
+
+        subscription.fail(testFailure);
+        subscription.demand();
+
+        assertEquals(1, notified.get());
+        Throwable failure = failureRef.get();
+        assertNotNull(failure);
+        assertSame(testFailure, failure);
+        assertEquals(1, failure.getSuppressed().length);
+    }
+
+    @ParameterizedTest
+    @MethodSource("smallContents")
+    public void testDemandAfterLastContentFails(Request.Content content) throws Exception
+    {
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Request.Content.Subscription> subscriptionRef = new AtomicReference<>();
+        AtomicReference<Throwable> failureRef = new AtomicReference<>();
+        Request.Content.Subscription subscription = content.subscribe(new Request.Content.Consumer()
+        {
+            @Override
+            public void onContent(ByteBuffer buffer, boolean last, Callback callback)
+            {
+                if (last)
+                    latch.countDown();
+                else
+                    subscriptionRef.get().demand();
+            }
+
+            @Override
+            public void onFailure(Throwable error)
+            {
+                error.addSuppressed(new Throwable("suppressed"));
+                failureRef.compareAndSet(null, error);
+            }
+        }, false);
+        subscriptionRef.set(subscription);
+
+        // Initial demand.
+        subscription.demand();
+
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+
+        // Demand more, should fail.
+        subscription.demand();
+
+        Throwable failure = failureRef.get();
+        assertNotNull(failure);
+        assertEquals(1, failure.getSuppressed().length);
+    }
+
+    @ParameterizedTest
+    @MethodSource("smallContents")
+    public void testReproducibleContentCanHaveMultipleSubscriptions(Request.Content content) throws Exception
+    {
+        assumeTrue(content.isReproducible());
+
+        CountDownLatch latch1 = new CountDownLatch(1);
+        Request.Content.Subscription subscription1 = content.subscribe((buffer, last, callback) ->
+        {
+            if (last)
+                latch1.countDown();
+        }, true);
+
+        CountDownLatch latch2 = new CountDownLatch(1);
+        Request.Content.Subscription subscription2 = content.subscribe((buffer, last, callback) ->
+        {
+            if (last)
+                latch2.countDown();
+        }, true);
+
+        // Initial demand.
+        subscription1.demand();
+        assertTrue(latch1.await(5, TimeUnit.SECONDS));
+
+        // Initial demand.
+        subscription2.demand();
+        assertTrue(latch2.await(5, TimeUnit.SECONDS));
+    }
+}

jetty-client/src/test/java/org/eclipse/jetty/client/util/SPNEGOAuthenticationTest.java

@@ -291,7 +291,7 @@ public class SPNEGOAuthenticationTest extends AbstractHttpClientServerTest
 
         requests.set(0);
         ByteArrayInputStream input = new ByteArrayInputStream("hello_world".getBytes(StandardCharsets.UTF_8));
-        request = client.newRequest(uri).method("POST").path("/secure").content(new InputStreamContentProvider(input));
+        request = client.newRequest(uri).method("POST").path("/secure").body(new InputStreamRequestContent(input));
         response = request.timeout(15, TimeUnit.SECONDS).send();
         assertEquals(200, response.getStatus());
         // Authentication expired, but POSTs are allowed.

jetty-client/src/test/java/org/eclipse/jetty/client/util/TypedContentProviderTest.java

@@ -108,7 +108,7 @@ public class TypedContentProviderTest extends AbstractHttpClientServerTest
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(new FormContentProvider(fields))
+            .body(new FormRequestContent(fields))
             .header(HttpHeader.CONTENT_TYPE, contentType)
             .send();
 
@@ -135,7 +135,7 @@ public class TypedContentProviderTest extends AbstractHttpClientServerTest
 
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
-            .content(new StringContentProvider(null, content, StandardCharsets.UTF_8))
+            .body(new StringRequestContent(null, content, StandardCharsets.UTF_8))
             .send();
 
         assertEquals(200, response.getStatus());

jetty-deploy/src/test/java/org/eclipse/jetty/deploy/BadAppDeployTest.java

@@ -29,7 +29,7 @@ import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.handler.ContextHandlerCollection;
 import org.eclipse.jetty.server.handler.DefaultHandler;
-import org.eclipse.jetty.server.handler.HandlerCollection;
+import org.eclipse.jetty.server.handler.HandlerList;
 import org.eclipse.jetty.toolchain.test.FS;
 import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
 import org.eclipse.jetty.toolchain.test.jupiter.WorkDir;
@@ -69,7 +69,7 @@ public class BadAppDeployTest
            It is important that this Order be maintained for an accurate test case.
            ### BEAN: QueuedThreadPool[qtp1327763628]@4f2410ac{STOPPED,8<=0<=200,i=0,r=-1,q=0}[NO_TRY]
            ### BEAN: ServerConnector@16f65612{HTTP/1.1,[http/1.1]}{0.0.0.0:8080}
-           ### BEAN: HandlerCollection@5f150435{STOPPED}
+           ### BEAN: HandlerList@5f150435{STOPPED}
            ### BEAN: DeploymentManager@1c53fd30{STOPPED}
          */
 
@@ -79,10 +79,8 @@ public class BadAppDeployTest
         server.addConnector(connector);
 
         ContextHandlerCollection contexts = new ContextHandlerCollection();
-        HandlerCollection handlers = new HandlerCollection();
-        handlers.addHandler(contexts);
-        handlers.addHandler(new DefaultHandler());
-        server.setHandler(handlers); // this should be done before addBean(deploymentManager)
+        // this should be done before addBean(deploymentManager)
+        server.setHandler(new HandlerList(contexts, new DefaultHandler()));
 
         DeploymentManager deploymentManager = new DeploymentManager();
         deploymentManager.setContexts(contexts);
@@ -121,7 +119,7 @@ public class BadAppDeployTest
            ### BEAN: QueuedThreadPool[qtp1530388690]@5b37e0d2{STOPPED,8<=0<=200,i=0,r=-1,q=0}[NO_TRY]
            ### BEAN: ServerConnector@5e265ba4{HTTP/1.1,[http/1.1]}{0.0.0.0:8080}
            ### BEAN: DeploymentManager@3419866c{STOPPED}
-           ### BEAN: HandlerCollection@63e31ee{STOPPED}
+           ### BEAN: HandlerList@63e31ee{STOPPED}
          */
 
         server = new Server();
@@ -146,12 +144,11 @@ public class BadAppDeployTest
         webAppProvider.setMonitoredDirName(webappsDir.toString());
         webAppProvider.setScanInterval(1);
 
-        server.addBean(deploymentManager); // this should be done before setHandler(handlers)
+        // this must be done before setHandler(handlers)
+        server.addBean(deploymentManager);
 
-        HandlerCollection handlers = new HandlerCollection();
-        handlers.addHandler(contexts);
-        handlers.addHandler(new DefaultHandler());
-        server.setHandler(handlers); // this should be done after addBean(deploymentManager)
+        // this must be done after addBean(deploymentManager)
+        server.setHandler(new HandlerList(contexts, new DefaultHandler()));
 
         assertTimeoutPreemptively(ofSeconds(10), () ->
         {

jetty-deploy/src/test/resources/jetty.xml

@@ -106,7 +106,7 @@
     <!-- RequestLogHandler after the default handler                 -->
     <!-- =========================================================== -->
     <Set name="handler">
-      <New id="Handlers" class="org.eclipse.jetty.server.handler.HandlerCollection">
+      <New id="Handlers" class="org.eclipse.jetty.server.handler.HandlerList">
         <Set name="handlers">
          <Array type="org.eclipse.jetty.server.Handler">
            <Item>

jetty-documentation/pom.xml

@@ -1,24 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~
-  ~ ========================================================================
-  ~ Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-  ~
-  ~ This program and the accompanying materials are made available under
-  ~ the terms of the Eclipse Public License 2.0 which is available at
-  ~ https://www.eclipse.org/legal/epl-2.0
-  ~
-  ~ This Source Code may also be made available under the following
-  ~ Secondary Licenses when the conditions for such availability set
-  ~ forth in the Eclipse Public License, v. 2.0 are satisfied:
-  ~ the Apache License v2.0 which is available at
-  ~ https://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-  ~ ========================================================================
-  ~
-  -->
-
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <parent>
     <groupId>org.eclipse.jetty</groupId>
@@ -41,7 +21,7 @@
     <asciidoctorj.epub.version>1.5.0-alpha.8.1</asciidoctorj.epub.version>
     <asciidoctorj.version>1.5.8.1</asciidoctorj.version>
     <jruby.version>1.7.27</jruby.version>
-    <web.resources.version>1.1</web.resources.version>
+    <web.resources.version>1.2</web.resources.version>
     <web.resources.directory>${project.build.directory}/web-resources</web.resources.directory>
     <!-- old properties -->
     <html.directory>${project.build.directory}/current</html.directory>
@@ -49,11 +29,66 @@
   </properties>
 
   <dependencies>
+    <dependency>
+      <groupId>org.eclipse.jetty.toolchain</groupId>
+      <artifactId>jetty-servlet-api</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-io</artifactId>
+      <artifactId>jetty-server</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlet</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlets</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-rewrite</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-webapp</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-alpn-server</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty.http2</groupId>
+      <artifactId>http2-server</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-client</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty.fcgi</groupId>
+      <artifactId>fcgi-client</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty.http2</groupId>
+      <artifactId>http2-http-client-transport</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-slf4j-impl</artifactId>
+      <version>${project.version}</version>
+      <scope>compile</scope>
+    </dependency>
   </dependencies>
 
   <build>
@@ -111,6 +146,13 @@
         <groupId>org.asciidoctor</groupId>
         <artifactId>asciidoctor-maven-plugin</artifactId>
         <version>${asciidoctor.maven.plugin.version}</version>
+        <dependencies>
+          <dependency>
+            <groupId>org.asciidoctor</groupId>
+            <artifactId>asciidoctorj-diagram</artifactId>
+            <version>2.0.1</version>
+          </dependency>
+        </dependencies>
         <configuration>
           <attributes>
             <JDURL>http://www.eclipse.org/jetty/javadoc/${javadoc.version}</JDURL>
@@ -125,7 +167,6 @@
             <MVNCENTRAL>http://central.maven.org/maven2</MVNCENTRAL>
             <VERSION>${project.version}</VERSION>
             <TIMESTAMP>${maven.build.timestamp}</TIMESTAMP>
-            <docbits>${basedir}/src/main/java</docbits>
           </attributes>
         </configuration>
         <executions>
@@ -169,6 +210,7 @@
               <sourceDirectory>${basedir}/src/main/asciidoc/contribution-guide</sourceDirectory>
               <sourceDocumentName>index.adoc</sourceDocumentName>
               <outputDirectory>${project.build.directory}/html/contribution-guide</outputDirectory>
+              <sourceHighlighter>coderay</sourceHighlighter>
             </configuration>
           </execution>
           <execution>
@@ -183,6 +225,10 @@
               <sourceDirectory>${basedir}/src/main/asciidoc/embedded-guide</sourceDirectory>
               <sourceDocumentName>index.adoc</sourceDocumentName>
               <outputDirectory>${project.build.directory}/html/embedded-guide</outputDirectory>
+              <sourceHighlighter>coderay</sourceHighlighter>
+              <requires>
+                <require>asciidoctor-diagram</require>
+              </requires>
             </configuration>
           </execution>
         </executions>

jetty-documentation/src/main/asciidoc/contribution-guide/index.adoc

@@ -23,8 +23,11 @@
 :revdate: {TIMESTAMP}
 :toc: left
 :toc-title: Contribution Guide
-:toc-image: ../../common/images/jetty-logo.svg
-:toc-image-url: /jetty/index.html
+:toc-style: 
+
+:header-style: eclipse-thin
+:breadcrumb-style: eclipse-thin
+:footer-style: default
 :breadcrumb: Home:../index.html | Contribution Guide:./index.html
 
 // docinfo lets you pull in shared content and/or influence via render type
@@ -43,12 +46,6 @@ endif::[]
 // options for special blocks, code snippets, screen, etc
 :sub-order: attributes+
 
-// suppress document footer generation
-//:nofooter:
-
-// suppress Eclipse footer
-:no-eclipse-footer:
-
 // uncomment to allow include::https:// style content inclusion
 //:allow-uri-read: true
 

jetty-documentation/src/main/asciidoc/distribution-guide/annotations/using-annotations.adoc

@@ -121,9 +121,8 @@ As is the case with annotation scanning, the link:#using-extra-classpath-method[
 
 ____
 [NOTE]
-As of Jetty-9.4.4, unless the `web.xml` is version 3.0 or greater, only `ServletContainerInitializers` that are on the container classpath will be discovered.
-Users wishing to use `ServletContainerInitializers` from within the webapp with older versions of `web.xml` must either upgrade their `web.xml` version, or call `WebAppContext.setConfigurationDiscovered(true)` either programmatically or in xml.
-Upgrading the `web.xml` version is preferable.
+As of Jetty 10, Annotations will be discovered even for old versions of `web.xml` (2.5).
+Users wishing not to use annotations from the webapp classpath must call `WebAppContext.setConfigurationDiscovered(false)` either programmatically or in xml.
 ____
 
 ===== Controlling the order of ServletContainerInitializer invocation

jetty-documentation/src/main/asciidoc/distribution-guide/contexts/setting-form-size.adoc

@@ -19,57 +19,51 @@
 [[setting-form-size]]
 === Setting Max Form Size
 
-Jetty limits the amount of data that can post back from a browser or other client to the server.
-This helps protect the server against denial of service attacks by malicious clients sending huge amounts of data.
-The default maximum size Jetty permits is 200000 bytes.
-You can change this default for a particular webapp, for all webapps on a particular Server instance, or all webapps within the same JVM.
+When a client issues a POST request with `Content-Type: application/x-www-form-urlencoded` there are 2 configurable limits imposed to help protect the server against denial of service attacks by malicious clients sending huge amounts of data.
+
+There is a maximum form size limit, and a maximum form keys limit.
+
+The default maximum form size Jetty permits is 200000 bytes.
+The default maximum form keys Jetty permits is 1000 keys.
+
+You can change these defaults for a particular webapp, or all webapps within the same JVM.
 
 ==== For a Single Webapp
 
-The method to invoke is: `ContextHandler.setMaxFormContentSize(int maxSize);`
+The methods to invoke are:
+
+* `ContextHandler.setMaxFormContentSize(int maxSize);`
+* `ContextHandler.setMaxFormKeys(int max);`
 
 This can be done either in a context XML deployment descriptor external to the webapp, or in a `jetty-web.xml` file in the webapp's `WEB-INF` directory.
 
 In either case the syntax of the XML file is the same:
 
-[source, xml, subs="{sub-order}"]
+[source,xml,subs="{sub-order}"]
 ----
 <Configure class="org.eclipse.jetty.webapp.WebAppContext">
   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -->
   <!-- Max Form Size                                                   -->
   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -->
-  <Set name="maxFormContentSize">200000</Set>
+  <Set name="maxFormContentSize">400000</Set>
+  <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -->
+  <!-- Max Form Keys                                                   -->
+  <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -->
+  <Set name="maxFormKeys">400</Set>
 </Configure>
 ----
 
-==== For All Apps on a Server
-
-Set an attribute in `jetty.xml` on the Server instance for which you want to modify the maximum form content size:
-
-[source, xml, subs="{sub-order}"]
-----
-<Configure class="org.eclipse.jetty.server.Server">
-  <Call name="setAttribute">
-    <Arg>org.eclipse.jetty.server.Request.maxFormContentSize</Arg>
-    <Arg>200000</Arg>
-  </Call>
-</Configure>
-----
-
-____
-[IMPORTANT]
-It is important to remember that you should *not* modify the XML files in your `$JETTY_HOME`.
-If you do for some reason feel you want to change the way an XML file operates, it is best to make a copy of it in your `$JETTY_BASE` in an `/etc` directory.
-Jetty will always look first to the `$JETTY_BASE` for configuration.
-____
-
 ==== For All Apps in the JVM
 
-Use the system property `org.eclipse.jetty.server.Request.maxFormContentSize`.
+Use the system properties:
+
+* `org.eclipse.jetty.server.Request.maxFormContentSize` - the maximum size of Form Content allowed
+* `org.eclipse.jetty.server.Request.maxFormKeys` - the maximum number of Form Keys allowed
+
 This can be set on the command line or in the `$JETTY_BASE\start.ini` or any `$JETTY_BASE\start.d\*.ini` link:#startup-modules[module ini file.]
 Using `$JETTY_BASE\start.d\server.ini` as an example:
 
-[source, console, subs="{sub-order}"]
+[source,console,subs="{sub-order}"]
 ----
 # ---------------------------------------
 # Module: server
@@ -83,5 +77,6 @@ Using `$JETTY_BASE\start.d\server.ini` as an example:
 
 ...
 
--Dorg.eclipse.jetty.server.Request.maxFormContentSize=200000
+-Dorg.eclipse.jetty.server.Request.maxFormContentSize=400000
+-Dorg.eclipse.jetty.server.Request.maxFormKeys=400
 ----

jetty-documentation/src/main/asciidoc/distribution-guide/extras/default-servlet.adoc

@@ -49,7 +49,7 @@ If `true`, welcome files are redirected rather that forwarded.
 welcomeServlets::
 If `true`, attempt to dispatch to welcome files that are servlets, but only after no matching static
 resources could be found. If `false`, then a welcome file must exist on disk. If `exact`, then exact
-servlet matches are supported without an existing file. Default is `true`. This must be `false` if you want directory listings,
+servlet matches are supported without an existing file. Default is `false`. This must be `false` if you want directory listings,
 but have index.jsp in your welcome file list.
 precompressed::
 If set to a comma separated list of encoding types (that may be listed in a requests Accept-Encoding header) to file extension mappings to look for and serve. 

jetty-documentation/src/main/asciidoc/distribution-guide/http2/introduction.adoc

@@ -50,5 +50,5 @@ The Jetty HTTP/2 implementation consists of the following sub-projects (each pro
 2.  `http2-hpack`: Contains the HTTP/2 HPACK implementation for HTTP header compression.
 3.  `http2-server`: Provides the server-side implementation of HTTP/2.
 4.  `http2-client`: Provides the implementation of HTTP/2 client with a low level HTTP/2 API, dealing with HTTP/2 streams, frames, etc.
-5.  `http2-http-client-transport`: Provides the implementation of the HTTP/2 transport for `HttpClient` (see xref:http-client[]).
+5.  `http2-http-client-transport`: Provides the implementation of the HTTP/2 transport for `HttpClient` (see xref:client-http[this section]).
 Applications can use the higher level API provided by `HttpClient` to send HTTP requests and receive HTTP responses, and the HTTP/2 transport will take care of converting them in HTTP/2 format (see also https://webtide.com/http2-support-for-httpclient/[this blog entry]).

jetty-documentation/src/main/asciidoc/distribution-guide/index.adoc

@@ -23,8 +23,11 @@
 :revdate: {TIMESTAMP}
 :toc: left
 :toc-title: Distribution Guide
-:toc-image: ../../common/images/jetty-logo.svg
-:toc-image-url: /jetty/index.html
+:toc-style:
+
+:header-style: eclipse-thin
+:breadcrumb-style: eclipse-thin
+:footer-style: default
 :breadcrumb: Home:../index.html | Distribution Guide:./index.html
 
 // docinfo lets you pull in shared content and/or influence via render type
@@ -43,12 +46,6 @@ endif::[]
 // options for special blocks, code snippets, screen, etc
 :sub-order: attributes+
 
-// suppress document footer generation
-//:nofooter:
-
-// suppress Eclipse footer
-:no-eclipse-footer:
-
 // uncomment to allow include::https:// style content inclusion
 //:allow-uri-read: true
 

jetty-documentation/src/main/asciidoc/distribution-guide/logging/example-logback-centralized-logging.adoc

@@ -34,74 +34,105 @@ This configuration is essentially the multiple logger configuration with added c
 
 The technique used by this configuration is to provide an link:{JDURL}org/eclipse/jetty/deploy/AppLifeCycle.Binding.html[AppLifeCycle.Binding] against the link:{JDURL}/org/eclipse/jetty/deploy/AppLifeCycle.html[`"deploying"`node] that modifies the
 link:{JDURL}/org/eclipse/jetty/webapp/WebAppContext.html#getSystemClasspathPattern()[WebAppContext.getSystemClasspathPattern().add(String)] for the common logging classes.
-See https://github.com/jetty-project/jetty-webapp-logging/blob/master/src/main/java/org/eclipse/jetty/webapp/logging/CentralizedWebAppLoggingBinding.java[org.eclipse.jetty.logging.CentralizedWebAppLoggingBinding] for actual implementation.
+See https://github.com/jetty-project/jetty-webapp-logging/blob/master/jetty-webapp-logging/src/main/java/org/eclipse/jetty/webapp/logging/CentralizedWebAppLoggingBinding.java[org.eclipse.jetty.logging.CentralizedWebAppLoggingBinding] for actual implementation.
 
 A convenient replacement `logging` module has been created to bootstrap your `${jetty.base}` directory for capturing all Jetty server logging from multiple logging frameworks into a single logging output file managed by Logback.
 
-[source, screen, subs="{sub-order}"]
+[source,screen,subs="{sub-order}"]
 ....
-[mybase]$ mkdir modules
-[mybase]$ cd modules
-
-[modules]$ curl -O https://raw.githubusercontent.com/jetty-project/logging-modules/master/capture-all/logging.mod
+[mybase]$ curl -O https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-webapp-logging/9.4.27/jetty-webapp-logging-9.4.27-config.jar
   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                  Dload  Upload   Total   Spent    Left  Speed
-100  1416  100  1416    0     0   4241      0 --:--:-- --:--:-- --:--:--  4252
+100  3402  100  3402    0     0  15823      0 --:--:-- --:--:-- --:--:-- 15750
 
-[master]$ curl -O https://raw.githubusercontent.com/jetty-project/logging-modules/master/centralized/webapp-logging.mod
-  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
-                                 Dload  Upload   Total   Spent    Left  Speed
-100   660  100   660    0     0   2032      0 --:--:-- --:--:-- --:--:--  2037
-[modules]$ cd ..
+[mybase]$ jar -xf jetty-webapp-logging-9.4.27-config.jar
 
-[mybase]$ java -jar /opt/jetty-dist/start.jar --add-to-start=logging,webapp-logging
-INFO: logging         initialised in ${jetty.base}/start.ini (appended)
-MKDIR: ${jetty.base}/logs
-DOWNLOAD: https://repo1.maven.org/maven2/org/slf4j/slf4j-api/1.6.6/slf4j-api-1.6.6.jar to lib/logging/slf4j-api-1.6.6.jar
-DOWNLOAD: https://repo1.maven.org/maven2/org/slf4j/log4j-over-slf4j/1.6.6/log4j-over-slf4j-1.6.6.jar to lib/logging/log4j-over-slf4j-1.6.6.jar
-DOWNLOAD: https://repo1.maven.org/maven2/org/slf4j/jul-to-slf4j/1.6.6/jul-to-slf4j-1.6.6.jar to lib/logging/jul-to-slf4j-1.6.6.jar
-DOWNLOAD: https://repo1.maven.org/maven2/org/slf4j/jcl-over-slf4j/1.6.6/jcl-over-slf4j-1.6.6.jar to lib/logging/jcl-over-slf4j-1.6.6.jar
-DOWNLOAD: https://repo1.maven.org/maven2/ch/qos/logback/logback-core/1.0.7/logback-core-1.0.7.jar to lib/logging/logback-core-1.0.7.jar
-DOWNLOAD: https://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.0.7/logback-classic-1.0.7.jar to lib/logging/logback-classic-1.0.7.jar
-DOWNLOAD: https://raw.githubusercontent.com/jetty-project/logging-modules/master/capture-all/logback.xml to resources/logback.xml
-DOWNLOAD: https://raw.githubusercontent.com/jetty-project/logging-modules/master/capture-all/jetty-logging.properties to resources/jetty-logging.properties
-DOWNLOAD: https://raw.githubusercontent.com/jetty-project/logging-modules/master/capture-all/jetty-logging.xml to etc/jetty-logging.xml
-INFO: resources       initialised transitively
-INFO: resources       enabled in     ${jetty.base}/start.ini
-INFO: webapp-logging  initialised in ${jetty.base}/start.ini (appended)
-DOWNLOAD: https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-webapp-logging/9.0.0/jetty-webapp-logging-9.0.0.jar to lib/webapp-logging/jetty-webapp-logging-9.0.0.jar
-DOWNLOAD: https://raw.githubusercontent.com/jetty-project/jetty-webapp-logging/master/src/main/config/etc/jetty-webapp-logging.xml to etc/jetty-webapp-logging.xml
-DOWNLOAD: https://raw.githubusercontent.com/jetty-project/jetty-webapp-logging/master/src/main/config/etc/jetty-mdc-handler.xml to etc/jetty-mdc-handler.xml
-INFO: deploy          initialised transitively
-INFO: deploy          enabled in     ${jetty.base}/start.ini
-INFO: logging         initialised transitively
-INFO: resources       initialised transitively
-INFO: resources       enabled in     ${jetty.base}/start.ini
+[mybase]$ java -jar /opt/jetty-hom/start.jar --create-startd --add-to-start=centralized-webapp-logging
 
-[mybase]$ java -jar /opt/jetty-dist/start.jar
+ALERT: There are enabled module(s) with licenses.
+The following 2 module(s):
+ + contains software not provided by the Eclipse Foundation!
+ + contains software not covered by the Eclipse Public License!
+ + has not been audited for compliance with its license
+
+ Module: logback-impl
+  + Logback: the reliable, generic, fast and flexible logging framework.
+  + Copyright (C) 1999-2012, QOS.ch. All rights reserved.
+  + This program and the accompanying materials are dual-licensed under
+  + either:
+  + the terms of the Eclipse Public License v1.0
+  + as published by the Eclipse Foundation:
+  + http://www.eclipse.org/legal/epl-v10.html
+  + or (per the licensee's choosing) under
+  + the terms of the GNU Lesser General Public License version 2.1
+  + as published by the Free Software Foundation:
+  + http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html
+
+ Module: slf4j-api
+  + SLF4J is distributed under the MIT License.
+  + Copyright (c) 2004-2013 QOS.ch
+  + All rights reserved.
+  + Permission is hereby granted, free  of charge, to any person obtaining
+  + a  copy  of this  software  and  associated  documentation files  (the
+  + "Software"), to  deal in  the Software without  restriction, including
+  + without limitation  the rights to  use, copy, modify,  merge, publish,
+  + distribute,  sublicense, and/or sell  copies of  the Software,  and to
+  + permit persons to whom the Software  is furnished to do so, subject to
+  + the following conditions:
+  + The  above  copyright  notice  and  this permission  notice  shall  be
+  + included in all copies or substantial portions of the Software.
+  + THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
+  + EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
+  + MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
+  + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+  + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+  + OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
+  + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Proceed (y/N)? y
+INFO  : slf4j-api       transitively enabled
+INFO  : log4j-over-slf4j transitively enabled
+INFO  : jcl-slf4j       transitively enabled
+INFO  : logback-impl    transitively enabled
+INFO  : jul-slf4j       transitively enabled
+INFO  : slf4j-logback   transitively enabled
+INFO  : centralized-webapp-logging initialized in ${jetty.base}/start.d/centralized-webapp-logging.ini
+INFO  : logging-logback transitively enabled
+INFO  : resources       transitively enabled
+MKDIR : ${jetty.base}/lib/slf4j
+DOWNLD: https://repo1.maven.org/maven2/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar to ${jetty.base}/lib/slf4j/slf4j-api-1.7.25.jar
+MKDIR : ${jetty.base}/lib/logging
+DOWNLD: https://repo1.maven.org/maven2/org/slf4j/log4j-over-slf4j/1.7.25/log4j-over-slf4j-1.7.25.jar to ${jetty.base}/lib/logging/log4j-over-slf4j-1.7.25.jar
+DOWNLD: https://repo1.maven.org/maven2/org/slf4j/jcl-over-slf4j/1.7.25/jcl-over-slf4j-1.7.25.jar to ${jetty.base}/lib/slf4j/jcl-over-slf4j-1.7.25.jar
+MKDIR : ${jetty.base}/lib/logback
+DOWNLD: https://repo1.maven.org/maven2/ch/qos/logback/logback-core/1.2.3/logback-core-1.2.3.jar to ${jetty.base}/lib/logback/logback-core-1.2.3.jar
+DOWNLD: https://repo1.maven.org/maven2/org/slf4j/jul-to-slf4j/1.7.25/jul-to-slf4j-1.7.25.jar to ${jetty.base}/lib/slf4j/jul-to-slf4j-1.7.25.jar
+COPY  : ${jetty.home}/modules/jul-slf4j/etc/java-util-logging.properties to ${jetty.base}/etc/java-util-logging.properties
+DOWNLD: https://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.2.3/logback-classic-1.2.3.jar to ${jetty.base}/lib/logback/logback-classic-1.2.3.jar
+MKDIR : ${jetty.base}/logs
+DOWNLD: https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-webapp-logging/9.4.27/jetty-webapp-logging-9.4.27.jar to ${jetty.base}/lib/logging/jetty-webapp-logging-9.4.27.jar
+INFO  : Base directory was modified
+
+$
 ....
 
-The replacement `logging.mod` performs a number of tasks.
+This replacement `centralized-webapp-logging.mod` performs a number of tasks.
 
-.  `mybase` is a `${jetty.base}` directory.
-.  The jetty-distribution is unpacked (and untouched) into `/opt/jetty-dist/` and becomes the `${jetty.home}` directory for this demonstration.
-.  The `curl` command downloads the replacement `logging.mod` and puts it into the `${jetty.base}/modules/` directory for use by mybase only.
-.  The `start.jar --add-to-start=logging,webapp-logging` command performs a number of steps to make the logging module available to the `${jetty.base}` configuration.
-..  Several entries are added to the `${jetty.base}/start.ini` configuration.
-*  `--module=logging` is added to enable the logging module.
-*  `--module=webapp-logging` is added to enable the webapp-logging module.
-..  Required `${jetty.base}` directories are created: `${jetty.base}/logs` and `${jetty.base}/resources`.
-..  Required logging libraries are downloaded (if not present already) to the `${jetty.base}/lib/logging/` directory:
+. `mybase` is a `${jetty.base}` directory.
+. The jetty-distribution is unpacked (and untouched) into `/opt/jetty-dist/` and becomes the `${jetty.home}` directory for this demonstration.
+. The `curl` command downloads the replacement config overlay for the `${jetty.base}/modules/` directory to use.
+. The `start.jar --add-to-start=centralized-webapp-logging` command performs a number of steps to make the centralized-webapp-logging module available to the `${jetty.base}` configuration.
+.. A new `${jetty.base}/start.d/centralized-webapp-logging.ini` configuration was created.
+.. Required `${jetty.base}` directories are created: `${jetty.base}/logs` and `${jetty.base}/resources`.
+.. Required logging libraries are downloaded (if not present already) to the `${jetty.base}/lib/logging/` directory:
 * `slf4j-api.jar` - API jar for Slf4j (used by most of the rest of the jars)
 * `log4j-over-slf4j.jar` - Slf4j jar that captures all log4j emitted logging events
 * `jul-to-slf4j.jar` - Slf4j jar that captures all java.util.logging events
 * `jcl-over-slf4j.jar` - Slf4j jar that captures all commons-logging events
 * `logback-classic.jar` - the Slf4j adapter jar that routes all of the captured logging events to logback itself.
 * `logback-core.jar` - the logback implementation jar, that handles all of the filtering and output of the logging events.
-..  Required webapp-logging library is downloaded (if not present already) to the `${jetty.base}/lib/webapp-logging/` directory:
+.. Required webapp-logging library is downloaded (if not present already) to the `${jetty.base}/lib/webapp-logging/` directory:
 * `jetty-webapp-logging.jar` - the Jetty side deployment manger app-lifecycle bindings for modifying the `WebAppClassloaders` of deployed webapps.
-..  Required configuration files are downloaded (if not present already) to the `${jetty.base}/resources/` directory: `jetty-logging.properties`, and `logback.xml`.
-..  Required initialization commands are downloaded (if not present already) to the `${jetty.base}/etc/` directory: `jetty-logging.xml`, `jetty-webapp-logging.xml`, and `jetty-mdc-handler.xml`.
 
 At this point the Jetty `mybase` is configured so that the jetty server itself will log using slf4j, and all other logging events from other Jetty Server components (such as database drivers, security layers, jsp, mail, and other 3rd party server components) are routed to logback for filtering and output.
 
@@ -109,4 +140,5 @@ All webapps deployed via the `DeploymentManager` have their `WebAppClassLoader`
 
 The server classpath can be verified by using the `start.jar --list-config` command.
 
-In essence, Jetty is now configured to emit its own logging events to slf4j, and various slf4j bridge jars are acting on behalf of log4j, `java.util.logging`, and `commons-logging`, routing all of the logging events to logback (a slf4j adapter) for routing (to console, file, etc...).
+In essence, Jetty is now configured to emit its own logging events to slf4j, and various slf4j bridge jars are acting on behalf of `log4j`, `java.util.logging`, and `commons-logging`, routing all of the logging events to `logback`
+(a slf4j implementation) for routing (to console, file, etc...).

jetty-documentation/src/main/asciidoc/embedded-guide/.asciidoctorconfig

@@ -0,0 +1,3 @@
+// Asciidoctor IDE configuration file.
+// See https://github.com/asciidoctor/asciidoctor-intellij-plugin/wiki/Support-project-specific-configurations
+:doc_code: ../../java

jetty-documentation/src/main/asciidoc/embedded-guide/arch-bean.adoc

@@ -0,0 +1,101 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-arch-bean]]
+=== Jetty Component Architecture
+
+Applications that use the Jetty libraries (both client and server) create objects from Jetty classes and compose them together to obtain the desired functionalities.
+
+A client application creates a `ClientConnector` instance, a `HttpClientTransport` instance and an `HttpClient` instance and compose them to have a working HTTP client that uses to call third party services.
+
+A server application creates a `ThreadPool` instance, a `Server` instance, a `ServerConnector` instance, a `Handler` instance and compose them together to expose an HTTP service.
+
+Internally, the Jetty libraries create even more instances of other components that also are composed together with the main ones created by applications.
+
+The end result is that an application based on the Jetty libraries is a _tree_ of components.
+In server application the root of the component tree is a `Server` instance, while in client applications the root of the component tree is an `HttpClient` instance.
+
+Having all the Jetty components in a tree is beneficial in a number of use cases.
+It makes possible to register the components in the tree as JMX MBeans (TODO: xref the JMX section) so that a JMX console can look at the internal state of the components.
+It also makes possible to dump the component tree (and therefore each component's internal state) to a log file or to the console for troubleshooting purposes (TODO: xref troubleshooting section).
+
+[[eg-arch-bean-lifecycle]]
+==== Jetty Component Lifecycle
+
+Jetty components typically have a life cycle: they can be started and stopped.
+The Jetty components that have a life cycle implement the `org.eclipse.jetty.util.component.LifeCycle` interface.
+
+Jetty components that contain other components extend the `org.eclipse.jetty.util.component.ContainerLifeCycle` class.
+`ContainerLifeCycle` can contain these type of components, also called __bean__s:
+
+* _managed_ beans, `LifeCycle` instances whose life cycle is tied to the life cycle of their container
+* _unmanaged_ beans, `LifeCycle` instances whose life cycle is _not_ tied to the life cycle of their container
+* _POJO_ (Plain Old Java Object) beans, instances that do not implement `LifeCycle`
+
+`ContainerLifeCycle` uses the following logic to determine if a bean should be _managed_, _unmanaged_ or _POJO_:
+
+* the bean implements `LifeCycle`
+** the bean is not started, add it as _managed_
+** the bean is started, add it as _unmanaged_
+* the bean does not implement `LifeCycle`, add it as _POJO_
+
+When a `ContainerLifeCycle` is started, it also starts recursively all its managed beans (if they implement `LifeCycle`); unmanaged beans are not started during the `ContainerLifeCycle` start cycle.
+Likewise, stopping a `ContainerLifeCycle` stops recursively and in reverse order all its managed beans; unmanaged beans are not stopped during the `ContainerLifeCycle` stop cycle.
+
+Components can also be started and stopped individually, therefore activating or deactivating the functionalities that they offer.
+
+Applications should first compose components in the desired structure, and then start the root component:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/ComponentDocs.java[tags=start]
+----
+
+The component tree is the following:
+
+[source,screen]
+----
+Root
+├── Monitor (MANAGED)
+└── Service (MANAGED)
+    └── ScheduledExecutorService (POJO)
+----
+
+When the `Root` instance is created, also the `Monitor` instance is created and added as bean, so `Monitor` is the first bean of `Root`.
+`Monitor` is a _managed_ bean, because it has been explicitly added to `Root` via `ContainerLifeCycle.addManaged(...)`.
+
+Then, the application creates a `Service` instance and adds it to `Root` via `ContainerLifeCycle.addBean(...)`, so `Service` is the second bean of `Root`.
+`Service` is a _managed_ bean too, because it is a `LifeCycle` and at the time it was added to `Root` is was not started.
+
+The `ScheduledExecutorService` within `Service` does not implement `LifeCycle` so it is added as a _POJO_ to `Service`.
+
+It is possible to stop and re-start any component in a tree, for example:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/ComponentDocs.java[tags=restart]
+----
+
+`Service` can be stopped independently of `Root`, and re-started.
+Starting and stopping a non-root component does not alter the structure of the component tree, just the state of the subtree starting from the component that has been stopped and re-started.
+
+[[eg-arch-bean-listener]]
+==== Jetty Component Listeners
+
+// TODO: LifeCycle.Listener
+// TODO: Container.Listener + InheritedListener

jetty-documentation/src/main/asciidoc/embedded-guide/arch-io.adoc

@@ -0,0 +1,181 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-arch-io]]
+=== Jetty I/O Architecture
+
+Jetty libraries (both client and server) use Java NIO to handle I/O, so that at its core Jetty I/O is completely non-blocking.
+
+[[eg-arch-io-selector-manager]]
+==== Jetty I/O: `SelectorManager`
+
+The core class of Jetty I/O is link:{JDURL}/org/eclipse/jetty/io/SelectorManager.html[`SelectorManager`].
+
+`SelectorManager` manages internally a configurable number of link:{JDURL}/org/eclipse/jetty/io/ManagedSelector.html[`ManagedSelector`]s.
+Each `ManagedSelector` wraps an instance of `java.nio.channels.Selector` that in turn manages a number of `java.nio.channels.SocketChannel` instances.
+
+NOTE: TODO: add image
+
+`SocketChannel` instances can be created by network clients when connecting to a server and by a network server when accepting connections from network clients.
+In both cases the `SocketChannel` instance is passed to `SelectorManager` (which passes it to `ManagedSelector` and eventually to `java.nio.channels.Selector`) to be registered for use within Jetty.
+
+It is possible for an application to create the `SocketChannel` instances outside Jetty, even perform some initial network traffic also outside Jetty (for example for authentication purposes), and then pass the `SocketChannel` instance to `SelectorManager` for use within Jetty.
+
+This example shows how a client can connect to a server:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/SelectorManagerDocs.java[tags=connect]
+----
+
+This example shows how a server accepts a client connection:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/SelectorManagerDocs.java[tags=accept]
+----
+
+[[eg-arch-io-endpoint-connection]]
+==== Jetty I/O: `EndPoint` and `Connection`
+
+``SocketChannel``s that are passed to `SelectorManager` are wrapped into two related components: an link:{JDURL}/org/eclipse/jetty/io/EndPoint.html[`EndPoint`] and a link:{JDURL}/org/eclipse/jetty/io/Connection.html[`Connection`].
+
+`EndPoint` is the Jetty abstraction for a `SocketChannel`: you can read bytes from an `EndPoint` via `EndPoint.fill(ByteBuffer)`, you can write bytes to an `EndPoint` via `EndPoint.flush(ByteBuffer...)` and `EndPoint.write(Callback, ByteBuffer...)`, you can close an `EndPoint` via `EndPoint.close()`, etc.
+
+`Connection` is the Jetty abstraction that is responsible to read bytes from the `EndPoint` and to deserialize the read bytes into objects.
+For example, a HTTP/1.1 server-side `Connection` implementation is responsible to deserialize HTTP/1.1 request bytes into a HTTP request object.
+Conversely, a HTTP/1.1 client-side `Connection` implementation is responsible to deserialize HTTP/1.1 response bytes into a HTTP response object.
+
+`Connection` is the abstraction that implements the reading side of a specific protocol such as HTTP/1.1, or HTTP/2, or WebSocket: it is able to read incoming communication in that protocol.
+
+The writing side for a specific protocol _may_ be implemented in the `Connection` but may also be implemented in other components, although eventually the bytes to be written will be written through the `EndPoint`.
+
+While there is primarily just one implementation of `EndPoint`,link:{JDURL}/org/eclipse/jetty/io/SocketChannelEndPoint.html[`SocketChannelEndPoint`] (used both on the client-side and on the server-side), there are many implementations of `Connection`, typically two for each protocol (one for the client-side and one for the server-side).
+
+The `EndPoint` and `Connection` pairs can be chained, for example in case of encrypted communication using the TLS protocol.
+There is an `EndPoint` and `Connection` TLS pair where the `EndPoint` reads the encrypted bytes from the network and the `Connection` decrypts them; next in the chain there is an `EndPoint` and `Connection` pair where the `EndPoint` "reads" decrypted bytes (provided by the previous `Connection`) and the `Connection` deserializes them into specific protocol objects (for example HTTP/2 frame objects).
+
+Certain protocols, such as WebSocket, start the communication with the server using one protocol (e.g. HTTP/1.1), but then change the communication to use another protocol (e.g. WebSocket).
+`EndPoint` supports changing the `Connection` object on-the-fly via `EndPoint.upgrade(Connection)`.
+This allows to use the HTTP/1.1 `Connection` during the initial communication and later to replace it with a WebSocket `Connection`.
+
+NOTE: TODO: add a section on `UpgradeFrom` and `UpgradeTo`?
+
+`SelectorManager` is an abstract class because while it knows how to create concrete `EndPoint` instances, it does not know how to create protocol specific `Connection` instances.
+
+Creating `Connection` instances is performed on the server-side by link:{JDURL}/org/eclipse/jetty/server/ConnectionFactory.html[`ConnectionFactory`]s and on the client-side by link:{JDURL}/org/eclipse/jetty/io/ClientConnectionFactory.html[`ClientConnectionFactory`]s
+
+On the server-side, the component that aggregates a `SelectorManager` with a set of ``ConnectionFactory``s is link:{JDURL}/org/eclipse/jetty/server/ServerConnector.html[`ServerConnector`]s, see xref:eg-server-io-arch[].
+
+On the client-side, the components that aggregates a `SelectorManager` with a set of ``ClientConnectionFactory``s are link:{JDURL}/org/eclipse/jetty/client/HttpClientTransport.html[`HttpClientTransport`] subclasses, see xref:eg-client-io-arch[].
+
+[[eg-arch-io-endpoint]]
+==== Jetty I/O: `EndPoint`
+
+The Jetty I/O library use Java NIO to handle I/O, so that I/O is non-blocking.
+
+At the Java NIO level, in order to be notified when a `SocketChannel` has data to be read, the `SelectionKey.OP_READ` flag must be set.
+
+In the Jetty I/O library, you can call `EndPoint.fillInterested(Callback)` to declare interest in the "read" (or "fill") event, and the `Callback` parameter is the object that is notified when such event occurs.
+
+At the Java NIO level, a `SocketChannel` is always writable, unless it becomes TCP congested.
+In order to be notified when a `SocketChannel` uncongests and it is therefore writable again, the `SelectionKey.OP_WRITE` flag must be set.
+
+In the Jetty I/O library, you can call `EndPoint.write(Callback, ByteBuffer...)` to write the ``ByteBuffer``s and the `Callback` parameter is the object that is notified when the whole write is finished (i.e. _all_ ``ByteBuffer``s have been fully written, even if they are delayed by TCP congestion/uncongestion).
+
+The `EndPoint` APIs abstract out the Java NIO details by providing non-blocking APIs based on `Callback` objects for I/O operations.
+The `EndPoint` APIs are typically called by `Connection` implementations, see xref:eg-arch-io-connection[this section].
+
+[[eg-arch-io-connection]]
+==== Jetty I/O: `Connection`
+
+`Connection` is the abstraction that deserializes incoming bytes into objects, for example a HTTP request object or a WebSocket frame object, that can be used by more abstract layers.
+
+`Connection` instances have two lifecycle methods:
+
+* `Connection.onOpen()`, invoked when the `Connection` is associated with the `EndPoint`
+* `Connection.onClose(Throwable)`, invoked when the `Connection` is disassociated from the `EndPoint`, where the `Throwable` parameter indicates whether the disassociation was normal (when the parameter is `null`) or was due to an error (when the parameter is not `null`)
+
+When a `Connection` is first created, it is not registered for any Java NIO event.
+It is therefore typical to implement `onOpen()` to call `EndPoint.fillInterested(Callback)` so that the `Connection` declares interest for read events and it is invoked (via the `Callback`) when the read event happens.
+
+Abstract class `AbstractConnection` partially implements `Connection` and provides simpler APIs.
+The example below shows a typical implementation that extends `AbstractConnection`:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/SelectorManagerDocs.java[tags=connection]
+----
+
+[[eg-arch-io-connection-listener]]
+===== Jetty I/O: `Connection.Listener`
+
+// TODO: Introduce Connection.Listener
+
+[[eg-arch-io-echo]]
+==== Jetty I/O: Network Echo
+
+With the concepts above it is now possible to write a simple, fully non-blocking, `Connection` implementation that simply echoes the bytes that it reads back to the other peer.
+
+A naive, but wrong, implementation may be the following:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/SelectorManagerDocs.java[tags=echo-wrong]
+----
+
+WARNING: The implementation above is wrong and leads to `StackOverflowError`.
+
+The problem with this implementation is that if the writes always complete synchronously (i.e. without being delayed by TCP congestion), you end up with this sequence of calls:
+
+----
+Connection.onFillable()
+  EndPoint.write()
+    Callback.succeeded()
+      Connection.onFillable()
+        EndPoint.write()
+          Callback.succeeded()
+          ...
+----
+
+which leads to `StackOverflowError`.
+
+This is a typical side effect of asynchronous programming using non-blocking APIs, and happens in the Jetty I/O library as well.
+
+NOTE: The callback is invoked synchronously for efficiency reasons.
+Submitting the invocation of the callback to an `Executor` to be invoked in a different thread would cause a context switch and make simple writes extremely inefficient.
+
+A correct implementation is the following:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/SelectorManagerDocs.java[tags=echo-correct]
+----
+
+The correct implementation performs consecutive reads in a loop (rather than recursively), but _only_ if the correspondent write is completed successfully.
+
+In order to detect whether the write is completed, a concurrent state machine is used.
+This is necessary because the notification of the completion of the write may happen in a different thread, while the original writing thread may still be changing the state.
+
+The original writing thread starts moves the state from `IDLE` to `WRITING`, then issues the actual `write()` call.
+The original writing thread then assumes that the `write()` did not complete and tries to move to the `PENDING` state just after the `write()`.
+If it fails to move from the `WRITING` state to the `PENDING` state, it means that the write was completed.
+Otherwise, the write is now `PENDING` and waiting for the callback to be notified of the completion at a later time.
+When the callback is notified of the `write()` completion, it checks whether the `write()` was `PENDING`, and if it was it resumes reading.
+
+NOTE: TODO: Introduce IteratingCallback?

jetty-documentation/src/main/asciidoc/embedded-guide/arch-listener.adoc

@@ -0,0 +1,33 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-arch-listener]]
+=== Jetty Listeners
+
+The Jetty architecture is based on xref:eg-arch-bean[components], typically organized in a component tree.
+These components have an internal state that varies with the component life cycle (that is, whether the component is started or stopped), as well as with the component use at runtime.
+The typical example is a thread pool, whose internal state -- such as the number of pooled threads or the job queue size -- changes as the thread pool is used by the running client or server.
+
+In many cases, the component state change produces an event that is broadcast to listeners.
+Applications can register listeners to these components to be notified of the events they produce.
+
+This section lists the listeners available in the Jetty components, but the events and listener APIs are discussed in the component specific sections.
+
+* xref:eg-arch-bean-listener[]
+* xref:eg-arch-io-connection-listener[]
+* xref:eg-server-http-channel-events[]

jetty-documentation/src/main/asciidoc/embedded-guide/arch.adoc

@@ -16,12 +16,10 @@
 // ========================================================================
 //
 
-[[http-client]]
-== HTTP Client
+[appendix]
+[[eg-arch]]
+== Jetty Architecture
 
-include::http-client-intro.adoc[]
-include::http-client-api.adoc[]
-include::http-client-cookie.adoc[]
-include::http-client-authentication.adoc[]
-include::http-client-proxy.adoc[]
-include::http-client-transport.adoc[]
+include::arch-bean.adoc[]
+include::arch-listener.adoc[]
+include::arch-io.adoc[]

jetty-documentation/src/main/asciidoc/embedded-guide/client/client-concepts.adoc

@@ -1,149 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-[[client-concepts]]
-=== Client Libraries Concepts
-
-The Jetty client libraries implement a network client speaking different protocols
-such as HTTP/1.1, HTTP/2, WebSocket and FastCGI.
-
-It is possible to implement your own custom protocol on top of the Jetty client
-libraries.
-
-NOTE: TODO: perhaps add a section about this.
-
-There are conceptually three layers that compose the Jetty client libraries, from
-more abstract to more concrete:
-
-. The API layer, that exposes semantic APIs to applications so that they can write
-code such as "GET me the resource at this URI"
-. The protocol layer, where the API request is converted into the appropriate
-protocol bytes, for example encrypted HTTP/2
-. The infrastructure layer, that handles the low level I/O and deals with network,
-buffer, threads, etc.
-
-Let's look at these layers starting from the more concrete (and low level) one
-and build up to the more abstract layer.
-
-[[client-concepts-infrastructure]]
-==== Client Libraries Infrastructure Layer
-
-The Jetty client libraries use the common I/O design described in
-link:#io-arch[this section].
-The main client-side component is the
-link:{JDURL}/org/eclipse/jetty/io/ClientConnector.html[`ClientConnector`].
-
-The `ClientConnector` primarily wraps the
-link:{JDURL}/org/eclipse/jetty/io/SelectorManager.html[`SelectorManager`]
-and aggregates other four components: the thread pool (in form of an `Executor`),
-the `Scheduler`, the `ByteBufferPool` and the `SslContextFactory.Client`.
-
-The `ClientConnector` is where you want to set those components after you
-have configured them.
-If you don't explicitly set those components on the `ClientConnector`, then
-appropriate defaults will be chosen when the `ClientConnector` starts.
-
-The simplest example that creates and starts a `ClientConnector`:
-
-[source,java,indent=0]
-----
-include::{docbits}/embedded/client/ClientConnectorDocSnippets.java[tags=simplest]
-----
-
-A more typical example:
-
-[source,java,indent=0]
-----
-include::{docbits}/embedded/client/ClientConnectorDocSnippets.java[tags=typical]
-----
-
-A more advanced example that customizes the `ClientConnector` by overriding
-factory methods:
-
-[source,java,indent=0]
-----
-include::{docbits}/embedded/client/ClientConnectorDocSnippets.java[tags=advanced]
-----
-
-Since `ClientConnector` is the component that handles the low-level network, it
-is also the component where you want to configure the parameters that control
-how it should handle the low-level network.
-
-The most common parameters are:
-
-* `ClientConnector.selectors`: the number of ``java.nio.Selector``s components
-(defaults to `1`) that are present to handle the ``SocketChannel``s opened by
-the `ClientConnector`. You typically want to increase the number of selectors
-only for those use cases where each selector should handle more than few hundreds
-_concurrent_ socket events.
-For example, one selector typically runs well for `250` _concurrent_ socket
-events; as a rule of thumb, you can multiply that number by `10` to obtain the
-number of opened sockets a selector can handle (`2500`), based on the assumption
-that not all the `2500` sockets will be active _at the same time_.
-* `ClientConnector.idleTimeout`: the duration of time after which
-`ClientConnector` closes a socket due to inactivity (defaults to `30` seconds).
-This is an important parameter to configure, and you typically want the client
-idle timeout to be shorter than the server idle timeout, to avoid race
-conditions where the client attempts to use a socket just before the client-side
-idle timeout expires, but the server-side idle timeout has already expired and
-the is already closing the socket.
-* `ClientConnector.connectBlocking`: whether the operation of connecting a
-socket to the server (i.e. `SocketChannel.connect(SocketAddress)`) must be a
-blocking or a non-blocking operation (defaults to `false`).
-For `localhost` or same datacenter hosts you want to set this parameter to
-`true` because DNS resolution will be immediate (and likely never fail).
-For generic Internet hosts (e.g. when you are implementing a web spider) you
-want to set this parameter to `false`.
-* `ClientConnector.connectTimeout`: the duration of time after which
-`ClientConnector` aborts a connection attempt to the server (defaults to `5`
-seconds).
-This time includes the DNS lookup time _and_ the TCP connect time.
-
-Please refer to the `ClientConnector`
-link:{JDURL}/org/eclipse/jetty/io/ClientConnector.html[javadocs]
-for the complete list of configurable parameters.
-
-Once the `ClientConnector` is configured and started, it can be used to connect
-to the server via `ClientConnector.connect(SocketAddress, Map<String, Object>)`
-which in turn will call `SocketChannel.connect(SocketAddress)`.
-
-
-// TODO: from down here, moved to io-arch.adoc
-
-When establishing a TCP connection to a server, applications need to tell
-`ClientConnector` how to create the `Connection` for that particular
-TCP connection.
-This is done via a
-link:{JDURL}/org/eclipse/jetty/io/ClientConnectionFactory.html[`ClientConnectionFactory`].
-that must be passed in the context `Map` as follows:
-
-[source,java,indent=0]
-----
-include::{docbits}/embedded/client/ClientConnectorDocSnippets.java[tags=connect]
-----
-
-
-TODO: expand on what is the API to use, what parameters the context Map must
-have, and basically how we can write a generic network client with it.
-
-[[client-concepts-protocol]]
-==== Client Libraries Protocol Layer
-
-The protocol layer builds on top of the infrastructure layer to generate the
-bytes to be written to the network and to parse the bytes received from the
-network.

jetty-documentation/src/main/asciidoc/embedded-guide/client/client-io-arch.adoc

@@ -0,0 +1,137 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-client-io-arch]]
+=== Client Libraries I/O Architecture
+
+The Jetty client libraries provide the basic components and APIs to implement a network client.
+
+They build on the common xref:eg-arch-io[Jetty I/O Architecture] and provide client specific concepts (such as establishing a connection to a server).
+
+There are conceptually two layers that compose the Jetty client libraries:
+
+. xref:eg-client-io-arch-network[The network layer], that handles the low level I/O and deals with buffers, threads, etc.
+. xref:eg-client-io-arch-protocol[The protocol layer], that handles the parsing of bytes read from the network and the generation of bytes to write to the network.
+
+[[eg-client-io-arch-network]]
+==== Client Libraries Network Layer
+
+The Jetty client libraries use the common I/O design described in link:#eg-arch-io[this section].
+The main client-side component is the link:{JDURL}/org/eclipse/jetty/io/ClientConnector.html[`ClientConnector`].
+
+The `ClientConnector` primarily wraps the link:{JDURL}/org/eclipse/jetty/io/SelectorManager.html[`SelectorManager`] and aggregates other four components:
+
+* a thread pool (in form of an `java.util.concurrent.Executor`)
+* a scheduler (in form of `org.eclipse.jetty.util.thread.Scheduler`)
+* a byte buffer pool (in form of `org.eclipse.jetty.io.ByteBufferPool`)
+* a TLS factory (in form of `org.eclipse.jetty.util.ssl.SslContextFactory.Client`)
+
+The `ClientConnector` is where you want to set those components after you have configured them.
+If you don't explicitly set those components on the `ClientConnector`, then appropriate defaults will be chosen when the `ClientConnector` starts.
+
+The simplest example that creates and starts a `ClientConnector` is the following:
+
+[source,java,indent=0]
+----
+include::../{doc_code}/embedded/client/ClientConnectorDocs.java[tags=simplest]
+----
+
+A more typical example:
+
+[source,java,indent=0]
+----
+include::../{doc_code}/embedded/client/ClientConnectorDocs.java[tags=typical]
+----
+
+A more advanced example that customizes the `ClientConnector` by overriding some of its methods:
+
+[source,java,indent=0]
+----
+include::../{doc_code}/embedded/client/ClientConnectorDocs.java[tags=advanced]
+----
+
+Since `ClientConnector` is the component that handles the low-level network, it is also the component where you want to configure the low-level network configuration.
+
+The most common parameters are:
+
+* `ClientConnector.selectors`: the number of ``java.nio.Selector``s components (defaults to `1`) that are present to handle the ``SocketChannel``s opened by the `ClientConnector`.
+You typically want to increase the number of selectors only for those use cases where each selector should handle more than few hundreds _concurrent_ socket events.
+For example, one selector typically runs well for `250` _concurrent_ socket events; as a rule of thumb, you can multiply that number by `10` to obtain the number of opened sockets a selector can handle (`2500`), based on the assumption that not all the `2500` sockets will be active _at the same time_.
+* `ClientConnector.idleTimeout`: the duration of time after which `ClientConnector` closes a socket due to inactivity (defaults to `30` seconds).
+This is an important parameter to configure, and you typically want the client idle timeout to be shorter than the server idle timeout, to avoid race conditions where the client attempts to use a socket just before the client-side idle timeout expires, but the server-side idle timeout has already expired and the is already closing the socket.
+* `ClientConnector.connectBlocking`: whether the operation of connecting a socket to the server (i.e. `SocketChannel.connect(SocketAddress)`) must be a blocking or a non-blocking operation (defaults to `false`).
+For `localhost` or same datacenter hosts you want to set this parameter to
+`true` because DNS resolution will be immediate (and likely never fail).
+For generic Internet hosts (e.g. when you are implementing a web spider) you want to set this parameter to `false`.
+* `ClientConnector.connectTimeout`: the duration of time after which `ClientConnector` aborts a connection attempt to the server (defaults to `5` seconds).
+This time includes the DNS lookup time _and_ the TCP connect time.
+
+Please refer to the `ClientConnector` link:{JDURL}/org/eclipse/jetty/io/ClientConnector.html[javadocs] for the complete list of configurable parameters.
+
+[[eg-client-io-arch-protocol]]
+==== Client Libraries Protocol Layer
+
+The protocol layer builds on top of the network layer to generate the bytes to be written to the network and to parse the bytes read from the network.
+
+Recall from link:#eg-arch-io-connection[this section] that Jetty uses the `Connection` abstraction to produce and interpret the network bytes.
+
+On the client side, a `ClientConnectionFactory` implementation is the component that creates `Connection` instances based on the protocol that the client wants to "speak" with the server.
+
+Applications use `ClientConnector.connect(SocketAddress, Map<String, Object>)` to establish a TCP connection to the server, and must tell `ClientConnector` how to create the `Connection` for that particular TCP connection, and how to notify back the application when the connection creation succeeds or fails.
+
+This is done by passing a link:{JDURL}/org/eclipse/jetty/io/ClientConnectionFactory.html[`ClientConnectionFactory`] (that creates `Connection` instances) and a link:{JDURL}/org/eclipse/jetty/util/Promise.html[`Promise`] (that is notified of connection creation success or failure) in the context `Map` as follows:
+
+[source,java,indent=0]
+----
+include::../{doc_code}/embedded/client/ClientConnectorDocs.java[tags=connect]
+----
+
+When a `Connection` is created successfully, its `onOpen()` method is invoked, and then the promise is completed successfully.
+
+It is now possible to write a super-simple `telnet` client that reads and writes string lines:
+
+[source,java,indent=0]
+----
+include::../{doc_code}/embedded/client/ClientConnectorDocs.java[tags=telnet]
+----
+
+Note how a very basic "telnet" API that applications could use is implemented in the form of the `onLine(Consumer<String>)` for the non-blocking receiving side and `writeLine(String, Callback)` for the non-blocking sending side.
+Note also how the `onFillable()` method implements some basic "parsing" by looking up the `\n` character in the buffer.
+
+NOTE: The "telnet" client above looks like a super-simple HTTP client because HTTP/1.0 can be seen as a line-based protocol.
+HTTP/1.0 was used just as an example, but we could have used any other line-based protocol such as link:https://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol[SMTP], provided that the server was able to understand it.
+
+This is very similar to what the Jetty client implementation does for real network protocols.
+Real network protocols are of course more complicated and so is the implementation code that handles them, but the general ideas are similar.
+
+The Jetty client implementation provides a number of `ClientConnectionFactory` implementations that can be composed to produce and interpret the network bytes.
+
+For example, it is simple to modify the above example to use the TLS protocol so that you will be able to connect to the server on port `443`, typically reserved for the encrypted HTTP protocol.
+
+The differences between the clear-text version and the TLS encrypted version are minimal:
+
+[source,java,indent=0]
+----
+include::../{doc_code}/embedded/client/ClientConnectorDocs.java[tags=tlsTelnet]
+----
+
+The differences with the clear-text version are only:
+
+* Change the port from `80` to `443`.
+* Wrap the `ClientConnectionFactory` with `SslClientConnectionFactory`.
+* Unwrap the `SslConnection` to access `TelnetConnection`.

jetty-documentation/src/main/asciidoc/embedded-guide/client/client.adoc

@@ -16,20 +16,23 @@
 // ========================================================================
 //
 
-[[client]]
-== Jetty Client Libraries
+[[eg-client]]
+== Client Libraries
 
-The Eclipse Jetty Project provides not only server-side libraries so that you
-can embed a server in your code, but it also provides client-side libraries
-that allow you to embed a client - for example a HTTP client invoking a third
-party HTTP service - in your application.
+The Eclipse Jetty Project provides also provides client-side libraries that allow you to embed a client in your applications.
+A typical example is a client application that needs to contact a third party service via HTTP (for example a REST service).
+Another example is a proxy application that receives HTTP requests and forwards them as FCGI requests to a PHP application such as WordPress, or receives HTTP/1.1 requests and converts them to HTTP/2. Yet another example is a client application that needs to received events from a WebSocket server.
 
-The client libraries are designed to be non-blocking and offer both synchronous
-and asynchronous APIs and come with a large number of configuration options.
+The client libraries are designed to be non-blocking and offer both synchronous and asynchronous APIs and come with a large number of configuration options.
 
-There are primarily two client libraries:
+These are the available client libraries:
 
-* link:#client-http[The HTTP client library]
-* link:#client-websocket[The WebSocket client library]
+* xref:eg-client-http[The HTTP Client Library]
+* xref:eg-client-http2[The HTTP/2 Client Library]
+* xref:eg-client-websocket[The WebSocket client library]
 
-include::client-concepts.adoc[]
+If you are interested in the low-level details of how the Eclipse Jetty client libraries work, or are interested in writing a custom protocol, look at the xref:eg-client-io-arch[Client I/O Architecture].
+
+include::http/client-http.adoc[]
+include::http2/client-http2.adoc[]
+include::client-io-arch.adoc[]

jetty-documentation/src/main/asciidoc/embedded-guide/client/http/client-http-api.adoc

@@ -0,0 +1,264 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-client-http-api]]
+=== HttpClient API Usage
+
+`HttpClient` provides two types of APIs: a blocking API and a non-blocking API.
+
+[[eg-client-http-blocking]]
+==== HttpClient Blocking APIs
+
+The simpler way to perform a HTTP request is the following:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=simpleBlockingGet]
+----
+
+The method `HttpClient.GET(...)` performs a HTTP `GET` request to the given URI and returns a `ContentResponse` when the request/response conversation completes successfully.
+
+The `ContentResponse` object contains the HTTP response information: status code, headers and possibly content.
+The content length is limited by default to 2 MiB; for larger content see xref:client-http-content-response[].
+
+If you want to customize the request, for example by issuing a `HEAD` request instead of a `GET`, and simulating a browser user agent, you can do it in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=headFluent]
+----
+
+This is a shorthand for:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=headNonFluent]
+----
+
+You first create a request object using `httpClient.newRequest(...)`, and then you customize it using the fluent API style (that is, a chained invocation of methods on the request object).
+When the request object is customized, you call `request.send()` that produces the `ContentResponse` when the request/response conversation is complete.
+
+Simple `POST` requests also have a shortcut method:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=postFluent]
+----
+
+The `POST` parameter values added via the `param()` method are automatically URL-encoded.
+
+Jetty's `HttpClient` automatically follows redirects, so it handles the typical web pattern http://en.wikipedia.org/wiki/Post/Redirect/Get[POST/Redirect/GET], and the response object contains the content of the response of the `GET` request.
+Following redirects is a feature that you can enable/disable on a per-request basis or globally.
+
+File uploads also require one line, and make use of `java.nio.file` classes:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=fileFluent]
+----
+
+It is possible to impose a total timeout for the request/response conversation using the `Request.timeout(...)` method as follows:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=totalTimeout]
+----
+
+In the example above, when the 5 seconds expire, the request is aborted and a `java.util.concurrent.TimeoutException` is thrown.
+
+[[eg-client-http-non-blocking]]
+==== HttpClient Non-Blocking APIs
+
+So far we have shown how to use Jetty HTTP client in a blocking style - that is, the thread that issues the request blocks until the request/response conversation is complete.
+
+This section will look at Jetty's `HttpClient` non-blocking, asynchronous APIs that are perfectly suited for large content downloads, for parallel processing of requests/responses and in cases where performance and efficient thread and resource utilization is a key factor.
+
+The asynchronous APIs rely heavily on listeners that are invoked at various stages of request and response processing.
+These listeners are implemented by applications and may perform any kind of logic.
+The implementation invokes these listeners in the same thread that is used to process the request or response.
+Therefore, if the application code in these listeners takes a long time to execute, the request or response processing is delayed until the listener returns.
+
+If you need to execute application code that takes long time inside a listener, you must spawn your own thread.
+
+Request and response processing are executed by two different threads and therefore may happen concurrently.
+A typical example of this concurrent processing is an echo server, where a large upload may be concurrent with the large download echoed back.
+As a side note, remember that responses may be processed and completed _before_ requests; a typical example is a large upload that triggers a quick response - for example an error - by the server: the response may arrive and be completed while the request content is still being uploaded.
+
+The application thread that calls `Request.send(Response.CompleteListener)` performs the processing of the request until either the request is fully processed or until it would block on I/O, then it returns (and therefore never blocks).
+If it would block on I/O, the thread asks the I/O system to emit an event when the I/O will be ready to continue, then returns.
+When such an event is fired, a thread taken from the `HttpClient` thread pool will resume the processing of the request.
+
+Response are processed from the I/O thread that fires the event that bytes are ready to be read.
+Response processing continues until either the response is fully processed or until it would block for I/O.
+If it would block for I/O, the thread asks the I/O system to emit an event when the I/O will be ready to continue, then returns.
+When such an event is fired, a thread taken from the `HttpClient` thread pool will resume the processing of the response.
+
+When the request and the response are both fully processed, the thread that finished the last processing (usually the thread that processes the response, but may also be the thread that processes the request - if the request takes more time than the response to be processed) is used to dequeue the next request for the same destination and processes it.
+
+A simple non-blocking `GET` request that discards the response content can be written in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=simpleNonBlocking]
+----
+
+Method `Request.send(Response.CompleteListener)` returns `void` and does not block; the `Response.CompleteListener` lambda provided as a parameter is notified when the request/response conversation is complete, and the `Result` parameter allows you to access the request and response objects as well as failures, if any.
+
+You can impose a total timeout for the request/response conversation in the same way used by the synchronous API:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=nonBlockingTotalTimeout]
+----
+
+The example above will impose a total timeout of 3 seconds on the request/response conversation.
+
+The HTTP client APIs use listeners extensively to provide hooks for all possible request and response events:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=listeners]
+----
+
+This makes Jetty HTTP client suitable for HTTP load testing because, for example, you can accurately time every step of the request/response conversation (thus knowing where the request/response time is really spent).
+
+Have a look at the link:{JDURL}/org/eclipse/jetty/client/api/Request.Listener.html[`Request.Listener`] class to know about request events, and to the link:{JDURL}/org/eclipse/jetty/client/api/Response.Listener.html[`Response.Listener`] class to know about response events.
+
+[[eg-client-http-content]]
+==== HttpClient Content Handling
+
+[[eg-client-http-content-request]]
+===== Request Content Handling
+
+Jetty's `HttpClient` provides a number of utility classes off the shelf to handle request content.
+
+You can provide request content as `String`, `byte[]`, `ByteBuffer`, `java.nio.file.Path`, `InputStream`, and provide your own implementation of `org.eclipse.jetty.client.api.Request.Content`.
+Here’s an example that provides the request content using `java.nio.file.Paths`:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=pathRequestContent]
+----
+
+Alternatively, you can use `FileInputStream` via the `InputStreamRequestContent` utility class:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=inputStreamRequestContent]
+----
+
+Since `InputStream` is blocking, then also the send of the request will block if the input stream blocks, even in case of usage of the non-blocking `HttpClient` APIs.
+
+If you have already read the content in memory, you can pass it as a `byte[]` (or a `String`) using the `BytesRequestContent` (or `StringRequestContent`) utility class:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=bytesStringRequestContent]
+----
+
+If the request content is not immediately available, but your application will be notified of the content to send, you can use `AsyncRequestContent` in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=asyncRequestContent]
+----
+
+While the request content is awaited and consequently uploaded by the client application, the server may be able to respond (at least with the response headers) completely asynchronously.
+In this case, `Response.Listener` callbacks will be invoked before the request is fully sent.
+This allows fine-grained control of the request/response conversation: for example the server may reject contents that are too big, send a response to the client, which in turn may stop the content upload.
+
+Another way to provide request content is by using an `OutputStreamRequestContent`, which allows applications to write request content when it is available to the `OutputStream` provided by `OutputStreamRequestContent`:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=outputStreamRequestContent]
+----
+
+[[eg-client-http-content-response]]
+===== Response Content Handling
+
+Jetty's `HttpClient` allows applications to handle response content in different ways.
+
+You can buffer the response content in memory; this is done when using the xref:client-http-blocking[blocking APIs] and the content is buffered within a `ContentResponse` up to 2 MiB.
+
+If you want to control the length of the response content (for example limiting to values smaller than the default of 2 MiB), then you can use a `org.eclipse.jetty.client.util.FutureResponseListener` in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=futureResponseListener]
+----
+
+If the response content length is exceeded, the response will be aborted, and an exception will be thrown by method `get(...)`.
+
+You can buffer the response content in memory also using the xref:client-http-non-blocking[non-blocking APIs], via the `BufferingResponseListener` utility class:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=bufferingResponseListener]
+----
+
+If you want to avoid buffering, you can wait for the response and then stream the content using the `InputStreamResponseListener` utility class:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=inputStreamResponseListener]
+----
+
+Finally, let's look at the advanced usage of the response content handling.
+
+The response content is provided by the `HttpClient` implementation to application listeners following a reactive model similar to that of `java.util.concurrent.Flow`.
+
+The listener that follows this model is `Response.DemandedContentListener`.
+
+After the response headers have been processed by the `HttpClient` implementation, `Response.DemandedContentListener.onBeforeContent(response, demand)` is invoked.
+This allows the application to control whether to demand the first content or not.
+The default implementation of this method calls `demand.accept(1)`, which demands one chunk of content to the implementation.
+The implementation will deliver the chunk of content as soon as it is available.
+
+The chunks of content are delivered to the application by invoking `Response.DemandedContentListener.onContent(response, demand, buffer, callback)`.
+Applications implement this method to process the content bytes in the `buffer`.
+Succeeding the `callback` signals to the implementation that the application has consumed the `buffer` so that the implementation can dispose/recycle the `buffer`.
+Failing the `callback` signals to the implementation to fail the response (no more content will be delivered, and the _response failed_ event will be emitted).
+
+IMPORTANT: Succeeding the `callback` must be done only after the `buffer` bytes have been consumed.
+When the `callback` is succeeded, the `HttpClient` implementation may reuse the `buffer` and overwrite the bytes with different bytes; if the application looks at the `buffer` _after_ having succeeded the `callback` is may see other, unrelated, bytes.
+
+The application uses the `demand` object to demand more content chunks.
+Applications will typically demand for just one more content via `demand.accept(1)`, but may decide to demand for more via `demand.accept(2)` or demand "infinitely" once via `demand.accept(Long.MAX_VALUE)`.
+Applications that demand for more than 1 chunk of content must be prepared to receive all the content that they have demanded.
+
+Demanding for content and consuming the content are orthogonal activities.
+
+An application can demand "infinitely" and store aside the pairs `(buffer, callback)` to consume them later.
+If not done carefully, this may lead to excessive memory consumption, since the ``buffer``s are not consumed.
+Succeeding the ``callback``s will result in the ``buffer``s to be disposed/recycled and may be performed at any time.
+
+An application can also demand one chunk of content, consume it (by succeeding the associated `callback`) and then _not_ demand for more content until a later time.
+
+Subclass `Response.AsyncContentListener` overrides the behavior of `Response.DemandedContentListener`; when an application implementing its `onContent(response, buffer, callback)` succeeds the `callback`, it will have _both_ the effect of disposing/recycling the `buffer` _and_ the effect of demanding one more chunk of content.
+
+Subclass `Response.ContentListener` overrides the behavior of `Response.AsyncContentListener`; when an application implementing its `onContent(response, buffer)` returns from the method itself, it will _both_ the effect of disposing/recycling the `buffer` _and_ the effect of demanding one more chunk of content.
+
+Previous examples of response content handling were inefficient because they involved copying the `buffer` bytes, either to accumulate them aside so that the application could use them when the request was completed, or because they were provided to an API such as `InputStream` that made use of `byte[]` (and therefore a copy from `ByteBuffer` to `byte[]` is necessary).
+
+An application that implements a forwarder between two servers can be implemented efficiently by handling the response content without copying the `buffer` bytes as in the following example:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=demandedContentListener]
+----

jetty-documentation/src/main/asciidoc/embedded-guide/client/http/client-http-authentication.adoc

@@ -0,0 +1,81 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-client-http-authentication]]
+=== HttpClient Authentication Support
+
+Jetty's `HttpClient` supports the `BASIC` and `DIGEST` authentication mechanisms defined by link:https://tools.ietf.org/html/rfc7235[RFC 7235], as well as the SPNEGO authentication mechanism defined in link:https://tools.ietf.org/html/rfc4559[RFC 4559].
+
+The HTTP _conversation_ - the sequence of related HTTP requests - for a request that needs authentication is the following:
+
+[plantuml]
+----
+skinparam backgroundColor transparent
+skinparam monochrome true
+skinparam shadowing false
+
+participant Application
+participant HttpClient
+participant Server
+
+Application -> Server : GET /path
+Server -> HttpClient : 401 + WWW-Authenticate
+HttpClient -> Server : GET + Authentication
+Server -> Application : 200 OK
+----
+
+Upon receiving a HTTP 401 response code, `HttpClient` looks at the `WWW-Authenticate` response header (the server _challenge_) and then tries to match configured authentication credentials to produce an `Authentication` header that contains the authentication credentials to access the resource.
+
+You can configure authentication credentials in the `HttpClient` instance as follows:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=addAuthentication]
+----
+
+``Authentication``s are matched against the server challenge first by mechanism (e.g. `BASIC` or `DIGEST`), then by realm and then by URI.
+
+If an `Authentication` match is found, the application does not receive events related to the HTTP 401 response.
+These events are handled internally by `HttpClient` which produces another (internal) request similar to the original request but with an additional `Authorization` header.
+
+If the authentication is successful, the server responds with a HTTP 200 and `HttpClient` caches the `Authentication.Result` so that subsequent requests for a matching URI will not incur in the additional rountrip caused by the HTTP 401 response.
+
+It is possible to clear ``Authentication.Result``s in order to force authentication again:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=clearResults]
+----
+
+Authentication results may be preempted to avoid the additional roundtrip due to the server challenge in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=preemptedResult]
+----
+
+In this way, requests for the given URI are enriched immediately with the `Authorization` header, and the server should respond with HTTP 200 (and the resource content) rather than with the 401 and the challenge.
+
+It is also possible to preempt the authentication for a single request only, in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=requestPreemptedResult]
+----
+
+See also the xref:eg-client-http-proxy-authentication[proxy authentication section] for further information about how authentication works with HTTP proxies.