Merge remote-tracking branch 'origin/jetty-10.0.x' into jetty-10.0.x-restore-owb-hack

examples/embedded/src/test/java/org/eclipse/jetty/embedded/ExampleServerTest.java

@@ -21,7 +21,7 @@ package org.eclipse.jetty.embedded;
 import java.net.URI;
 
 import org.eclipse.jetty.client.api.ContentResponse;
-import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.client.util.StringRequestContent;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.server.Server;
@@ -75,7 +75,7 @@ public class ExampleServerTest extends AbstractEmbeddedTest
         String postBody = "Greetings from " + ExampleServerTest.class;
         ContentResponse response = client.newRequest(uri)
             .method(HttpMethod.POST)
-            .content(new StringContentProvider(postBody))
+            .body(new StringRequestContent(postBody))
             .send();
 
         // Check the response status code

jetty-alpn/jetty-alpn-client/src/main/java/org/eclipse/jetty/alpn/client/ALPNClientConnection.java

@@ -44,9 +44,6 @@ public class ALPNClientConnection extends NegotiatingClientConnection
 
     public void selected(String protocol)
     {
-        if (protocol == null || !protocols.contains(protocol))
+        completed(protocol);
-            close();
-        else
-            completed(protocol);
     }
 }

jetty-alpn/jetty-alpn-client/src/main/java/org/eclipse/jetty/alpn/client/ALPNClientConnectionFactory.java

@@ -110,12 +110,4 @@ public class ALPNClientConnectionFactory extends NegotiatingClientConnectionFact
         }
         throw new IllegalStateException("No ALPNProcessor for " + engine);
     }
-
-    public static class ALPN extends Info
-    {
-        public ALPN(Executor executor, ClientConnectionFactory factory, List<String> protocols)
-        {
-            super(List.of("alpn"), new ALPNClientConnectionFactory(executor, factory, protocols));
-        }
-    }
 }

jetty-alpn/jetty-alpn-java-client/src/main/java/org/eclipse/jetty/alpn/java/client/JDK9ClientALPNProcessor.java

@@ -75,8 +75,11 @@ public class JDK9ClientALPNProcessor implements ALPNProcessor.Client
         {
             String protocol = alpnConnection.getSSLEngine().getApplicationProtocol();
             if (LOG.isDebugEnabled())
-                LOG.debug("selected protocol {}", protocol);
+                LOG.debug("selected protocol '{}'", protocol);
-            alpnConnection.selected(protocol);
+            if (protocol != null && !protocol.isEmpty())
+                alpnConnection.selected(protocol);
+            else
+                alpnConnection.selected(null);
         }
     }
 }

jetty-annotations/src/main/java/org/eclipse/jetty/annotations/RunAsAnnotationHandler.java

@@ -21,7 +21,6 @@ package org.eclipse.jetty.annotations;
 import javax.servlet.Servlet;
 
 import org.eclipse.jetty.annotations.AnnotationIntrospector.AbstractIntrospectableAnnotationHandler;
-import org.eclipse.jetty.plus.annotation.RunAsCollection;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.webapp.Descriptor;
 import org.eclipse.jetty.webapp.MetaData;
@@ -61,14 +60,7 @@ public class RunAsAnnotationHandler extends AbstractIntrospectableAnnotationHand
                     if (d == null)
                     {
                         metaData.setOrigin(holder.getName() + ".servlet.run-as", runAs, clazz);
-                        org.eclipse.jetty.plus.annotation.RunAs ra = new org.eclipse.jetty.plus.annotation.RunAs(clazz.getName(), role);
+                        holder.setRunAsRole(role);
-                        RunAsCollection raCollection = (RunAsCollection)_context.getAttribute(RunAsCollection.RUNAS_COLLECTION);
-                        if (raCollection == null)
-                        {
-                            raCollection = new RunAsCollection();
-                            _context.setAttribute(RunAsCollection.RUNAS_COLLECTION, raCollection);
-                        }
-                        raCollection.add(ra);
                     }
                 }
             }

jetty-annotations/src/test/java/org/eclipse/jetty/annotations/TestAnnotationConfiguration.java

@@ -118,25 +118,25 @@ public class TestAnnotationConfiguration
     @Test
     public void testAnnotationScanControl() throws Exception
     {
-        //check that a 2.5 webapp won't discover annotations
+        //check that a 2.5 webapp with configurationDiscovered will discover annotations
         TestableAnnotationConfiguration config25 = new TestableAnnotationConfiguration();
         WebAppContext context25 = new WebAppContext();
         context25.setClassLoader(Thread.currentThread().getContextClassLoader());
         context25.setAttribute(AnnotationConfiguration.MULTI_THREADED, Boolean.FALSE);
         context25.setAttribute(AnnotationConfiguration.MAX_SCAN_WAIT, 0);
+        context25.setConfigurationDiscovered(false);
         context25.getMetaData().setWebDescriptor(new WebDescriptor(Resource.newResource(web25)));
         context25.getServletContext().setEffectiveMajorVersion(2);
         context25.getServletContext().setEffectiveMinorVersion(5);
         config25.configure(context25);
         config25.assertAnnotationDiscovery(false);
 
-        //check that a 2.5 webapp with configurationDiscovered will discover annotations
+        //check that a 2.5 webapp discover annotations
         TestableAnnotationConfiguration config25b = new TestableAnnotationConfiguration();
         WebAppContext context25b = new WebAppContext();
         context25b.setClassLoader(Thread.currentThread().getContextClassLoader());
         context25b.setAttribute(AnnotationConfiguration.MULTI_THREADED, Boolean.FALSE);
         context25b.setAttribute(AnnotationConfiguration.MAX_SCAN_WAIT, 0);
-        context25b.setConfigurationDiscovered(true);
         context25b.getMetaData().setWebDescriptor(new WebDescriptor(Resource.newResource(web25)));
         context25b.getServletContext().setEffectiveMajorVersion(2);
         context25b.getServletContext().setEffectiveMinorVersion(5);
@@ -293,6 +293,7 @@ public class TestAnnotationConfiguration
             AnnotationConfiguration config = new AnnotationConfiguration();
             WebAppContext context = new WebAppContext();
             List<ServletContainerInitializer> scis;
+            context.setConfigurationDiscovered(false);
             context.setClassLoader(webAppLoader);
             context.getMetaData().setWebDescriptor(new WebDescriptor(Resource.newResource(web25)));
             context.getMetaData().setWebInfClassesResources(classes);

jetty-annotations/src/test/java/org/eclipse/jetty/annotations/TestRunAsAnnotation.java

@@ -0,0 +1,63 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.annotations;
+
+import java.io.File;
+
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
+import org.eclipse.jetty.util.resource.Resource;
+import org.eclipse.jetty.webapp.WebAppContext;
+import org.eclipse.jetty.webapp.WebDescriptor;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TestRunAsAnnotation
+{
+    @Test
+    public void testRunAsAnnotation() throws Exception
+    {
+        WebAppContext wac = new WebAppContext();
+        
+        //pre-add a servlet but not by descriptor
+        ServletHolder holder = new ServletHolder();
+        holder.setName("foo1");
+        holder.setHeldClass(ServletC.class);
+        holder.setInitOrder(1); //load on startup
+        wac.getServletHandler().addServletWithMapping(holder, "/foo/*");
+        
+        //add another servlet of the same class, but as if by descriptor
+        ServletHolder holder2 = new ServletHolder();
+        holder2.setName("foo2");
+        holder2.setHeldClass(ServletC.class);
+        holder2.setInitOrder(1);
+        wac.getServletHandler().addServletWithMapping(holder2, "/foo2/*");
+        Resource fakeXml = Resource.newResource(new File(MavenTestingUtils.getTargetTestingDir("run-as"), "fake.xml"));
+        wac.getMetaData().setOrigin(holder2.getName() + ".servlet.run-as", new WebDescriptor(fakeXml));
+        
+        AnnotationIntrospector parser = new AnnotationIntrospector(wac);
+        RunAsAnnotationHandler handler = new RunAsAnnotationHandler(wac);
+        parser.registerHandler(handler);
+        parser.introspect(new ServletC(), null);
+        
+        assertEquals("admin", holder.getRunAsRole());
+        assertEquals(null, holder2.getRunAsRole());
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/AbstractConnectorHttpClientTransport.java

@@ -72,7 +72,7 @@ public abstract class AbstractConnectorHttpClientTransport extends AbstractHttpC
         context.put(ClientConnector.CLIENT_CONNECTION_FACTORY_CONTEXT_KEY, destination.getClientConnectionFactory());
         @SuppressWarnings("unchecked")
         Promise<Connection> promise = (Promise<Connection>)context.get(HTTP_CONNECTION_PROMISE_CONTEXT_KEY);
-        context.put(ClientConnector.CONNECTION_PROMISE_CONTEXT_KEY, promise);
+        context.put(ClientConnector.CONNECTION_PROMISE_CONTEXT_KEY, Promise.from(ioConnection -> {}, promise::failed));
         connector.connect(address, context);
     }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/AsyncContentProvider.java

@@ -21,10 +21,14 @@ package org.eclipse.jetty.client;
 import java.util.EventListener;
 
 import org.eclipse.jetty.client.api.ContentProvider;
+import org.eclipse.jetty.client.api.Request;
 
 /**
  * A {@link ContentProvider} that notifies listeners that content is available.
+ *
+ * @deprecated no replacement, use {@link Request.Content} instead.
  */
+@Deprecated
 public interface AsyncContentProvider extends ContentProvider
 {
     /**

jetty-client/src/main/java/org/eclipse/jetty/client/AuthenticationProtocolHandler.java

@@ -30,7 +30,6 @@ import java.util.regex.Pattern;
 import org.eclipse.jetty.client.api.Authentication;
 import org.eclipse.jetty.client.api.Authentication.HeaderInfo;
 import org.eclipse.jetty.client.api.Connection;
-import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
@@ -187,8 +186,8 @@ public abstract class AuthenticationProtocolHandler implements ProtocolHandler
                 return;
             }
 
-            ContentProvider requestContent = request.getContent();
+            Request.Content requestContent = request.getBody();
-            if (requestContent != null && !requestContent.isReproducible())
+            if (!requestContent.isReproducible())
             {
                 if (LOG.isDebugEnabled())
                     LOG.debug("Request content not reproducible for {}", request);

jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java

@@ -53,7 +53,7 @@ import org.eclipse.jetty.client.api.Destination;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
-import org.eclipse.jetty.client.util.FormContentProvider;
+import org.eclipse.jetty.client.util.FormRequestContent;
 import org.eclipse.jetty.http.HttpCompliance;
 import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpHeader;
@@ -380,7 +380,7 @@ public class HttpClient extends ContainerLifeCycle
      */
     public ContentResponse FORM(URI uri, Fields fields) throws InterruptedException, ExecutionException, TimeoutException
     {
-        return POST(uri).content(new FormContentProvider(fields)).send();
+        return POST(uri).body(new FormRequestContent(fields)).send();
     }
 
     /**
@@ -447,7 +447,7 @@ public class HttpClient extends ContainerLifeCycle
         Request newRequest = newHttpRequest(oldRequest.getConversation(), newURI);
         newRequest.method(oldRequest.getMethod())
             .version(oldRequest.getVersion())
-            .content(oldRequest.getContent())
+            .body(oldRequest.getBody())
             .idleTimeout(oldRequest.getIdleTimeout(), TimeUnit.MILLISECONDS)
             .timeout(oldRequest.getTimeout(), TimeUnit.MILLISECONDS)
             .followRedirects(oldRequest.isFollowRedirects());

jetty-client/src/main/java/org/eclipse/jetty/client/HttpConnection.java

@@ -27,9 +27,9 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
 import org.eclipse.jetty.client.api.Authentication;
-import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.util.BytesRequestContent;
 import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpVersion;
@@ -129,11 +129,6 @@ public abstract class HttpConnection implements IConnection
         if (normalized)
             return;
 
-        HttpVersion version = request.getVersion();
-        HttpFields headers = request.getHeaders();
-        ContentProvider content = request.getContent();
-        ProxyConfiguration.Proxy proxy = destination.getProxy();
-
         // Make sure the path is there
         String path = request.getPath();
         if (path.trim().length() == 0)
@@ -144,6 +139,7 @@ public abstract class HttpConnection implements IConnection
 
         URI uri = request.getURI();
 
+        ProxyConfiguration.Proxy proxy = destination.getProxy();
         if (proxy instanceof HttpProxy && !HttpClient.isSchemeSecure(request.getScheme()) && uri != null)
         {
             path = uri.toString();
@@ -151,6 +147,8 @@ public abstract class HttpConnection implements IConnection
         }
 
         // If we are HTTP 1.1, add the Host header
+        HttpVersion version = request.getVersion();
+        HttpFields headers = request.getHeaders();
         if (version.getVersion() <= 11)
         {
             if (!headers.containsKey(HttpHeader.HOST.asString()))
@@ -158,13 +156,16 @@ public abstract class HttpConnection implements IConnection
         }
 
         // Add content headers
-        if (content != null)
+        Request.Content content = request.getBody();
+        if (content == null)
+        {
+            request.body(new BytesRequestContent());
+        }
+        else
         {
             if (!headers.containsKey(HttpHeader.CONTENT_TYPE.asString()))
             {
-                String contentType = null;
+                String contentType = content.getContentType();
-                if (content instanceof ContentProvider.Typed)
-                    contentType = ((ContentProvider.Typed)content).getContentType();
                 if (contentType != null)
                 {
                     headers.put(HttpHeader.CONTENT_TYPE, contentType);

jetty-client/src/main/java/org/eclipse/jetty/client/HttpContent.java

@@ -1,237 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-package org.eclipse.jetty.client;
-
-import java.io.Closeable;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.Iterator;
-
-import org.eclipse.jetty.client.api.ContentProvider;
-import org.eclipse.jetty.util.BufferUtil;
-import org.eclipse.jetty.util.Callback;
-import org.eclipse.jetty.util.IO;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * {@link HttpContent} is a stateful, linear representation of the request content provided
- * by a {@link ContentProvider} that can be traversed one-way to obtain content buffers to
- * send to an HTTP server.
- * <p>
- * {@link HttpContent} offers the notion of a one-way cursor to traverse the content.
- * The cursor starts in a virtual "before" position and can be advanced using {@link #advance()}
- * until it reaches a virtual "after" position where the content is fully consumed.
- * <pre>
- *      +---+  +---+  +---+  +---+  +---+
- *      |   |  |   |  |   |  |   |  |   |
- *      +---+  +---+  +---+  +---+  +---+
- *   ^           ^                    ^    ^
- *   |           | --&gt; advance()      |    |
- *   |           |                  last   |
- *   |           |                         |
- * before        |                        after
- *               |
- *            current
- * </pre>
- * At each valid (non-before and non-after) cursor position, {@link HttpContent} provides the following state:
- * <ul>
- * <li>the buffer containing the content to send, via {@link #getByteBuffer()}</li>
- * <li>a copy of the content buffer that can be used for notifications, via {@link #getContent()}</li>
- * <li>whether the buffer to write is the last one, via {@link #isLast()}</li>
- * </ul>
- * {@link HttpContent} may not have content, if the related {@link ContentProvider} is {@code null}, and this
- * is reflected by {@link #hasContent()}.
- * <p>
- * {@link HttpContent} may have {@link AsyncContentProvider deferred content}, in which case {@link #advance()}
- * moves the cursor to a position that provides {@code null} {@link #getByteBuffer() buffer} and
- * {@link #getContent() content}. When the deferred content is available, a further call to {@link #advance()}
- * will move the cursor to a position that provides non {@code null} buffer and content.
- */
-public class HttpContent implements Callback, Closeable
-{
-    private static final Logger LOG = LoggerFactory.getLogger(HttpContent.class);
-    private static final ByteBuffer AFTER = ByteBuffer.allocate(0);
-    private static final ByteBuffer CLOSE = ByteBuffer.allocate(0);
-
-    private final ContentProvider provider;
-    private final Iterator<ByteBuffer> iterator;
-    private ByteBuffer buffer;
-    private ByteBuffer content;
-    private boolean last;
-
-    public HttpContent(ContentProvider provider)
-    {
-        this.provider = provider;
-        this.iterator = provider == null ? Collections.<ByteBuffer>emptyIterator() : provider.iterator();
-    }
-
-    /**
-     * @return true if the buffer is the sentinel instance {@link CLOSE}
-     */
-    private static boolean isTheCloseBuffer(ByteBuffer buffer)
-    {
-        @SuppressWarnings("ReferenceEquality")
-        boolean isTheCloseBuffer = (buffer == CLOSE);
-        return isTheCloseBuffer;
-    }
-
-    /**
-     * @return whether there is any content at all
-     */
-    public boolean hasContent()
-    {
-        return provider != null;
-    }
-
-    /**
-     * @return whether the cursor points to the last content
-     */
-    public boolean isLast()
-    {
-        return last;
-    }
-
-    /**
-     * @return the {@link ByteBuffer} containing the content at the cursor's position
-     */
-    public ByteBuffer getByteBuffer()
-    {
-        return buffer;
-    }
-
-    /**
-     * @return a {@link ByteBuffer#slice()} of {@link #getByteBuffer()} at the cursor's position
-     */
-    public ByteBuffer getContent()
-    {
-        return content;
-    }
-
-    /**
-     * Advances the cursor to the next block of content.
-     * <p>
-     * The next block of content may be valid (which yields a non-null buffer
-     * returned by {@link #getByteBuffer()}), but may also be deferred
-     * (which yields a null buffer returned by {@link #getByteBuffer()}).
-     * <p>
-     * If the block of content pointed by the new cursor position is valid, this method returns true.
-     *
-     * @return true if there is content at the new cursor's position, false otherwise.
-     */
-    public boolean advance()
-    {
-        if (iterator instanceof Synchronizable)
-        {
-            synchronized (((Synchronizable)iterator).getLock())
-            {
-                return advance(iterator);
-            }
-        }
-        else
-        {
-            return advance(iterator);
-        }
-    }
-
-    private boolean advance(Iterator<ByteBuffer> iterator)
-    {
-        boolean hasNext = iterator.hasNext();
-        ByteBuffer bytes = hasNext ? iterator.next() : null;
-        boolean hasMore = hasNext && iterator.hasNext();
-        boolean wasLast = last;
-        last = !hasMore;
-
-        if (hasNext)
-        {
-            buffer = bytes;
-            content = bytes == null ? null : bytes.slice();
-            if (LOG.isDebugEnabled())
-                LOG.debug("Advanced content to {} chunk {}", hasMore ? "next" : "last", String.valueOf(bytes));
-            return bytes != null;
-        }
-        else
-        {
-            // No more content, but distinguish between last and consumed.
-            if (wasLast)
-            {
-                buffer = content = AFTER;
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Advanced content past last chunk");
-            }
-            else
-            {
-                buffer = content = CLOSE;
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Advanced content to last chunk");
-            }
-            return false;
-        }
-    }
-
-    /**
-     * @return whether the cursor has been advanced past the {@link #isLast() last} position.
-     */
-    @SuppressWarnings("ReferenceEquality")
-    public boolean isConsumed()
-    {
-        return buffer == AFTER;
-    }
-
-    @Override
-    public void succeeded()
-    {
-        if (isConsumed())
-            return;
-        if (isTheCloseBuffer(buffer))
-            return;
-        if (iterator instanceof Callback)
-            ((Callback)iterator).succeeded();
-    }
-
-    @Override
-    public void failed(Throwable x)
-    {
-        if (isConsumed())
-            return;
-        if (isTheCloseBuffer(buffer))
-            return;
-        if (iterator instanceof Callback)
-            ((Callback)iterator).failed(x);
-    }
-
-    @Override
-    public void close()
-    {
-        if (iterator instanceof Closeable)
-            IO.close((Closeable)iterator);
-    }
-
-    @Override
-    public String toString()
-    {
-        return String.format("%s@%x - has=%b,last=%b,consumed=%b,buffer=%s",
-            getClass().getSimpleName(),
-            hashCode(),
-            hasContent(),
-            isLast(),
-            isConsumed(),
-            BufferUtil.toDetailString(getContent()));
-    }
-}

jetty-client/src/main/java/org/eclipse/jetty/client/HttpExchange.java

@@ -20,6 +20,7 @@ package org.eclipse.jetty.client;
 
 import java.util.List;
 
+import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
 import org.slf4j.Logger;
@@ -237,6 +238,12 @@ public class HttpExchange
 
         // We failed this exchange, deal with it.
 
+        // Applications could be blocked providing
+        // request content, notify them of the failure.
+        Request.Content body = request.getBody();
+        if (abortRequest && body != null)
+            body.fail(failure);
+
         // Case #1: exchange was in the destination queue.
         if (destination.remove(this))
         {

jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java

@@ -528,7 +528,7 @@ public abstract class HttpReceiver
         HttpResponse response = exchange.getResponse();
 
         if (LOG.isDebugEnabled())
-            LOG.debug("Response complete {}", response);
+            LOG.debug("Response complete {}, result: {}", response, result);
 
         if (result != null)
         {

jetty-client/src/main/java/org/eclipse/jetty/client/HttpRequest.java

@@ -49,8 +49,9 @@ import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
+import org.eclipse.jetty.client.internal.RequestContentAdapter;
 import org.eclipse.jetty.client.util.FutureResponseListener;
-import org.eclipse.jetty.client.util.PathContentProvider;
+import org.eclipse.jetty.client.util.PathRequestContent;
 import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
@@ -81,7 +82,7 @@ public class HttpRequest implements Request
     private long idleTimeout = -1;
     private long timeout;
     private long timeoutAt;
-    private ContentProvider content;
+    private Content content;
     private boolean followRedirects;
     private List<HttpCookie> cookies;
     private Map<String, Object> attributes;
@@ -647,7 +648,9 @@ public class HttpRequest implements Request
     @Override
     public ContentProvider getContent()
     {
-        return content;
+        if (content instanceof RequestContentAdapter)
+            return ((RequestContentAdapter)content).getContentProvider();
+        return null;
     }
 
     @Override
@@ -661,6 +664,18 @@ public class HttpRequest implements Request
     {
         if (contentType != null)
             header(HttpHeader.CONTENT_TYPE, contentType);
+        return body(ContentProvider.toRequestContent(content));
+    }
+
+    @Override
+    public Content getBody()
+    {
+        return content;
+    }
+
+    @Override
+    public Request body(Content content)
+    {
         this.content = content;
         return this;
     }
@@ -674,7 +689,7 @@ public class HttpRequest implements Request
     @Override
     public Request file(Path file, String contentType) throws IOException
     {
-        return content(new PathContentProvider(contentType, file));
+        return body(new PathRequestContent(contentType, file));
     }
 
     @Override
@@ -809,11 +824,7 @@ public class HttpRequest implements Request
     public boolean abort(Throwable cause)
     {
         if (aborted.compareAndSet(null, Objects.requireNonNull(cause)))
-        {
-            if (content instanceof Callback)
-                ((Callback)content).failed(cause);
             return conversation.abort(cause);
-        }
         return false;
     }
 

jetty-client/src/main/java/org/eclipse/jetty/client/HttpSender.java

@@ -23,52 +23,38 @@ import java.util.concurrent.Executor;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
 
-import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpHeaderValue;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.Callback;
-import org.eclipse.jetty.util.IteratingCallback;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * {@link HttpSender} abstracts the algorithm to send HTTP requests, so that subclasses only implement
+ * <p>HttpSender abstracts the algorithm to send HTTP requests, so that subclasses only
- * the transport-specific code to send requests over the wire, implementing
+ * implement the transport-specific code to send requests over the wire, implementing
- * {@link #sendHeaders(HttpExchange, HttpContent, Callback)} and
+ * {@link #sendHeaders(HttpExchange, ByteBuffer, boolean, Callback)} and
- * {@link #sendContent(HttpExchange, HttpContent, Callback)}.
+ * {@link #sendContent(HttpExchange, ByteBuffer, boolean, Callback)}.</p>
- * <p>
+ * <p>HttpSender governs the request state machines, which is updated as the various
- * {@link HttpSender} governs two state machines.
+ * steps of sending a request are executed, see {@code RequestState}.
- * <p>
+ * At any point in time, a user thread may abort the request, which may (if the request
- * The request state machine is updated by {@link HttpSender} as the various steps of sending a request
+ * has not been completely sent yet) move the request state machine to {@code RequestState#FAILURE}.
- * are executed, see {@code RequestState}.
+ * The request state machine guarantees that the request steps are executed (by I/O threads)
- * At any point in time, a user thread may abort the request, which may (if the request has not been
+ * only if the request has not been failed already.</p>
- * completely sent yet) move the request state machine to {@code RequestState#FAILURE}.
- * The request state machine guarantees that the request steps are executed (by I/O threads) only if
- * the request has not been failed already.
- * <p>
- * The sender state machine is updated by {@link HttpSender} from three sources: deferred content notifications
- * (via {@link #onContent()}), 100-continue notifications (via {@link #proceed(HttpExchange, Throwable)})
- * and normal request send (via {@link #sendContent(HttpExchange, HttpContent, Callback)}).
- * This state machine must guarantee that the request sending is never executed concurrently: only one of
- * those sources may trigger the call to {@link #sendContent(HttpExchange, HttpContent, Callback)}.
  *
  * @see HttpReceiver
  */
-public abstract class HttpSender implements AsyncContentProvider.Listener
+public abstract class HttpSender
 {
     private static final Logger LOG = LoggerFactory.getLogger(HttpSender.class);
 
+    private final ContentConsumer consumer = new ContentConsumer();
     private final AtomicReference<RequestState> requestState = new AtomicReference<>(RequestState.QUEUED);
-    private final AtomicReference<SenderState> senderState = new AtomicReference<>(SenderState.IDLE);
+    private final AtomicReference<Throwable> failure = new AtomicReference<>();
-    private final Callback commitCallback = new CommitCallback();
-    private final IteratingCallback contentCallback = new ContentCallback();
-    private final Callback lastCallback = new LastCallback();
     private final HttpChannel channel;
-    private HttpContent content;
+    private Request.Content.Subscription subscription;
-    private Throwable failure;
 
     protected HttpSender(HttpChannel channel)
     {
@@ -90,126 +76,15 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         return requestState.get() == RequestState.FAILURE;
     }
 
-    @Override
-    public void onContent()
-    {
-        HttpExchange exchange = getHttpExchange();
-        if (exchange == null)
-            return;
-
-        while (true)
-        {
-            SenderState current = senderState.get();
-            switch (current)
-            {
-                case IDLE:
-                {
-                    SenderState newSenderState = SenderState.SENDING;
-                    if (updateSenderState(current, newSenderState))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Deferred content available, {} -> {}", current, newSenderState);
-                        contentCallback.iterate();
-                        return;
-                    }
-                    break;
-                }
-                case SENDING:
-                {
-                    SenderState newSenderState = SenderState.SENDING_WITH_CONTENT;
-                    if (updateSenderState(current, newSenderState))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Deferred content available, {} -> {}", current, newSenderState);
-                        return;
-                    }
-                    break;
-                }
-                case EXPECTING:
-                {
-                    SenderState newSenderState = SenderState.EXPECTING_WITH_CONTENT;
-                    if (updateSenderState(current, newSenderState))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Deferred content available, {} -> {}", current, newSenderState);
-                        return;
-                    }
-                    break;
-                }
-                case PROCEEDING:
-                {
-                    SenderState newSenderState = SenderState.PROCEEDING_WITH_CONTENT;
-                    if (updateSenderState(current, newSenderState))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Deferred content available, {} -> {}", current, newSenderState);
-                        return;
-                    }
-                    break;
-                }
-                case SENDING_WITH_CONTENT:
-                case EXPECTING_WITH_CONTENT:
-                case PROCEEDING_WITH_CONTENT:
-                case WAITING:
-                case COMPLETED:
-                case FAILED:
-                {
-                    if (LOG.isDebugEnabled())
-                        LOG.debug("Deferred content available, {}", current);
-                    return;
-                }
-                default:
-                {
-                    illegalSenderState(current);
-                    return;
-                }
-            }
-        }
-    }
-
     public void send(HttpExchange exchange)
     {
         if (!queuedToBegin(exchange))
             return;
 
-        Request request = exchange.getRequest();
-        ContentProvider contentProvider = request.getContent();
-        HttpContent content = this.content = new HttpContent(contentProvider);
-
-        SenderState newSenderState = SenderState.SENDING;
-        if (expects100Continue(request))
-            newSenderState = content.hasContent() ? SenderState.EXPECTING_WITH_CONTENT : SenderState.EXPECTING;
-
-        out:
-        while (true)
-        {
-            SenderState current = senderState.get();
-            switch (current)
-            {
-                case IDLE:
-                case COMPLETED:
-                {
-                    if (updateSenderState(current, newSenderState))
-                        break out;
-                    break;
-                }
-                default:
-                {
-                    illegalSenderState(current);
-                    return;
-                }
-            }
-        }
-
-        // Setting the listener may trigger calls to onContent() by other
-        // threads so we must set it only after the sender state has been updated
-        if (contentProvider instanceof AsyncContentProvider)
-            ((AsyncContentProvider)contentProvider).setListener(this);
-
         if (!beginToHeaders(exchange))
             return;
 
-        sendHeaders(exchange, content, commitCallback);
+        demand();
     }
 
     protected boolean expects100Continue(Request request)
@@ -228,10 +103,16 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         RequestNotifier notifier = getHttpChannel().getHttpDestination().getRequestNotifier();
         notifier.notifyBegin(request);
 
+        Request.Content body = request.getBody();
+
+        consumer.exchange = exchange;
+        consumer.expect100 = expects100Continue(request);
+        subscription = body.subscribe(consumer, !consumer.expect100);
+
         if (updateRequestState(RequestState.TRANSIENT, RequestState.BEGIN))
             return true;
 
-        terminateRequest(exchange);
+        abortRequest(exchange);
         return false;
     }
 
@@ -249,7 +130,7 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         if (updateRequestState(RequestState.TRANSIENT, RequestState.HEADERS))
             return true;
 
-        terminateRequest(exchange);
+        abortRequest(exchange);
         return false;
     }
 
@@ -267,7 +148,7 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         if (updateRequestState(RequestState.TRANSIENT, RequestState.COMMIT))
             return true;
 
-        terminateRequest(exchange);
+        abortRequest(exchange);
         return false;
     }
 
@@ -291,7 +172,7 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
                 if (updateRequestState(RequestState.TRANSIENT, RequestState.CONTENT))
                     return true;
 
-                terminateRequest(exchange);
+                abortRequest(exchange);
                 return false;
             }
             default:
@@ -353,6 +234,20 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
             executeAbort(exchange, failure);
     }
 
+    private void demand()
+    {
+        try
+        {
+            subscription.demand();
+        }
+        catch (Throwable x)
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Failure invoking demand()", x);
+            anyToFailure(x);
+        }
+    }
+
     private void executeAbort(HttpExchange exchange, Throwable failure)
     {
         try
@@ -368,13 +263,23 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         }
     }
 
-    private void terminateRequest(HttpExchange exchange)
+    private void abortRequest(HttpExchange exchange)
     {
-        // In abort(), the state is updated before the failure is recorded
+        Throwable failure = this.failure.get();
-        // to avoid to overwrite it, so here we may read a null failure.
+
-        Throwable failure = this.failure;
+        if (subscription != null)
-        if (failure == null)
+            subscription.fail(failure);
-            failure = new HttpRequestException("Concurrent failure", exchange.getRequest());
+
+        dispose();
+
+        Request request = exchange.getRequest();
+        if (LOG.isDebugEnabled())
+            LOG.debug("Request abort {} {} on {}: {}", request, exchange, getHttpChannel(), failure);
+        HttpDestination destination = getHttpChannel().getHttpDestination();
+        destination.getRequestNotifier().notifyFailure(request, failure);
+
+        // Mark atomically the request as terminated, with
+        // respect to concurrency between request and response.
         Result result = exchange.terminateRequest();
         terminateRequest(exchange, failure, result);
     }
@@ -415,163 +320,73 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
     }
 
     /**
-     * Implementations should send the HTTP headers over the wire, possibly with some content,
+     * <p>Implementations should send the HTTP headers over the wire, possibly with some content,
-     * in a single write, and notify the given {@code callback} of the result of this operation.
+     * in a single write, and notify the given {@code callback} of the result of this operation.</p>
-     * <p>
+     * <p>If there is more content to send, then {@link #sendContent(HttpExchange, ByteBuffer, boolean, Callback)}
-     * If there is more content to send, then {@link #sendContent(HttpExchange, HttpContent, Callback)}
+     * will be invoked.</p>
-     * will be invoked.
      *
-     * @param exchange the exchange to send
+     * @param exchange the exchange
-     * @param content the content to send
+     * @param contentBuffer the content to send
+     * @param lastContent whether the content is the last content to send
      * @param callback the callback to notify
      */
-    protected abstract void sendHeaders(HttpExchange exchange, HttpContent content, Callback callback);
+    protected abstract void sendHeaders(HttpExchange exchange, ByteBuffer contentBuffer, boolean lastContent, Callback callback);
 
     /**
-     * Implementations should send the content at the {@link HttpContent} cursor position over the wire.
+     * <p>Implementations should send the given HTTP content over the wire.</p>
-     * <p>
-     * The {@link HttpContent} cursor is advanced by HttpSender at the right time, and if more
-     * content needs to be sent, this method is invoked again; subclasses need only to send the content
-     * at the {@link HttpContent} cursor position.
-     * <p>
-     * This method is invoked one last time when {@link HttpContent#isConsumed()} is true and therefore
-     * there is no actual content to send.
-     * This is done to allow subclasses to write "terminal" bytes (such as the terminal chunk when the
-     * transfer encoding is chunked) if their protocol needs to.
      *
-     * @param exchange the exchange to send
+     * @param exchange the exchange
-     * @param content the content to send
+     * @param contentBuffer the content to send
+     * @param lastContent whether the content is the last content to send
      * @param callback the callback to notify
      */
-    protected abstract void sendContent(HttpExchange exchange, HttpContent content, Callback callback);
+    protected abstract void sendContent(HttpExchange exchange, ByteBuffer contentBuffer, boolean lastContent, Callback callback);
 
     protected void reset()
     {
-        HttpContent content = this.content;
+        consumer.reset();
-        this.content = null;
-        content.close();
-        senderState.set(SenderState.COMPLETED);
     }
 
     protected void dispose()
     {
-        HttpContent content = this.content;
-        this.content = null;
-        if (content != null)
-            content.close();
-        senderState.set(SenderState.FAILED);
     }
 
     public void proceed(HttpExchange exchange, Throwable failure)
     {
-        if (!expects100Continue(exchange.getRequest()))
+        consumer.expect100 = false;
-            return;
+        if (failure == null)
-
+            demand();
-        if (failure != null)
+        else
-        {
             anyToFailure(failure);
-            return;
-        }
-
-        while (true)
-        {
-            SenderState current = senderState.get();
-            switch (current)
-            {
-                case EXPECTING:
-                {
-                    // We are still sending the headers, but we already got the 100 Continue.
-                    if (updateSenderState(current, SenderState.PROCEEDING))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Proceeding while expecting");
-                        return;
-                    }
-                    break;
-                }
-                case EXPECTING_WITH_CONTENT:
-                {
-                    // More deferred content was submitted to onContent(), we already
-                    // got the 100 Continue, but we may be still sending the headers
-                    // (for example, with SSL we may have sent the encrypted data,
-                    // received the 100 Continue but not yet updated the decrypted
-                    // WriteFlusher so sending more content now may result in a
-                    // WritePendingException).
-                    if (updateSenderState(current, SenderState.PROCEEDING_WITH_CONTENT))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Proceeding while scheduled");
-                        return;
-                    }
-                    break;
-                }
-                case WAITING:
-                {
-                    // We received the 100 Continue, now send the content if any.
-                    if (updateSenderState(current, SenderState.SENDING))
-                    {
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Proceeding while waiting");
-                        contentCallback.iterate();
-                        return;
-                    }
-                    break;
-                }
-                case FAILED:
-                {
-                    return;
-                }
-                default:
-                {
-                    illegalSenderState(current);
-                    return;
-                }
-            }
-        }
     }
 
     public boolean abort(HttpExchange exchange, Throwable failure)
     {
+        // Store only the first failure.
+        this.failure.compareAndSet(null, failure);
+
         // Update the state to avoid more request processing.
-        boolean terminate;
+        boolean abort;
-        out:
         while (true)
         {
             RequestState current = requestState.get();
-            switch (current)
+            if (current == RequestState.FAILURE)
             {
-                case FAILURE:
+                return false;
+            }
+            else
+            {
+                if (updateRequestState(current, RequestState.FAILURE))
                 {
-                    return false;
+                    abort = current != RequestState.TRANSIENT;
-                }
-                default:
-                {
-                    if (updateRequestState(current, RequestState.FAILURE))
-                    {
-                        terminate = current != RequestState.TRANSIENT;
-                        break out;
-                    }
                     break;
                 }
             }
         }
 
-        this.failure = failure;
+        if (abort)
-
-        dispose();
-
-        Request request = exchange.getRequest();
-        if (LOG.isDebugEnabled())
-            LOG.debug("Request abort {} {} on {}: {}", request, exchange, getHttpChannel(), failure);
-        HttpDestination destination = getHttpChannel().getHttpDestination();
-        destination.getRequestNotifier().notifyFailure(request, failure);
-
-        if (terminate)
         {
-            // Mark atomically the request as terminated, with
+            abortRequest(exchange);
-            // respect to concurrency between request and response.
-            Result result = exchange.terminateRequest();
-            terminateRequest(exchange, failure, result);
             return true;
         }
         else
@@ -590,27 +405,13 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         return updated;
     }
 
-    private boolean updateSenderState(SenderState from, SenderState to)
-    {
-        boolean updated = senderState.compareAndSet(from, to);
-        if (!updated && LOG.isDebugEnabled())
-            LOG.debug("SenderState update failed: {} -> {}: {}", from, to, senderState.get());
-        return updated;
-    }
-
-    private void illegalSenderState(SenderState current)
-    {
-        anyToFailure(new IllegalStateException("Expected " + current + " found " + senderState.get() + " instead"));
-    }
-
     @Override
     public String toString()
     {
-        return String.format("%s@%x(req=%s,snd=%s,failure=%s)",
+        return String.format("%s@%x(req=%s,failure=%s)",
             getClass().getSimpleName(),
             hashCode(),
             requestState,
-            senderState,
             failure);
     }
 
@@ -649,286 +450,98 @@ public abstract class HttpSender implements AsyncContentProvider.Listener
         FAILURE
     }
 
-    /**
+    private class ContentConsumer implements Request.Content.Consumer, Callback
-     * The sender states {@link HttpSender} goes through when sending a request.
-     */
-    private enum SenderState
     {
-        /**
+        private HttpExchange exchange;
-         * {@link HttpSender} is not sending request headers nor request content
+        private boolean expect100;
-         */
+        private ByteBuffer contentBuffer;
-        IDLE,
+        private boolean lastContent;
-        /**
+        private Callback callback;
-         * {@link HttpSender} is sending the request header or request content
+        private boolean committed;
-         */
+
-        SENDING,
+        private void reset()
-        /**
+        {
-         * {@link HttpSender} is currently sending the request, and deferred content is available to be sent
+            exchange = null;
-         */
+            contentBuffer = null;
-        SENDING_WITH_CONTENT,
+            lastContent = false;
-        /**
+            callback = null;
-         * {@link HttpSender} is sending the headers but will wait for 100 Continue before sending the content
+            committed = false;
-         */
+        }
-        EXPECTING,
+
-        /**
+        @Override
-         * {@link HttpSender} is currently sending the headers, will wait for 100 Continue, and deferred content is available to be sent
+        public void onContent(ByteBuffer buffer, boolean last, Callback callback)
-         */
+        {
-        EXPECTING_WITH_CONTENT,
+            if (LOG.isDebugEnabled())
-        /**
+                LOG.debug("Content {} last={} for {}", BufferUtil.toDetailString(buffer), last, exchange.getRequest());
-         * {@link HttpSender} has sent the headers and is waiting for 100 Continue
+            this.contentBuffer = buffer.slice();
-         */
+            this.lastContent = last;
-        WAITING,
+            this.callback = callback;
-        /**
+            if (committed)
-         * {@link HttpSender} is sending the headers, while 100 Continue has arrived
+                sendContent(exchange, buffer, last, this);
-         */
+            else
-        PROCEEDING,
+                sendHeaders(exchange, buffer, last, this);
-        /**
+        }
-         * {@link HttpSender} is sending the headers, while 100 Continue has arrived, and deferred content is available to be sent
+
-         */
+        @Override
-        PROCEEDING_WITH_CONTENT,
+        public void onFailure(Throwable failure)
-        /**
+        {
-         * {@link HttpSender} has finished to send the request
+            failed(failure);
-         */
+        }
-        COMPLETED,
-        /**
-         * {@link HttpSender} has failed to send the request
-         */
-        FAILED
-    }
 
-    private class CommitCallback implements Callback
-    {
         @Override
         public void succeeded()
         {
-            try
+            boolean proceed = false;
+            if (committed)
             {
-                HttpContent content = HttpSender.this.content;
+                proceed = someToContent(exchange, contentBuffer);
-                if (content == null)
-                    return;
-                content.succeeded();
-                process();
-            }
-            catch (Throwable x)
-            {
-                anyToFailure(x);
-            }
-        }
-
-        @Override
-        public void failed(Throwable failure)
-        {
-            HttpContent content = HttpSender.this.content;
-            if (content == null)
-                return;
-            content.failed(failure);
-            anyToFailure(failure);
-        }
-
-        private void process() throws Exception
-        {
-            HttpExchange exchange = getHttpExchange();
-            if (exchange == null)
-                return;
-
-            if (!headersToCommit(exchange))
-                return;
-
-            HttpContent content = HttpSender.this.content;
-            if (content == null)
-                return;
-
-            if (!content.hasContent())
-            {
-                // No content to send, we are done.
-                someToSuccess(exchange);
             }
             else
             {
-                // Was any content sent while committing?
+                committed = true;
-                ByteBuffer contentBuffer = content.getContent();
+                if (headersToCommit(exchange))
-                if (contentBuffer != null)
                 {
-                    if (!someToContent(exchange, contentBuffer))
+                    proceed = true;
-                        return;
+                    // Was any content sent while committing?
-                }
+                    if (contentBuffer.hasRemaining())
-
+                        proceed = someToContent(exchange, contentBuffer);
-                while (true)
-                {
-                    SenderState current = senderState.get();
-                    switch (current)
-                    {
-                        case SENDING:
-                        {
-                            contentCallback.iterate();
-                            return;
-                        }
-                        case SENDING_WITH_CONTENT:
-                        {
-                            // We have deferred content to send.
-                            updateSenderState(current, SenderState.SENDING);
-                            break;
-                        }
-                        case EXPECTING:
-                        {
-                            // We sent the headers, wait for the 100 Continue response.
-                            if (updateSenderState(current, SenderState.WAITING))
-                                return;
-                            break;
-                        }
-                        case EXPECTING_WITH_CONTENT:
-                        {
-                            // We sent the headers, we have deferred content to send,
-                            // wait for the 100 Continue response.
-                            if (updateSenderState(current, SenderState.WAITING))
-                                return;
-                            break;
-                        }
-                        case PROCEEDING:
-                        {
-                            // We sent the headers, we have the 100 Continue response,
-                            // we have no content to send.
-                            if (updateSenderState(current, SenderState.IDLE))
-                                return;
-                            break;
-                        }
-                        case PROCEEDING_WITH_CONTENT:
-                        {
-                            // We sent the headers, we have the 100 Continue response,
-                            // we have deferred content to send.
-                            updateSenderState(current, SenderState.SENDING);
-                            break;
-                        }
-                        case FAILED:
-                        {
-                            return;
-                        }
-                        default:
-                        {
-                            illegalSenderState(current);
-                            return;
-                        }
-                    }
                 }
             }
-        }
-    }
 
-    private class ContentCallback extends IteratingCallback
+            // Succeed the content callback only after emitting the request content event.
-    {
+            callback.succeeded();
-        @Override
-        protected Action process() throws Exception
-        {
-            HttpExchange exchange = getHttpExchange();
-            if (exchange == null)
-                return Action.IDLE;
 
-            HttpContent content = HttpSender.this.content;
+            // There was some concurrent error?
-            if (content == null)
+            if (!proceed)
-                return Action.IDLE;
+                return;
 
-            while (true)
+            if (lastContent)
+            {
+                someToSuccess(exchange);
+            }
+            else if (expect100)
             {
-                boolean advanced = content.advance();
-                boolean lastContent = content.isLast();
                 if (LOG.isDebugEnabled())
-                    LOG.debug("Content present {}, last {}, consumed {} for {}", advanced, lastContent, content.isConsumed(), exchange.getRequest());
+                    LOG.debug("Expecting 100 Continue for {}", exchange.getRequest());
-
+            }
-                if (advanced)
+            else
-                {
+            {
-                    sendContent(exchange, content, this);
+                demand();
-                    return Action.SCHEDULED;
-                }
-
-                if (lastContent)
-                {
-                    sendContent(exchange, content, lastCallback);
-                    return Action.IDLE;
-                }
-
-                SenderState current = senderState.get();
-                switch (current)
-                {
-                    case SENDING:
-                    {
-                        if (updateSenderState(current, SenderState.IDLE))
-                        {
-                            if (LOG.isDebugEnabled())
-                                LOG.debug("Content is deferred for {}", exchange.getRequest());
-                            return Action.IDLE;
-                        }
-                        break;
-                    }
-                    case SENDING_WITH_CONTENT:
-                    {
-                        updateSenderState(current, SenderState.SENDING);
-                        break;
-                    }
-                    default:
-                    {
-                        illegalSenderState(current);
-                        return Action.IDLE;
-                    }
-                }
             }
         }
 
         @Override
-        public void succeeded()
+        public void failed(Throwable x)
         {
-            HttpExchange exchange = getHttpExchange();
+            if (callback != null)
-            if (exchange == null)
+                callback.failed(x);
-                return;
+            anyToFailure(x);
-            HttpContent content = HttpSender.this.content;
-            if (content == null)
-                return;
-            content.succeeded();
-            ByteBuffer buffer = content.getContent();
-            someToContent(exchange, buffer);
-            super.succeeded();
         }
 
         @Override
-        public void onCompleteFailure(Throwable failure)
+        public InvocationType getInvocationType()
         {
-            HttpContent content = HttpSender.this.content;
+            return InvocationType.NON_BLOCKING;
-            if (content == null)
-                return;
-            content.failed(failure);
-            anyToFailure(failure);
-        }
-
-        @Override
-        protected void onCompleteSuccess()
-        {
-            // Nothing to do, since we always return IDLE from process().
-            // Termination is obtained via LastCallback.
-        }
-    }
-
-    private class LastCallback implements Callback
-    {
-        @Override
-        public void succeeded()
-        {
-            HttpExchange exchange = getHttpExchange();
-            if (exchange == null)
-                return;
-            HttpContent content = HttpSender.this.content;
-            if (content == null)
-                return;
-            content.succeeded();
-            someToSuccess(exchange);
-        }
-
-        @Override
-        public void failed(Throwable failure)
-        {
-            HttpContent content = HttpSender.this.content;
-            if (content == null)
-                return;
-            content.failed(failure);
-            anyToFailure(failure);
         }
     }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/RequestNotifier.java

@@ -158,10 +158,10 @@ public class RequestNotifier
 
     public void notifyContent(Request request, ByteBuffer content)
     {
-        // Slice the buffer to avoid that listeners peek into data they should not look at.
-        content = content.slice();
         if (!content.hasRemaining())
             return;
+        // Slice the buffer to avoid that listeners peek into data they should not look at.
+        content = content.slice();
         // Optimized to avoid allocations of iterator instances.
         List<Request.RequestListener> requestListeners = request.getRequestListeners(null);
         for (int i = 0; i < requestListeners.size(); ++i)

jetty-client/src/main/java/org/eclipse/jetty/client/api/ContentProvider.java

@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
 import java.util.Iterator;
 
 import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.internal.RequestContentAdapter;
 import org.eclipse.jetty.client.util.ByteBufferContentProvider;
 import org.eclipse.jetty.client.util.PathContentProvider;
 
@@ -41,9 +42,24 @@ import org.eclipse.jetty.client.util.PathContentProvider;
  * header set by applications; if the length is negative, it typically removes
  * any {@code Content-Length} header set by applications, resulting in chunked
  * content (i.e. {@code Transfer-Encoding: chunked}) being sent to the server.</p>
+ *
+ * @deprecated use {@link Request.Content} instead, or {@link #toRequestContent(ContentProvider)}
+ * to convert ContentProvider to {@link Request.Content}.
  */
+@Deprecated
 public interface ContentProvider extends Iterable<ByteBuffer>
 {
+    /**
+     * <p>Converts a ContentProvider to a {@link Request.Content}.</p>
+     *
+     * @param provider the ContentProvider to convert
+     * @return a {@link Request.Content} that wraps the ContentProvider
+     */
+    public static Request.Content toRequestContent(ContentProvider provider)
+    {
+        return new RequestContentAdapter(provider);
+    }
+
     /**
      * @return the content length, if known, or -1 if the content length is unknown
      */
@@ -68,7 +84,10 @@ public interface ContentProvider extends Iterable<ByteBuffer>
     /**
      * An extension of {@link ContentProvider} that provides a content type string
      * to be used as a {@code Content-Type} HTTP header in requests.
+     *
+     * @deprecated use {@link Request.Content} instead
      */
+    @Deprecated
     public interface Typed extends ContentProvider
     {
         /**

jetty-client/src/main/java/org/eclipse/jetty/client/api/Request.java

@@ -19,6 +19,7 @@
 package org.eclipse.jetty.client.api;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.HttpCookie;
 import java.net.URI;
 import java.net.URLEncoder;
@@ -37,6 +38,7 @@ import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.Fields;
 
 /**
@@ -216,22 +218,39 @@ public interface Request
 
     /**
      * @return the content provider of this request
+     * @deprecated use {@link #getBody()} instead
      */
+    @Deprecated
     ContentProvider getContent();
 
     /**
      * @param content the content provider of this request
      * @return this request object
+     * @deprecated use {@link #body(Content)} instead
      */
+    @Deprecated
     Request content(ContentProvider content);
 
     /**
      * @param content the content provider of this request
      * @param contentType the content type
      * @return this request object
+     * @deprecated use {@link #body(Content)} instead
      */
+    @Deprecated
     Request content(ContentProvider content, String contentType);
 
+    /**
+     * @return the request content of this request
+     */
+    Content getBody();
+
+    /**
+     * @param content the request content of this request
+     * @return this request object
+     */
+    Request body(Content content);
+
     /**
      * Shortcut method to specify a file as a content for this request, with the default content type of
      * "application/octect-stream".
@@ -615,4 +634,158 @@ public interface Request
         {
         }
     }
+
+    /**
+     * <p>A reactive model to produce request content, similar to {@link java.util.concurrent.Flow.Publisher}.</p>
+     * <p>Implementations receive the content consumer via {@link #subscribe(Consumer, boolean)},
+     * and return a {@link Subscription} as the link between producer and consumer.</p>
+     * <p>Content producers must notify content to the consumer only if there is demand.</p>
+     * <p>Content consumers can generate demand for content by invoking {@link Subscription#demand()}.</p>
+     * <p>Content production must follow this algorithm:</p>
+     * <ul>
+     *   <li>the first time content is demanded
+     *   <ul>
+     *     <li>when the content is not available =&gt; produce an empty content</li>
+     *     <li>when the content is available:
+     *       <ul>
+     *         <li>when {@code emitInitialContent == false} =&gt; produce an empty content</li>
+     *         <li>when {@code emitInitialContent == true} =&gt; produce the content</li>
+     *       </ul>
+     *     </li>
+     *   </ul>
+     *   </li>
+     *   <li>the second and subsequent times content is demanded
+     *     <ul>
+     *       <li>when the content is not available =&gt; do not produce content</li>
+     *       <li>when the content is available =&gt; produce the content</li>
+     *     </ul>
+     *   </li>
+     * </ul>
+     *
+     * @see #subscribe(Consumer, boolean)
+     */
+    public interface Content
+    {
+        /**
+         * @return the content type string such as "application/octet-stream" or
+         * "application/json;charset=UTF8", or null if no content type must be set
+         */
+        public default String getContentType()
+        {
+            return "application/octet-stream";
+        }
+
+        /**
+         * @return the content length, if known, or -1 if the content length is unknown
+         */
+        public default long getLength()
+        {
+            return -1;
+        }
+
+        /**
+         * <p>Whether this content producer can produce exactly the same content more
+         * than once.</p>
+         * <p>Implementations should return {@code true} only if the content can be
+         * produced more than once, which means that {@link #subscribe(Consumer, boolean)}
+         * may be called again.</p>
+         * <p>The {@link HttpClient} implementation may use this method in particular
+         * cases where it detects that it is safe to retry a request that failed.</p>
+         *
+         * @return whether the content can be produced more than once
+         */
+        public default boolean isReproducible()
+        {
+            return false;
+        }
+
+        /**
+         * <p>Initializes this content producer with the content consumer, and with
+         * the indication of whether initial content, if present, must be emitted
+         * upon the initial demand of content (to support delaying the send of the
+         * request content in case of {@code Expect: 100-Continue} when
+         * {@code emitInitialContent} is {@code false}).</p>
+         *
+         * @param consumer the content consumer to invoke when there is demand for content
+         * @param emitInitialContent whether to emit initial content, if present
+         * @return the Subscription that links this producer to the consumer
+         */
+        public Subscription subscribe(Consumer consumer, boolean emitInitialContent);
+
+        /**
+         * <p>Fails this request content, possibly failing and discarding accumulated
+         * content that was not demanded.</p>
+         * <p>The failure may be notified to the consumer at a later time, when the
+         * consumer demands for content.</p>
+         * <p>Typical failure: the request being aborted by user code, or idle timeouts.</p>
+         *
+         * @param failure the reason of the failure
+         */
+        public default void fail(Throwable failure)
+        {
+        }
+
+        /**
+         * <p>A reactive model to consume request content, similar to {@link java.util.concurrent.Flow.Subscriber}.</p>
+         * <p>Callback methods {@link #onContent(ByteBuffer, boolean, Callback)} and {@link #onFailure(Throwable)}
+         * are invoked in strict sequential order and never concurrently, although possibly by different threads.</p>
+         */
+        public interface Consumer
+        {
+            /**
+             * <p>Callback method invoked by the producer when there is content available
+             * <em>and</em> there is demand for content.</p>
+             * <p>The {@code callback} is associated with the {@code buffer} to
+             * signal when the content buffer has been consumed.</p>
+             * <p>Failing the {@code callback} does not have any effect on content
+             * production. To stop the content production, the consumer must call
+             * {@link Subscription#fail(Throwable)}.</p>
+             * <p>In case an exception is thrown by this method, it is equivalent to
+             * a call to {@link Subscription#fail(Throwable)}.</p>
+             *
+             * @param buffer the content buffer to consume
+             * @param last whether it's the last content
+             * @param callback a callback to invoke when the content buffer is consumed
+             */
+            public void onContent(ByteBuffer buffer, boolean last, Callback callback);
+
+            /**
+             * <p>Callback method invoked by the producer when it failed to produce content.</p>
+             * <p>Typical failure: a producer getting an exception while reading from an
+             * {@link InputStream} to produce content.</p>
+             *
+             * @param failure the reason of the failure
+             */
+            public default void onFailure(Throwable failure)
+            {
+            }
+        }
+
+        /**
+         * <p>The link between a content producer and a content consumer.</p>
+         * <p>Content consumers can demand more content via {@link #demand()},
+         * or ask the content producer to stop producing content via
+         * {@link #fail(Throwable)}.</p>
+         */
+        public interface Subscription
+        {
+            /**
+             * <p>Demands more content, which eventually results in
+             * {@link Consumer#onContent(ByteBuffer, boolean, Callback)} to be invoked.</p>
+             */
+            public void demand();
+
+            /**
+             * <p>Fails the subscription, notifying the content producer to stop producing
+             * content.</p>
+             * <p>Typical failure: a proxy consumer waiting for more content (or waiting
+             * to demand content) that is failed by an error response from the server.</p>
+             *
+             * @param failure the reason of the failure
+             */
+            public default void fail(Throwable failure)
+            {
+            }
+        }
+    }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/dynamic/HttpClientTransportDynamic.java

@@ -19,12 +19,14 @@
 package org.eclipse.jetty.client.dynamic;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.eclipse.jetty.alpn.client.ALPNClientConnection;
 import org.eclipse.jetty.alpn.client.ALPNClientConnectionFactory;
@@ -37,6 +39,7 @@ import org.eclipse.jetty.client.MultiplexConnectionPool;
 import org.eclipse.jetty.client.MultiplexHttpDestination;
 import org.eclipse.jetty.client.Origin;
 import org.eclipse.jetty.client.http.HttpClientConnectionFactory;
+import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.io.ClientConnectionFactory;
 import org.eclipse.jetty.io.ClientConnector;
@@ -105,7 +108,7 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
             factoryInfos = new Info[]{HttpClientConnectionFactory.HTTP11};
         this.factoryInfos = Arrays.asList(factoryInfos);
         this.protocols = Arrays.stream(factoryInfos)
-                .flatMap(info -> info.getProtocols().stream())
+                .flatMap(info -> Stream.concat(info.getProtocols(false).stream(), info.getProtocols(true).stream()))
                 .distinct()
                 .map(p -> p.toLowerCase(Locale.ENGLISH))
                 .collect(Collectors.toList());
@@ -117,9 +120,9 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
     @Override
     public Origin newOrigin(HttpRequest request)
     {
-        boolean ssl = HttpClient.isSchemeSecure(request.getScheme());
+        boolean secure = HttpClient.isSchemeSecure(request.getScheme());
         String http1 = "http/1.1";
-        String http2 = ssl ? "h2" : "h2c";
+        String http2 = secure ? "h2" : "h2c";
         List<String> protocols = List.of();
         if (request.isVersionExplicit())
         {
@@ -130,16 +133,23 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
         }
         else
         {
-            if (ssl)
+            if (secure)
             {
                 // There may be protocol negotiation, so preserve the order
                 // of protocols chosen by the application.
                 // We need to keep multiple protocols in case the protocol
                 // is negotiated: e.g. [http/1.1, h2] negotiates [h2], but
                 // here we don't know yet what will be negotiated.
+                List<String> http = List.of("http/1.1", "h2c", "h2");
                 protocols = this.protocols.stream()
-                    .filter(p -> p.equals(http1) || p.equals(http2))
+                    .filter(http::contains)
-                    .collect(Collectors.toList());
+                    .collect(Collectors.toCollection(ArrayList::new));
+
+                // The http/1.1 upgrade to http/2 over TLS implicitly
+                // "negotiates" [h2c], so we need to remove [h2]
+                // because we don't want to negotiate using ALPN.
+                if (request.getHeaders().contains(HttpHeader.UPGRADE, "h2c"))
+                    protocols.remove("h2");
             }
             else
             {
@@ -149,7 +159,7 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
         }
         Origin.Protocol protocol = null;
         if (!protocols.isEmpty())
-            protocol = new Origin.Protocol(protocols, ssl && protocols.contains(http2));
+            protocol = new Origin.Protocol(protocols, secure && protocols.contains(http2));
         return getHttpClient().createOrigin(request, protocol);
     }
 
@@ -164,32 +174,33 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
     {
         HttpDestination destination = (HttpDestination)context.get(HTTP_DESTINATION_CONTEXT_KEY);
         Origin.Protocol protocol = destination.getOrigin().getProtocol();
-        ClientConnectionFactory.Info factoryInfo;
+        ClientConnectionFactory factory;
         if (protocol == null)
         {
             // Use the default ClientConnectionFactory.
-            factoryInfo = factoryInfos.get(0);
+            factory = factoryInfos.get(0).getClientConnectionFactory();
         }
         else
         {
             if (destination.isSecure() && protocol.isNegotiate())
             {
-                factoryInfo = new ALPNClientConnectionFactory.ALPN(getClientConnector().getExecutor(), this::newNegotiatedConnection, protocol.getProtocols());
+                factory = new ALPNClientConnectionFactory(getClientConnector().getExecutor(), this::newNegotiatedConnection, protocol.getProtocols());
             }
             else
             {
-                factoryInfo = findClientConnectionFactoryInfo(protocol.getProtocols())
+                factory = findClientConnectionFactoryInfo(protocol.getProtocols(), destination.isSecure())
-                        .orElseThrow(() -> new IOException("Cannot find " + ClientConnectionFactory.class.getSimpleName() + " for " + protocol));
+                    .orElseThrow(() -> new IOException("Cannot find " + ClientConnectionFactory.class.getSimpleName() + " for " + protocol))
+                    .getClientConnectionFactory();
             }
         }
-        return factoryInfo.getClientConnectionFactory().newConnection(endPoint, context);
+        return factory.newConnection(endPoint, context);
     }
 
     public void upgrade(EndPoint endPoint, Map<String, Object> context)
     {
         HttpDestination destination = (HttpDestination)context.get(HTTP_DESTINATION_CONTEXT_KEY);
         Origin.Protocol protocol = destination.getOrigin().getProtocol();
-        Info info = findClientConnectionFactoryInfo(protocol.getProtocols())
+        Info info = findClientConnectionFactoryInfo(protocol.getProtocols(), destination.isSecure())
             .orElseThrow(() -> new IllegalStateException("Cannot find " + ClientConnectionFactory.class.getSimpleName() + " to upgrade to " + protocol));
         info.upgrade(endPoint, context);
     }
@@ -200,13 +211,22 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
         {
             ALPNClientConnection alpnConnection = (ALPNClientConnection)endPoint.getConnection();
             String protocol = alpnConnection.getProtocol();
-            if (LOG.isDebugEnabled())
+            Info factoryInfo;
-                LOG.debug("ALPN negotiated {} among {}", protocol, alpnConnection.getProtocols());
+            if (protocol != null)
-            if (protocol == null)
+            {
-                throw new IOException("Could not negotiate protocol among " + alpnConnection.getProtocols());
+                if (LOG.isDebugEnabled())
-            List<String> protocols = List.of(protocol);
+                    LOG.debug("ALPN negotiated {} among {}", protocol, alpnConnection.getProtocols());
-            Info factoryInfo = findClientConnectionFactoryInfo(protocols)
+                List<String> protocols = List.of(protocol);
+                factoryInfo = findClientConnectionFactoryInfo(protocols, true)
                     .orElseThrow(() -> new IOException("Cannot find " + ClientConnectionFactory.class.getSimpleName() + " for negotiated protocol " + protocol));
+            }
+            else
+            {
+                // Server does not support ALPN, let's try the first protocol.
+                factoryInfo = factoryInfos.get(0);
+                if (LOG.isDebugEnabled())
+                    LOG.debug("No ALPN protocol, using {}", factoryInfo);
+            }
             return factoryInfo.getClientConnectionFactory().newConnection(endPoint, context);
         }
         catch (Throwable failure)
@@ -216,10 +236,10 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
         }
     }
 
-    private Optional<Info> findClientConnectionFactoryInfo(List<String> protocols)
+    private Optional<Info> findClientConnectionFactoryInfo(List<String> protocols, boolean secure)
     {
         return factoryInfos.stream()
-                .filter(info -> info.matches(protocols))
+                .filter(info -> info.matches(protocols, secure))
                 .findFirst();
     }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpClientConnectionFactory.java

@@ -21,12 +21,16 @@ package org.eclipse.jetty.client.http;
 import java.util.List;
 import java.util.Map;
 
+import org.eclipse.jetty.client.dynamic.HttpClientTransportDynamic;
 import org.eclipse.jetty.io.ClientConnectionFactory;
 import org.eclipse.jetty.io.EndPoint;
 
 public class HttpClientConnectionFactory implements ClientConnectionFactory
 {
-    public static final Info HTTP11 = new Info(List.of("http/1.1"), new HttpClientConnectionFactory());
+    /**
+     * <p>Representation of the {@code HTTP/1.1} application protocol used by {@link HttpClientTransportDynamic}.</p>
+     */
+    public static final Info HTTP11 = new HTTP11(new HttpClientConnectionFactory());
 
     @Override
     public org.eclipse.jetty.io.Connection newConnection(EndPoint endPoint, Map<String, Object> context)
@@ -34,4 +38,26 @@ public class HttpClientConnectionFactory implements ClientConnectionFactory
         HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, context);
         return customize(connection, context);
     }
+
+    private static class HTTP11 extends Info
+    {
+        private static final List<String> protocols = List.of("http/1.1");
+
+        private HTTP11(ClientConnectionFactory factory)
+        {
+            super(factory);
+        }
+
+        @Override
+        public List<String> getProtocols(boolean secure)
+        {
+            return protocols;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s@%x%s", getClass().getSimpleName(), hashCode(), protocols);
+        }
+    }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpSenderOverHTTP.java

@@ -21,12 +21,11 @@ package org.eclipse.jetty.client.http;
 import java.nio.ByteBuffer;
 
 import org.eclipse.jetty.client.HttpClient;
-import org.eclipse.jetty.client.HttpContent;
 import org.eclipse.jetty.client.HttpExchange;
 import org.eclipse.jetty.client.HttpRequest;
 import org.eclipse.jetty.client.HttpRequestException;
 import org.eclipse.jetty.client.HttpSender;
-import org.eclipse.jetty.client.api.ContentProvider;
+import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.http.HttpGenerator;
 import org.eclipse.jetty.http.HttpURI;
 import org.eclipse.jetty.http.MetaData;
@@ -42,7 +41,14 @@ public class HttpSenderOverHTTP extends HttpSender
 {
     private static final Logger LOG = LoggerFactory.getLogger(HttpSenderOverHTTP.class);
 
+    private final IteratingCallback headersCallback = new HeadersCallback();
+    private final IteratingCallback contentCallback = new ContentCallback();
     private final HttpGenerator generator = new HttpGenerator();
+    private HttpExchange exchange;
+    private MetaData.Request metaData;
+    private ByteBuffer contentBuffer;
+    private boolean lastContent;
+    private Callback callback;
     private boolean shutdown;
 
     public HttpSenderOverHTTP(HttpChannelOverHTTP channel)
@@ -57,11 +63,26 @@ public class HttpSenderOverHTTP extends HttpSender
     }
 
     @Override
-    protected void sendHeaders(HttpExchange exchange, HttpContent content, Callback callback)
+    protected void sendHeaders(HttpExchange exchange, ByteBuffer contentBuffer, boolean lastContent, Callback callback)
     {
         try
         {
-            new HeadersCallback(exchange, content, callback).iterate();
+            this.exchange = exchange;
+            this.contentBuffer = contentBuffer;
+            this.lastContent = lastContent;
+            this.callback = callback;
+            HttpRequest request = exchange.getRequest();
+            Request.Content requestContent = request.getBody();
+            long contentLength = requestContent == null ? -1 : requestContent.getLength();
+            String path = request.getPath();
+            String query = request.getQuery();
+            if (query != null)
+                path += "?" + query;
+            metaData = new MetaData.Request(request.getMethod(), new HttpURI(path), request.getVersion(), request.getHeaders(), contentLength);
+            metaData.setTrailerSupplier(request.getTrailers());
+            if (LOG.isDebugEnabled())
+                LOG.debug("Sending headers with content {} last={} for {}", BufferUtil.toDetailString(contentBuffer), lastContent, exchange.getRequest());
+            headersCallback.iterate();
         }
         catch (Throwable x)
         {
@@ -72,67 +93,17 @@ public class HttpSenderOverHTTP extends HttpSender
     }
 
     @Override
-    protected void sendContent(HttpExchange exchange, HttpContent content, Callback callback)
+    protected void sendContent(HttpExchange exchange, ByteBuffer contentBuffer, boolean lastContent, Callback callback)
     {
         try
         {
-            HttpClient httpClient = getHttpChannel().getHttpDestination().getHttpClient();
+            this.exchange = exchange;
-            ByteBufferPool bufferPool = httpClient.getByteBufferPool();
+            this.contentBuffer = contentBuffer;
-            boolean useDirectByteBuffers = httpClient.isUseOutputDirectByteBuffers();
+            this.lastContent = lastContent;
-            ByteBuffer chunk = null;
+            this.callback = callback;
-            while (true)
+            if (LOG.isDebugEnabled())
-            {
+                LOG.debug("Sending content {} last={} for {}", BufferUtil.toDetailString(contentBuffer), lastContent, exchange.getRequest());
-                ByteBuffer contentBuffer = content.getByteBuffer();
+            contentCallback.iterate();
-                boolean lastContent = content.isLast();
-                HttpGenerator.Result result = generator.generateRequest(null, null, chunk, contentBuffer, lastContent);
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Generated content ({} bytes) - {}/{}",
-                        contentBuffer == null ? -1 : contentBuffer.remaining(),
-                        result, generator);
-                switch (result)
-                {
-                    case NEED_CHUNK:
-                    {
-                        chunk = bufferPool.acquire(HttpGenerator.CHUNK_SIZE, useDirectByteBuffers);
-                        break;
-                    }
-                    case NEED_CHUNK_TRAILER:
-                    {
-                        chunk = bufferPool.acquire(httpClient.getRequestBufferSize(), useDirectByteBuffers);
-                        break;
-                    }
-                    case FLUSH:
-                    {
-                        EndPoint endPoint = getHttpChannel().getHttpConnection().getEndPoint();
-                        if (chunk != null)
-                            endPoint.write(new ByteBufferRecyclerCallback(callback, bufferPool, chunk), chunk, contentBuffer);
-                        else
-                            endPoint.write(callback, contentBuffer);
-                        return;
-                    }
-                    case SHUTDOWN_OUT:
-                    {
-                        shutdownOutput();
-                        break;
-                    }
-                    case CONTINUE:
-                    {
-                        if (lastContent)
-                            break;
-                        callback.succeeded();
-                        return;
-                    }
-                    case DONE:
-                    {
-                        callback.succeeded();
-                        return;
-                    }
-                    default:
-                    {
-                        throw new IllegalStateException(result.toString());
-                    }
-                }
-            }
         }
         catch (Throwable x)
         {
@@ -145,6 +116,8 @@ public class HttpSenderOverHTTP extends HttpSender
     @Override
     protected void reset()
     {
+        headersCallback.reset();
+        contentCallback.reset();
         generator.reset();
         super.reset();
     }
@@ -177,54 +150,30 @@ public class HttpSenderOverHTTP extends HttpSender
 
     private class HeadersCallback extends IteratingCallback
     {
-        private final HttpExchange exchange;
-        private final Callback callback;
-        private final MetaData.Request metaData;
         private ByteBuffer headerBuffer;
         private ByteBuffer chunkBuffer;
-        private ByteBuffer contentBuffer;
-        private boolean lastContent;
         private boolean generated;
 
-        public HeadersCallback(HttpExchange exchange, HttpContent content, Callback callback)
+        private HeadersCallback()
         {
             super(false);
-            this.exchange = exchange;
-            this.callback = callback;
-
-            HttpRequest request = exchange.getRequest();
-            ContentProvider requestContent = request.getContent();
-            long contentLength = requestContent == null ? -1 : requestContent.getLength();
-            String path = request.getPath();
-            String query = request.getQuery();
-            if (query != null)
-                path += "?" + query;
-            metaData = new MetaData.Request(request.getMethod(), new HttpURI(path), request.getVersion(), request.getHeaders(), contentLength);
-            metaData.setTrailerSupplier(request.getTrailers());
-
-            if (!expects100Continue(request))
-            {
-                content.advance();
-                contentBuffer = content.getByteBuffer();
-                lastContent = content.isLast();
-            }
         }
 
         @Override
         protected Action process() throws Exception
         {
+            HttpClient httpClient = getHttpChannel().getHttpDestination().getHttpClient();
+            ByteBufferPool byteBufferPool = httpClient.getByteBufferPool();
+            boolean useDirectByteBuffers = httpClient.isUseOutputDirectByteBuffers();
             while (true)
             {
                 HttpGenerator.Result result = generator.generateRequest(metaData, headerBuffer, chunkBuffer, contentBuffer, lastContent);
                 if (LOG.isDebugEnabled())
-                    LOG.debug("Generated headers ({} bytes), chunk ({} bytes), content ({} bytes) - {}/{}",
+                    LOG.debug("Generated headers ({} bytes), chunk ({} bytes), content ({} bytes) - {}/{} for {}",
                         headerBuffer == null ? -1 : headerBuffer.remaining(),
                         chunkBuffer == null ? -1 : chunkBuffer.remaining(),
                         contentBuffer == null ? -1 : contentBuffer.remaining(),
-                        result, generator);
+                        result, generator, exchange.getRequest());
-                HttpClient httpClient = getHttpChannel().getHttpDestination().getHttpClient();
-                ByteBufferPool byteBufferPool = httpClient.getByteBufferPool();
-                boolean useDirectByteBuffers = httpClient.isUseOutputDirectByteBuffers();
                 switch (result)
                 {
                     case NEED_HEADER:
@@ -332,37 +281,86 @@ public class HttpSenderOverHTTP extends HttpSender
         }
     }
 
-    private class ByteBufferRecyclerCallback extends Callback.Nested
+    private class ContentCallback extends IteratingCallback
     {
-        private final ByteBufferPool pool;
+        private ByteBuffer chunkBuffer;
-        private final ByteBuffer[] buffers;
 
-        private ByteBufferRecyclerCallback(Callback callback, ByteBufferPool pool, ByteBuffer... buffers)
+        public ContentCallback()
         {
-            super(callback);
+            super(false);
-            this.pool = pool;
-            this.buffers = buffers;
         }
 
         @Override
-        public void succeeded()
+        protected Action process() throws Exception
         {
-            for (ByteBuffer buffer : buffers)
+            HttpClient httpClient = getHttpChannel().getHttpDestination().getHttpClient();
+            ByteBufferPool bufferPool = httpClient.getByteBufferPool();
+            boolean useDirectByteBuffers = httpClient.isUseOutputDirectByteBuffers();
+            while (true)
             {
-                assert !buffer.hasRemaining();
+                HttpGenerator.Result result = generator.generateRequest(null, null, chunkBuffer, contentBuffer, lastContent);
-                pool.release(buffer);
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Generated content ({} bytes, last={}) - {}/{}",
+                        contentBuffer == null ? -1 : contentBuffer.remaining(),
+                        lastContent, result, generator);
+                switch (result)
+                {
+                    case NEED_CHUNK:
+                    {
+                        chunkBuffer = bufferPool.acquire(HttpGenerator.CHUNK_SIZE, useDirectByteBuffers);
+                        break;
+                    }
+                    case NEED_CHUNK_TRAILER:
+                    {
+                        chunkBuffer = bufferPool.acquire(httpClient.getRequestBufferSize(), useDirectByteBuffers);
+                        break;
+                    }
+                    case FLUSH:
+                    {
+                        EndPoint endPoint = getHttpChannel().getHttpConnection().getEndPoint();
+                        if (chunkBuffer != null)
+                            endPoint.write(this, chunkBuffer, contentBuffer);
+                        else
+                            endPoint.write(this, contentBuffer);
+                        return Action.SCHEDULED;
+                    }
+                    case SHUTDOWN_OUT:
+                    {
+                        shutdownOutput();
+                        break;
+                    }
+                    case CONTINUE:
+                    {
+                        break;
+                    }
+                    case DONE:
+                    {
+                        release();
+                        callback.succeeded();
+                        return Action.IDLE;
+                    }
+                    default:
+                    {
+                        throw new IllegalStateException(result.toString());
+                    }
+                }
             }
-            super.succeeded();
         }
 
         @Override
-        public void failed(Throwable x)
+        protected void onCompleteFailure(Throwable cause)
         {
-            for (ByteBuffer buffer : buffers)
+            release();
-            {
+            callback.failed(cause);
-                pool.release(buffer);
+        }
-            }
+
-            super.failed(x);
+        private void release()
+        {
+            HttpClient httpClient = getHttpChannel().getHttpDestination().getHttpClient();
+            ByteBufferPool bufferPool = httpClient.getByteBufferPool();
+            bufferPool.release(chunkBuffer);
+            chunkBuffer = null;
+            contentBuffer = null;
         }
     }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/internal/RequestContentAdapter.java

@@ -0,0 +1,329 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.internal;
+
+import java.io.Closeable;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+
+import org.eclipse.jetty.client.AsyncContentProvider;
+import org.eclipse.jetty.client.Synchronizable;
+import org.eclipse.jetty.client.api.ContentProvider;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.IO;
+import org.eclipse.jetty.util.thread.AutoLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>Implements the conversion from {@link ContentProvider} to {@link Request.Content}.</p>
+ */
+public class RequestContentAdapter implements Request.Content, Request.Content.Subscription, AsyncContentProvider.Listener, Callback
+{
+    private static final Logger LOG = LoggerFactory.getLogger(RequestContentAdapter.class);
+
+    private final AutoLock lock = new AutoLock();
+    private final ContentProvider provider;
+    private Iterator<ByteBuffer> iterator;
+    private Consumer consumer;
+    private boolean emitInitialContent;
+    private boolean lastContent;
+    private boolean committed;
+    private int demand;
+    private boolean stalled;
+    private boolean hasContent;
+    private Throwable failure;
+
+    public RequestContentAdapter(ContentProvider provider)
+    {
+        this.provider = provider;
+        if (provider instanceof AsyncContentProvider)
+            ((AsyncContentProvider)provider).setListener(this);
+    }
+
+    public ContentProvider getContentProvider()
+    {
+        return provider;
+    }
+
+    @Override
+    public String getContentType()
+    {
+        return provider instanceof ContentProvider.Typed ? ((ContentProvider.Typed)provider).getContentType() : null;
+    }
+
+    @Override
+    public long getLength()
+    {
+        return provider.getLength();
+    }
+
+    @Override
+    public boolean isReproducible()
+    {
+        return provider.isReproducible();
+    }
+
+    @Override
+    public Subscription subscribe(Consumer consumer, boolean emitInitialContent)
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            if (this.consumer != null && !isReproducible())
+                throw new IllegalStateException("Multiple subscriptions not supported on " + this);
+            this.iterator = provider.iterator();
+            this.consumer = consumer;
+            this.emitInitialContent = emitInitialContent;
+            this.lastContent = false;
+            this.committed = false;
+            this.demand = 0;
+            this.stalled = true;
+            this.hasContent = false;
+        }
+        return this;
+    }
+
+    @Override
+    public void demand()
+    {
+        boolean produce;
+        try (AutoLock ignored = lock.lock())
+        {
+            ++demand;
+            produce = stalled;
+            if (stalled)
+                stalled = false;
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("Content demand, producing {} for {}", produce, this);
+        if (produce)
+            produce();
+    }
+
+    @Override
+    public void fail(Throwable failure)
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            if (this.failure == null)
+                this.failure = failure;
+        }
+        failed(failure);
+    }
+
+    @Override
+    public void onContent()
+    {
+        boolean produce = false;
+        try (AutoLock ignored = lock.lock())
+        {
+            hasContent = true;
+            if (demand > 0)
+            {
+                produce = stalled;
+                if (stalled)
+                    stalled = false;
+            }
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("Content event, processing {} for {}", produce, this);
+        if (produce)
+            produce();
+    }
+
+    @Override
+    public void succeeded()
+    {
+        if (iterator instanceof Callback)
+            ((Callback)iterator).succeeded();
+        if (lastContent && iterator instanceof Closeable)
+            IO.close((Closeable)iterator);
+    }
+
+    @Override
+    public void failed(Throwable x)
+    {
+        if (iterator == null)
+            failed(provider, x);
+        else
+            failed(iterator, x);
+    }
+
+    private void failed(Object object, Throwable failure)
+    {
+        if (object instanceof Callback)
+            ((Callback)object).failed(failure);
+        if (object instanceof Closeable)
+            IO.close((Closeable)object);
+    }
+
+    @Override
+    public InvocationType getInvocationType()
+    {
+        return InvocationType.NON_BLOCKING;
+    }
+
+    private void produce()
+    {
+        while (true)
+        {
+            Throwable failure;
+            try (AutoLock ignored = lock.lock())
+            {
+                failure = this.failure;
+            }
+            if (failure != null)
+            {
+                notifyFailure(failure);
+                return;
+            }
+
+            if (committed)
+            {
+                ByteBuffer content = advance();
+                if (content != null)
+                {
+                    notifyContent(content, lastContent);
+                }
+                else
+                {
+                    try (AutoLock ignored = lock.lock())
+                    {
+                        // Call to advance() said there was no content,
+                        // but some content may have arrived meanwhile.
+                        if (hasContent)
+                        {
+                            hasContent = false;
+                            continue;
+                        }
+                        else
+                        {
+                            stalled = true;
+                        }
+                    }
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("No content, processing stalled for {}", this);
+                    return;
+                }
+            }
+            else
+            {
+                committed = true;
+                if (emitInitialContent)
+                {
+                    ByteBuffer content = advance();
+                    if (content != null)
+                        notifyContent(content, lastContent);
+                    else
+                        notifyContent(BufferUtil.EMPTY_BUFFER, false);
+                }
+                else
+                {
+                    notifyContent(BufferUtil.EMPTY_BUFFER, false);
+                }
+            }
+            boolean noDemand;
+            try (AutoLock ignored = lock.lock())
+            {
+                noDemand = demand == 0;
+                if (noDemand)
+                    stalled = true;
+            }
+            if (noDemand)
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("No demand, processing stalled for {}", this);
+                return;
+            }
+        }
+    }
+
+    private ByteBuffer advance()
+    {
+        if (iterator instanceof Synchronizable)
+        {
+            synchronized (((Synchronizable)iterator).getLock())
+            {
+                return next();
+            }
+        }
+        else
+        {
+            return next();
+        }
+    }
+
+    private ByteBuffer next()
+    {
+        boolean hasNext = iterator.hasNext();
+        ByteBuffer bytes = hasNext ? iterator.next() : null;
+        boolean hasMore = hasNext && iterator.hasNext();
+        lastContent = !hasMore;
+        return hasNext ? bytes : BufferUtil.EMPTY_BUFFER;
+    }
+
+    private void notifyContent(ByteBuffer buffer, boolean last)
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            --demand;
+            hasContent = false;
+        }
+
+        try
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Notifying content last={} {} for {}", last, BufferUtil.toDetailString(buffer), this);
+            consumer.onContent(buffer, last, this);
+        }
+        catch (Throwable x)
+        {
+            fail(x);
+        }
+    }
+
+    private void notifyFailure(Throwable failure)
+    {
+        try
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Notifying failure for {}", this, failure);
+            consumer.onFailure(failure);
+        }
+        catch (Exception x)
+        {
+            LOG.trace("Failure while notifying content failure {}", failure, x);
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        int demand;
+        boolean stalled;
+        try (AutoLock ignored = lock.lock())
+        {
+            demand = this.demand;
+            stalled = this.stalled;
+        }
+        return String.format("%s@%x[demand=%d,stalled=%b]", getClass().getSimpleName(), hashCode(), demand, stalled);
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/AbstractRequestContent.java

@@ -0,0 +1,257 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.nio.ByteBuffer;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.thread.AutoLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>Partial implementation of {@link Request.Content}.</p>
+ */
+public abstract class AbstractRequestContent implements Request.Content
+{
+    private static final Logger LOG = LoggerFactory.getLogger(AbstractRequestContent.class);
+
+    private final AutoLock lock = new AutoLock();
+    private final String contentType;
+
+    protected AbstractRequestContent(String contentType)
+    {
+        this.contentType = contentType;
+    }
+
+    @Override
+    public String getContentType()
+    {
+        return contentType;
+    }
+
+    @Override
+    public Subscription subscribe(Consumer consumer, boolean emitInitialContent)
+    {
+        Subscription subscription = newSubscription(consumer, emitInitialContent);
+        if (LOG.isDebugEnabled())
+            LOG.debug("Content subscription for {}: {}", subscription, consumer);
+        return subscription;
+    }
+
+    protected abstract Subscription newSubscription(Consumer consumer, boolean emitInitialContent);
+
+    /**
+     * <p>Partial implementation of {@code Subscription}.</p>
+     * <p>Implements the algorithm described in {@link Request.Content}.</p>
+     */
+    public abstract class AbstractSubscription implements Subscription
+    {
+        private final Consumer consumer;
+        private final boolean emitInitialContent;
+        private Throwable failure;
+        private int demand;
+        // Whether content production was stalled because there was no demand.
+        private boolean stalled;
+        // Whether the first content has been produced.
+        private boolean committed;
+
+        public AbstractSubscription(Consumer consumer, boolean emitInitialContent)
+        {
+            this.consumer = consumer;
+            this.emitInitialContent = emitInitialContent;
+            this.stalled = true;
+        }
+
+        @Override
+        public void demand()
+        {
+            boolean produce;
+            try (AutoLock ignored = lock.lock())
+            {
+                ++demand;
+                produce = stalled;
+                if (stalled)
+                    stalled = false;
+            }
+            if (LOG.isDebugEnabled())
+                LOG.debug("Content demand, producing {} for {}", produce, this);
+            if (produce)
+                produce();
+        }
+
+        private void produce()
+        {
+            while (true)
+            {
+                Throwable failure;
+                boolean committed;
+                try (AutoLock ignored = lock.lock())
+                {
+                    failure = this.failure;
+                    committed = this.committed;
+                }
+                if (failure != null)
+                {
+                    notifyFailure(failure);
+                    return;
+                }
+
+                if (committed || emitInitialContent)
+                {
+                    try
+                    {
+                        if (!produceContent(this::processContent))
+                            return;
+                    }
+                    catch (Throwable x)
+                    {
+                        // Fail and loop around to notify the failure.
+                        fail(x);
+                    }
+                }
+                else
+                {
+                    if (!processContent(BufferUtil.EMPTY_BUFFER, false, Callback.NOOP))
+                        return;
+                }
+            }
+        }
+
+        /**
+         * <p>Subclasses implement this method to produce content,
+         * without worrying about demand or exception handling.</p>
+         * <p>Typical implementation (pseudo code):</p>
+         * <pre>
+         * protected boolean produceContent(Producer producer) throws Exception
+         * {
+         *     // Step 1: try to produce content, exceptions may be thrown during production
+         *     //  (for example, producing content reading from an InputStream may throw).
+         *
+         *     // Step 2A: content could be produced.
+         *     ByteBuffer buffer = ...;
+         *     boolean last = ...;
+         *     Callback callback = ...;
+         *     return producer.produce(buffer, last, callback);
+         *
+         *     // Step 2B: content could not be produced.
+         *     //  (for example it is not available yet)
+         *     return false;
+         * }
+         * </pre>
+         *
+         * @param producer the producer to notify when content can be produced
+         * @return whether content production should continue
+         * @throws Exception when content production fails
+         */
+        protected abstract boolean produceContent(Producer producer) throws Exception;
+
+        @Override
+        public void fail(Throwable failure)
+        {
+            try (AutoLock ignored = lock.lock())
+            {
+                if (this.failure == null)
+                    this.failure = failure;
+            }
+        }
+
+        private boolean processContent(ByteBuffer content, boolean last, Callback callback)
+        {
+            try (AutoLock ignored = lock.lock())
+            {
+                committed = true;
+                --demand;
+            }
+
+            if (content != null)
+                notifyContent(content, last, callback);
+            else
+                callback.succeeded();
+
+            boolean noDemand;
+            try (AutoLock ignored = lock.lock())
+            {
+                noDemand = demand == 0;
+                if (noDemand)
+                    stalled = true;
+            }
+            if (noDemand)
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("No demand, processing stalled for {}", this);
+                return false;
+            }
+            return true;
+        }
+
+        protected void notifyContent(ByteBuffer buffer, boolean last, Callback callback)
+        {
+            try
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Notifying content last={} {} for {}", last, BufferUtil.toDetailString(buffer), this);
+                consumer.onContent(buffer, last, callback);
+            }
+            catch (Throwable x)
+            {
+                callback.failed(x);
+                fail(x);
+            }
+        }
+
+        private void notifyFailure(Throwable failure)
+        {
+            try
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Notifying failure for {}", this, failure);
+                consumer.onFailure(failure);
+            }
+            catch (Exception x)
+            {
+                LOG.trace("Failure while notifying content failure {}", failure, x);
+            }
+        }
+
+        @Override
+        public String toString()
+        {
+            int demand;
+            boolean stalled;
+            boolean committed;
+            try (AutoLock ignored = lock.lock())
+            {
+                demand = this.demand;
+                stalled = this.stalled;
+                committed = this.committed;
+            }
+            return String.format("%s.%s@%x[demand=%d,stalled=%b,committed=%b,emitInitial=%b]",
+                getClass().getEnclosingClass().getSimpleName(),
+                getClass().getSimpleName(), hashCode(), demand, stalled, committed, emitInitialContent);
+        }
+    }
+
+    public interface Producer
+    {
+        boolean produce(ByteBuffer content, boolean lastContent, Callback callback);
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/AbstractTypedContentProvider.java

@@ -20,6 +20,10 @@ package org.eclipse.jetty.client.util;
 
 import org.eclipse.jetty.client.api.ContentProvider;
 
+/**
+ * @deprecated use {@link AbstractRequestContent} instead.
+ */
+@Deprecated
 public abstract class AbstractTypedContentProvider implements ContentProvider.Typed
 {
     private final String contentType;

jetty-client/src/main/java/org/eclipse/jetty/client/util/AsyncRequestContent.java

@@ -0,0 +1,385 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.Closeable;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.locks.Condition;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.thread.AutoLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AsyncRequestContent implements Request.Content, Request.Content.Subscription, Closeable
+{
+    private static final Logger LOG = LoggerFactory.getLogger(AsyncRequestContent.class);
+
+    private final AutoLock lock = new AutoLock();
+    private final Condition flush = lock.newCondition();
+    private final Deque<Chunk> chunks = new ArrayDeque<>();
+    private final String contentType;
+    private long length = -1;
+    private Consumer consumer;
+    private boolean emitInitialContent;
+    private int demand;
+    private boolean stalled;
+    private boolean committed;
+    private boolean closed;
+    private boolean terminated;
+    private Throwable failure;
+
+    public AsyncRequestContent(ByteBuffer... buffers)
+    {
+        this("application/octet-stream", buffers);
+    }
+
+    public AsyncRequestContent(String contentType, ByteBuffer... buffers)
+    {
+        this.contentType = contentType;
+        Stream.of(buffers).forEach(this::offer);
+    }
+
+    @Override
+    public String getContentType()
+    {
+        return contentType;
+    }
+
+    @Override
+    public long getLength()
+    {
+        return length;
+    }
+
+    @Override
+    public Subscription subscribe(Consumer consumer, boolean emitInitialContent)
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            if (this.consumer != null)
+                throw new IllegalStateException("Multiple subscriptions not supported on " + this);
+            this.consumer = consumer;
+            this.emitInitialContent = emitInitialContent;
+            this.stalled = true;
+            if (closed)
+                length = chunks.stream().mapToLong(chunk -> chunk.buffer.remaining()).sum();
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("Content subscription for {}: {}", this, consumer);
+        return this;
+    }
+
+    @Override
+    public void demand()
+    {
+        boolean produce;
+        try (AutoLock ignored = lock.lock())
+        {
+            ++demand;
+            produce = stalled;
+            if (stalled)
+                stalled = false;
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("Content demand, producing {} for {}", produce, this);
+        if (produce)
+            produce();
+    }
+
+    @Override
+    public void fail(Throwable failure)
+    {
+        List<Callback> toFail = List.of();
+        try (AutoLock ignored = lock.lock())
+        {
+            if (this.failure == null)
+            {
+                this.failure = failure;
+                // Transfer all chunks to fail them all.
+                toFail = chunks.stream()
+                    .map(chunk -> chunk.callback)
+                    .collect(Collectors.toList());
+                chunks.clear();
+                flush.signal();
+            }
+        }
+        toFail.forEach(c -> c.failed(failure));
+    }
+
+    public boolean offer(ByteBuffer buffer)
+    {
+        return offer(buffer, Callback.NOOP);
+    }
+
+    public boolean offer(ByteBuffer buffer, Callback callback)
+    {
+        return offer(new Chunk(buffer, callback));
+    }
+
+    private boolean offer(Chunk chunk)
+    {
+        boolean produce = false;
+        Throwable failure;
+        try (AutoLock ignored = lock.lock())
+        {
+            failure = this.failure;
+            if (failure == null)
+            {
+                if (closed)
+                {
+                    failure = new IOException("closed");
+                }
+                else
+                {
+                    chunks.offer(chunk);
+                    if (demand > 0)
+                    {
+                        if (stalled)
+                        {
+                            stalled = false;
+                            produce = true;
+                        }
+                    }
+                }
+            }
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("Content offer {}, producing {} for {}", failure == null ? "succeeded" : "failed", produce, this, failure);
+        if (failure != null)
+        {
+            chunk.callback.failed(failure);
+            return false;
+        }
+        else if (produce)
+        {
+            produce();
+        }
+        return true;
+    }
+
+    private void produce()
+    {
+        while (true)
+        {
+            Throwable failure;
+            try (AutoLock ignored = lock.lock())
+            {
+                failure = this.failure;
+            }
+            if (failure != null)
+            {
+                notifyFailure(consumer, failure);
+                return;
+            }
+
+            try
+            {
+                Consumer consumer;
+                Chunk chunk = Chunk.EMPTY;
+                boolean lastContent = false;
+                try (AutoLock ignored = lock.lock())
+                {
+                    if (terminated)
+                        throw new EOFException("Demand after last content");
+                    consumer = this.consumer;
+                    if (committed || emitInitialContent)
+                    {
+                        chunk = chunks.poll();
+                        lastContent = closed && chunks.isEmpty();
+                        if (lastContent)
+                            terminated = true;
+                    }
+                    if (chunk == null && (lastContent || !committed))
+                        chunk = Chunk.EMPTY;
+                    if (chunk == null)
+                    {
+                        stalled = true;
+                    }
+                    else
+                    {
+                        --demand;
+                        committed = true;
+                    }
+                }
+                if (chunk == null)
+                {
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("No content, processing stalled for {}", this);
+                    return;
+                }
+
+                notifyContent(consumer, chunk.buffer, lastContent, Callback.from(this::notifyFlush, chunk.callback));
+
+                boolean noDemand;
+                try (AutoLock ignored = lock.lock())
+                {
+                    noDemand = demand == 0;
+                    if (noDemand)
+                        stalled = true;
+                }
+                if (noDemand)
+                {
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("No demand, processing stalled for {}", this);
+                    return;
+                }
+            }
+            catch (Throwable x)
+            {
+                // Fail and loop around to notify the failure.
+                fail(x);
+            }
+        }
+    }
+
+    private void notifyContent(Consumer consumer, ByteBuffer buffer, boolean last, Callback callback)
+    {
+        try
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Notifying content last={} {} for {}", last, BufferUtil.toDetailString(buffer), this);
+            consumer.onContent(buffer, last, callback);
+        }
+        catch (Throwable x)
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Failure while notifying content", x);
+            callback.failed(x);
+            fail(x);
+        }
+    }
+
+    private void notifyFailure(Consumer consumer, Throwable failure)
+    {
+        try
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Notifying failure for {}", this, failure);
+            consumer.onFailure(failure);
+        }
+        catch (Throwable x)
+        {
+            LOG.trace("Failure while notifying content failure {}", failure, x);
+        }
+    }
+
+    private void notifyFlush()
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            flush.signal();
+        }
+    }
+
+    public void flush() throws IOException
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            try
+            {
+                while (true)
+                {
+                    // Always wrap the exception to make sure
+                    // the stack trace comes from flush().
+                    if (failure != null)
+                        throw new IOException(failure);
+                    if (chunks.isEmpty())
+                        return;
+                    flush.await();
+                }
+            }
+            catch (InterruptedException x)
+            {
+                throw new InterruptedIOException();
+            }
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        boolean produce = false;
+        try (AutoLock ignored = lock.lock())
+        {
+            if (closed)
+                return;
+            closed = true;
+            if (demand > 0)
+            {
+                if (stalled)
+                {
+                    stalled = false;
+                    produce = true;
+                }
+            }
+            flush.signal();
+        }
+        if (produce)
+            produce();
+    }
+
+    public boolean isClosed()
+    {
+        try (AutoLock ignored = lock.lock())
+        {
+            return closed;
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        int demand;
+        boolean stalled;
+        int chunks;
+        try (AutoLock ignored = lock.lock())
+        {
+            demand = this.demand;
+            stalled = this.stalled;
+            chunks = this.chunks.size();
+        }
+        return String.format("%s@%x[demand=%d,stalled=%b,chunks=%d]", getClass().getSimpleName(), hashCode(), demand, stalled, chunks);
+    }
+
+    private static class Chunk
+    {
+        private static final Chunk EMPTY = new Chunk(BufferUtil.EMPTY_BUFFER, Callback.NOOP);
+
+        private final ByteBuffer buffer;
+        private final Callback callback;
+
+        private Chunk(ByteBuffer buffer, Callback callback)
+        {
+            this.buffer = Objects.requireNonNull(buffer);
+            this.callback = Objects.requireNonNull(callback);
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/ByteBufferContentProvider.java

@@ -30,7 +30,10 @@ import org.eclipse.jetty.client.api.ContentProvider;
  * The position and limit of the {@link ByteBuffer}s passed to the constructor are not modified,
  * and each invocation of the {@link #iterator()} method returns a {@link ByteBuffer#slice() slice}
  * of the original {@link ByteBuffer}.
+ *
+ * @deprecated use {@link ByteBufferRequestContent} instead.
  */
+@Deprecated
 public class ByteBufferContentProvider extends AbstractTypedContentProvider
 {
     private final ByteBuffer[] buffers;

jetty-client/src/main/java/org/eclipse/jetty/client/util/ByteBufferRequestContent.java

@@ -0,0 +1,94 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+
+/**
+ * <p>A {@link Request.Content} for {@link ByteBuffer}s.</p>
+ * <p>The position and limit of the {@link ByteBuffer}s passed to the constructor are not modified;
+ * content production returns a {@link ByteBuffer#slice() slice} of the original {@link ByteBuffer}.
+ */
+public class ByteBufferRequestContent extends AbstractRequestContent
+{
+    private final ByteBuffer[] buffers;
+    private final long length;
+
+    public ByteBufferRequestContent(ByteBuffer... buffers)
+    {
+        this("application/octet-stream", buffers);
+    }
+
+    public ByteBufferRequestContent(String contentType, ByteBuffer... buffers)
+    {
+        super(contentType);
+        this.buffers = buffers;
+        this.length = Arrays.stream(buffers).mapToLong(Buffer::remaining).sum();
+    }
+
+    @Override
+    public long getLength()
+    {
+        return length;
+    }
+
+    @Override
+    public boolean isReproducible()
+    {
+        return true;
+    }
+
+    @Override
+    protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
+    {
+        return new SubscriptionImpl(consumer, emitInitialContent);
+    }
+
+    private class SubscriptionImpl extends AbstractSubscription
+    {
+        private int index;
+
+        private SubscriptionImpl(Consumer consumer, boolean emitInitialContent)
+        {
+            super(consumer, emitInitialContent);
+        }
+
+        @Override
+        protected boolean produceContent(Producer producer) throws IOException
+        {
+            if (index < 0)
+                throw new EOFException("Demand after last content");
+            ByteBuffer buffer = BufferUtil.EMPTY_BUFFER;
+            if (index < buffers.length)
+                buffer = buffers[index++];
+            boolean lastContent = index == buffers.length;
+            if (lastContent)
+                index = -1;
+            return producer.produce(buffer.slice(), lastContent, Callback.NOOP);
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/BytesContentProvider.java

@@ -26,7 +26,10 @@ import org.eclipse.jetty.client.api.ContentProvider;
 
 /**
  * A {@link ContentProvider} for byte arrays.
+ *
+ * @deprecated use {@link BytesRequestContent} instead.
  */
+@Deprecated
 public class BytesContentProvider extends AbstractTypedContentProvider
 {
     private final byte[][] bytes;

jetty-client/src/main/java/org/eclipse/jetty/client/util/BytesRequestContent.java

@@ -0,0 +1,91 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+
+/**
+ * A {@link Request.Content} for byte arrays.
+ */
+public class BytesRequestContent extends AbstractRequestContent
+{
+    private final byte[][] bytes;
+    private final long length;
+
+    public BytesRequestContent(byte[]... bytes)
+    {
+        this("application/octet-stream", bytes);
+    }
+
+    public BytesRequestContent(String contentType, byte[]... bytes)
+    {
+        super(contentType);
+        this.bytes = bytes;
+        this.length = Arrays.stream(bytes).mapToLong(a -> a.length).sum();
+    }
+
+    @Override
+    public long getLength()
+    {
+        return length;
+    }
+
+    @Override
+    public boolean isReproducible()
+    {
+        return true;
+    }
+
+    @Override
+    protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
+    {
+        return new SubscriptionImpl(consumer, emitInitialContent);
+    }
+
+    private class SubscriptionImpl extends AbstractSubscription
+    {
+        private int index;
+
+        private SubscriptionImpl(Consumer consumer, boolean emitInitialContent)
+        {
+            super(consumer, emitInitialContent);
+        }
+
+        @Override
+        protected boolean produceContent(Producer producer) throws IOException
+        {
+            if (index < 0)
+                throw new EOFException("Demand after last content");
+            ByteBuffer buffer = BufferUtil.EMPTY_BUFFER;
+            if (index < bytes.length)
+                buffer = ByteBuffer.wrap(bytes[index++]);
+            boolean lastContent = index == bytes.length;
+            if (lastContent)
+                index = -1;
+            return producer.produce(buffer, lastContent, Callback.NOOP);
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/DeferredContentProvider.java

@@ -85,7 +85,10 @@ import org.eclipse.jetty.util.Callback;
  *     content.offer(ByteBuffer.wrap("some content".getBytes()));
  * }
  * </pre>
+ *
+ * @deprecated use {@link AsyncRequestContent} instead.
  */
+@Deprecated
 public class DeferredContentProvider implements AsyncContentProvider, Callback, Closeable
 {
     private static final Chunk CLOSE = new Chunk(BufferUtil.EMPTY_BUFFER, Callback.NOOP);
@@ -285,6 +288,7 @@ public class DeferredContentProvider implements AsyncContentProvider, Callback,
             synchronized (lock)
             {
                 chunk = current;
+                current = null;
                 if (chunk != null)
                 {
                     --size;

jetty-client/src/main/java/org/eclipse/jetty/client/util/FormContentProvider.java

@@ -30,7 +30,10 @@ import org.eclipse.jetty.util.Fields;
 /**
  * A {@link ContentProvider} for form uploads with the
  * "application/x-www-form-urlencoded" content type.
+ *
+ * @deprecated use {@link FormRequestContent} instead.
  */
+@Deprecated
 public class FormContentProvider extends StringContentProvider
 {
     public FormContentProvider(Fields fields)

jetty-client/src/main/java/org/eclipse/jetty/client/util/FormRequestContent.java

@@ -0,0 +1,78 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.charset.UnsupportedCharsetException;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.Fields;
+
+/**
+ * <p>A {@link Request.Content} for form uploads with the
+ * "application/x-www-form-urlencoded" content type.</p>
+ */
+public class FormRequestContent extends StringRequestContent
+{
+    public FormRequestContent(Fields fields)
+    {
+        this(fields, StandardCharsets.UTF_8);
+    }
+
+    public FormRequestContent(Fields fields, Charset charset)
+    {
+        super("application/x-www-form-urlencoded", convert(fields, charset), charset);
+    }
+
+    public static String convert(Fields fields)
+    {
+        return convert(fields, StandardCharsets.UTF_8);
+    }
+
+    public static String convert(Fields fields, Charset charset)
+    {
+        // Assume 32 chars between name and value.
+        StringBuilder builder = new StringBuilder(fields.getSize() * 32);
+        for (Fields.Field field : fields)
+        {
+            for (String value : field.getValues())
+            {
+                if (builder.length() > 0)
+                    builder.append("&");
+                builder.append(encode(field.getName(), charset)).append("=").append(encode(value, charset));
+            }
+        }
+        return builder.toString();
+    }
+
+    private static String encode(String value, Charset charset)
+    {
+        try
+        {
+            return URLEncoder.encode(value, charset.name());
+        }
+        catch (UnsupportedEncodingException x)
+        {
+            throw new UnsupportedCharsetException(charset.name());
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/InputStreamContentProvider.java

@@ -50,7 +50,10 @@ import org.slf4j.LoggerFactory;
  * The {@link InputStream} passed to the constructor is by default closed when is it fully
  * consumed (or when an exception is thrown while reading it), unless otherwise specified
  * to the {@link #InputStreamContentProvider(java.io.InputStream, int, boolean) constructor}.
+ *
+ * @deprecated use {@link InputStreamRequestContent} instead
  */
+@Deprecated
 public class InputStreamContentProvider implements ContentProvider, Callback, Closeable
 {
     private static final Logger LOG = LoggerFactory.getLogger(InputStreamContentProvider.class);

jetty-client/src/main/java/org/eclipse/jetty/client/util/InputStreamRequestContent.java

@@ -0,0 +1,149 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.IO;
+
+/**
+ * <p>A {@link Request.Content} that produces content from an {@link InputStream}.</p>
+ * <p>The input stream is read once and therefore fully consumed.</p>
+ * <p>It is possible to specify, at the constructor, a buffer size used to read
+ * content from the stream, by default 1024 bytes.</p>
+ * <p>The {@link InputStream} passed to the constructor is by default closed
+ * when is it fully consumed.</p>
+ */
+public class InputStreamRequestContent extends AbstractRequestContent
+{
+    private static final int DEFAULT_BUFFER_SIZE = 4096;
+
+    private final InputStream stream;
+    private final int bufferSize;
+    private Subscription subscription;
+
+    public InputStreamRequestContent(InputStream stream)
+    {
+        this(stream, DEFAULT_BUFFER_SIZE);
+    }
+
+    public InputStreamRequestContent(String contentType, InputStream stream)
+    {
+        this(contentType, stream, DEFAULT_BUFFER_SIZE);
+    }
+
+    public InputStreamRequestContent(InputStream stream, int bufferSize)
+    {
+        this("application/octet-stream", stream, bufferSize);
+    }
+
+    public InputStreamRequestContent(String contentType, InputStream stream, int bufferSize)
+    {
+        super(contentType);
+        this.stream = stream;
+        this.bufferSize = bufferSize;
+    }
+
+    @Override
+    protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
+    {
+        if (subscription != null)
+            throw new IllegalStateException("Multiple subscriptions not supported on " + this);
+        return subscription = new SubscriptionImpl(consumer, emitInitialContent);
+    }
+
+    @Override
+    public void fail(Throwable failure)
+    {
+        super.fail(failure);
+        close();
+    }
+
+    protected ByteBuffer onRead(byte[] buffer, int offset, int length)
+    {
+        return ByteBuffer.wrap(buffer, offset, length);
+    }
+
+    protected void onReadFailure(Throwable failure)
+    {
+    }
+
+    private void close()
+    {
+        IO.close(stream);
+    }
+
+    private class SubscriptionImpl extends AbstractSubscription
+    {
+        private boolean terminated;
+
+        private SubscriptionImpl(Consumer consumer, boolean emitInitialContent)
+        {
+            super(consumer, emitInitialContent);
+        }
+
+        @Override
+        protected boolean produceContent(Producer producer) throws IOException
+        {
+            if (terminated)
+                throw new EOFException("Demand after last content");
+            byte[] bytes = new byte[bufferSize];
+            int read = read(bytes);
+            ByteBuffer buffer = BufferUtil.EMPTY_BUFFER;
+            boolean last = true;
+            if (read < 0)
+            {
+                close();
+                terminated = true;
+            }
+            else
+            {
+                buffer = onRead(bytes, 0, read);
+                last = false;
+            }
+            return producer.produce(buffer, last, Callback.NOOP);
+        }
+
+        private int read(byte[] bytes) throws IOException
+        {
+            try
+            {
+                return stream.read(bytes);
+            }
+            catch (Throwable x)
+            {
+                onReadFailure(x);
+                throw x;
+            }
+        }
+
+        @Override
+        public void fail(Throwable failure)
+        {
+            super.fail(failure);
+            close();
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/InputStreamResponseListener.java

@@ -27,6 +27,7 @@ import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.Queue;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
@@ -76,12 +77,13 @@ import org.slf4j.LoggerFactory;
 public class InputStreamResponseListener extends Listener.Adapter
 {
     private static final Logger LOG = LoggerFactory.getLogger(InputStreamResponseListener.class);
-    private static final DeferredContentProvider.Chunk EOF = new DeferredContentProvider.Chunk(BufferUtil.EMPTY_BUFFER, Callback.NOOP);
+    private static final Chunk EOF = new Chunk(BufferUtil.EMPTY_BUFFER, Callback.NOOP);
+
     private final Object lock = this;
     private final CountDownLatch responseLatch = new CountDownLatch(1);
     private final CountDownLatch resultLatch = new CountDownLatch(1);
     private final AtomicReference<InputStream> stream = new AtomicReference<>();
-    private final Queue<DeferredContentProvider.Chunk> chunks = new ArrayDeque<>();
+    private final Queue<Chunk> chunks = new ArrayDeque<>();
     private Response response;
     private Result result;
     private Throwable failure;
@@ -120,7 +122,7 @@ public class InputStreamResponseListener extends Listener.Adapter
             {
                 if (LOG.isDebugEnabled())
                     LOG.debug("Queueing content {}", content);
-                chunks.add(new DeferredContentProvider.Chunk(content, callback));
+                chunks.add(new Chunk(content, callback));
                 lock.notifyAll();
             }
         }
@@ -268,7 +270,7 @@ public class InputStreamResponseListener extends Listener.Adapter
         {
             while (true)
             {
-                DeferredContentProvider.Chunk chunk = chunks.peek();
+                Chunk chunk = chunks.peek();
                 if (chunk == null || chunk == EOF)
                     break;
                 callbacks.add(chunk.callback);
@@ -299,7 +301,7 @@ public class InputStreamResponseListener extends Listener.Adapter
                 Callback callback = null;
                 synchronized (lock)
                 {
-                    DeferredContentProvider.Chunk chunk;
+                    Chunk chunk;
                     while (true)
                     {
                         chunk = chunks.peek();
@@ -367,4 +369,16 @@ public class InputStreamResponseListener extends Listener.Adapter
             super.close();
         }
     }
+
+    private static class Chunk
+    {
+        private final ByteBuffer buffer;
+        private final Callback callback;
+
+        private Chunk(ByteBuffer buffer, Callback callback)
+        {
+            this.buffer = Objects.requireNonNull(buffer);
+            this.callback = Objects.requireNonNull(callback);
+        }
+    }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/util/MultiPartContentProvider.java

@@ -63,7 +63,10 @@ import org.slf4j.LoggerFactory;
  *     <input type="file" name="icon" />
  * </form>
  * </pre>
+ *
+ * @deprecated use {@link MultiPartRequestContent} instead.
  */
+@Deprecated
 public class MultiPartContentProvider extends AbstractTypedContentProvider implements AsyncContentProvider, Closeable
 {
     private static final Logger LOG = LoggerFactory.getLogger(MultiPartContentProvider.class);

jetty-client/src/main/java/org/eclipse/jetty/client/util/MultiPartRequestContent.java

@@ -0,0 +1,392 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.http.HttpField;
+import org.eclipse.jetty.http.HttpFields;
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.io.RuntimeIOException;
+import org.eclipse.jetty.util.Callback;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>A {@link Request.Content} for form uploads with the {@code "multipart/form-data"}
+ * content type.</p>
+ * <p>Example usage:</p>
+ * <pre>
+ * MultiPartRequestContent multiPart = new MultiPartRequestContent();
+ * multiPart.addFieldPart("field", new StringRequestContent("foo"), null);
+ * multiPart.addFilePart("icon", "img.png", new PathRequestContent(Paths.get("/tmp/img.png")), null);
+ * multiPart.close();
+ * ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
+ *         .method(HttpMethod.POST)
+ *         .content(multiPart)
+ *         .send();
+ * </pre>
+ * <p>The above example would be the equivalent of submitting this form:</p>
+ * <pre>
+ * &lt;form method="POST" enctype="multipart/form-data"  accept-charset="UTF-8"&gt;
+ *     &lt;input type="text" name="field" value="foo" /&gt;
+ *     &lt;input type="file" name="icon" /&gt;
+ * &lt;/form&gt;
+ * </pre>
+ */
+public class MultiPartRequestContent extends AbstractRequestContent implements Closeable
+{
+    private static final Logger LOG = LoggerFactory.getLogger(MultiPartRequestContent.class);
+    private static final byte[] COLON_SPACE_BYTES = new byte[]{':', ' '};
+    private static final byte[] CR_LF_BYTES = new byte[]{'\r', '\n'};
+
+    private static String makeBoundary()
+    {
+        Random random = new Random();
+        StringBuilder builder = new StringBuilder("JettyHttpClientBoundary");
+        int length = builder.length();
+        while (builder.length() < length + 16)
+        {
+            long rnd = random.nextLong();
+            builder.append(Long.toString(rnd < 0 ? -rnd : rnd, 36));
+        }
+        builder.setLength(length + 16);
+        return builder.toString();
+    }
+
+    private final List<Part> parts = new ArrayList<>();
+    private final ByteBuffer firstBoundary;
+    private final ByteBuffer middleBoundary;
+    private final ByteBuffer onlyBoundary;
+    private final ByteBuffer lastBoundary;
+    private long length;
+    private boolean closed;
+    private Subscription subscription;
+
+    public MultiPartRequestContent()
+    {
+        this(makeBoundary());
+    }
+
+    public MultiPartRequestContent(String boundary)
+    {
+        super("multipart/form-data; boundary=" + boundary);
+        String firstBoundaryLine = "--" + boundary + "\r\n";
+        this.firstBoundary = ByteBuffer.wrap(firstBoundaryLine.getBytes(StandardCharsets.US_ASCII));
+        String middleBoundaryLine = "\r\n" + firstBoundaryLine;
+        this.middleBoundary = ByteBuffer.wrap(middleBoundaryLine.getBytes(StandardCharsets.US_ASCII));
+        String onlyBoundaryLine = "--" + boundary + "--\r\n";
+        this.onlyBoundary = ByteBuffer.wrap(onlyBoundaryLine.getBytes(StandardCharsets.US_ASCII));
+        String lastBoundaryLine = "\r\n" + onlyBoundaryLine;
+        this.lastBoundary = ByteBuffer.wrap(lastBoundaryLine.getBytes(StandardCharsets.US_ASCII));
+        this.length = -1;
+    }
+
+    @Override
+    public long getLength()
+    {
+        return length;
+    }
+
+    @Override
+    protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
+    {
+        if (!closed)
+            throw new IllegalStateException("MultiPartRequestContent must be closed before sending the request");
+        if (subscription != null)
+            throw new IllegalStateException("Multiple subscriptions not supported on " + this);
+        length = calculateLength();
+        return subscription = new SubscriptionImpl(consumer, emitInitialContent);
+    }
+
+    @Override
+    public void fail(Throwable failure)
+    {
+        parts.stream()
+            .map(part -> part.content)
+            .forEach(content -> content.fail(failure));
+    }
+
+    /**
+     * <p>Adds a field part with the given {@code name} as field name, and the given
+     * {@code content} as part content.</p>
+     * <p>The {@code Content-Type} of this part will be obtained from:</p>
+     * <ul>
+     * <li>the {@code Content-Type} header in the {@code fields} parameter; otherwise</li>
+     * <li>the {@link Request.Content#getContentType()}</li>
+     * </ul>
+     *
+     * @param name the part name
+     * @param content the part content
+     * @param fields the headers associated with this part
+     */
+    public void addFieldPart(String name, Request.Content content, HttpFields fields)
+    {
+        addPart(new Part(name, null, content, fields));
+    }
+
+    /**
+     * <p>Adds a file part with the given {@code name} as field name, the given
+     * {@code fileName} as file name, and the given {@code content} as part content.</p>
+     * <p>The {@code Content-Type} of this part will be obtained from:</p>
+     * <ul>
+     * <li>the {@code Content-Type} header in the {@code fields} parameter; otherwise</li>
+     * <li>the {@link Request.Content#getContentType()}</li>
+     * </ul>
+     *
+     * @param name the part name
+     * @param fileName the file name associated to this part
+     * @param content the part content
+     * @param fields the headers associated with this part
+     */
+    public void addFilePart(String name, String fileName, Request.Content content, HttpFields fields)
+    {
+        addPart(new Part(name, fileName, content, fields));
+    }
+
+    private void addPart(Part part)
+    {
+        parts.add(part);
+        if (LOG.isDebugEnabled())
+            LOG.debug("Added {}", part);
+    }
+
+    @Override
+    public void close()
+    {
+        closed = true;
+    }
+
+    private long calculateLength()
+    {
+        // Compute the length, if possible.
+        if (parts.isEmpty())
+        {
+            return onlyBoundary.remaining();
+        }
+        else
+        {
+            long result = 0;
+            for (int i = 0; i < parts.size(); ++i)
+            {
+                result += (i == 0) ? firstBoundary.remaining() : middleBoundary.remaining();
+                Part part = parts.get(i);
+                long partLength = part.length;
+                result += partLength;
+                if (partLength < 0)
+                {
+                    result = -1;
+                    break;
+                }
+            }
+            if (result > 0)
+                result += lastBoundary.remaining();
+            return result;
+        }
+    }
+
+    private static class Part
+    {
+        private final String name;
+        private final String fileName;
+        private final Request.Content content;
+        private final HttpFields fields;
+        private final ByteBuffer headers;
+        private final long length;
+
+        private Part(String name, String fileName, Request.Content content, HttpFields fields)
+        {
+            this.name = name;
+            this.fileName = fileName;
+            this.content = content;
+            this.fields = fields;
+            this.headers = headers();
+            this.length = content.getLength() < 0 ? -1 : headers.remaining() + content.getLength();
+        }
+
+        private ByteBuffer headers()
+        {
+            try
+            {
+                // Compute the Content-Disposition.
+                String contentDisposition = "Content-Disposition: form-data; name=\"" + name + "\"";
+                if (fileName != null)
+                    contentDisposition += "; filename=\"" + fileName + "\"";
+                contentDisposition += "\r\n";
+
+                // Compute the Content-Type.
+                String contentType = fields == null ? null : fields.get(HttpHeader.CONTENT_TYPE);
+                if (contentType == null)
+                    contentType = content.getContentType();
+                contentType = "Content-Type: " + contentType + "\r\n";
+
+                if (fields == null || fields.size() == 0)
+                {
+                    String headers = contentDisposition;
+                    headers += contentType;
+                    headers += "\r\n";
+                    return ByteBuffer.wrap(headers.getBytes(StandardCharsets.UTF_8));
+                }
+
+                ByteArrayOutputStream buffer = new ByteArrayOutputStream((fields.size() + 1) * contentDisposition.length());
+                buffer.write(contentDisposition.getBytes(StandardCharsets.UTF_8));
+                buffer.write(contentType.getBytes(StandardCharsets.UTF_8));
+                for (HttpField field : fields)
+                {
+                    if (HttpHeader.CONTENT_TYPE.equals(field.getHeader()))
+                        continue;
+                    buffer.write(field.getName().getBytes(StandardCharsets.US_ASCII));
+                    buffer.write(COLON_SPACE_BYTES);
+                    String value = field.getValue();
+                    if (value != null)
+                        buffer.write(value.getBytes(StandardCharsets.UTF_8));
+                    buffer.write(CR_LF_BYTES);
+                }
+                buffer.write(CR_LF_BYTES);
+                return ByteBuffer.wrap(buffer.toByteArray());
+            }
+            catch (IOException x)
+            {
+                throw new RuntimeIOException(x);
+            }
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s@%x[name=%s,fileName=%s,length=%d,headers=%s]",
+                getClass().getSimpleName(),
+                hashCode(),
+                name,
+                fileName,
+                content.getLength(),
+                fields);
+        }
+    }
+
+    private class SubscriptionImpl extends AbstractSubscription implements Consumer
+    {
+        private State state = State.FIRST_BOUNDARY;
+        private int index;
+        private Subscription subscription;
+
+        private SubscriptionImpl(Consumer consumer, boolean emitInitialContent)
+        {
+            super(consumer, emitInitialContent);
+        }
+
+        @Override
+        protected boolean produceContent(Producer producer) throws IOException
+        {
+            ByteBuffer buffer;
+            boolean last = false;
+            switch (state)
+            {
+                case FIRST_BOUNDARY:
+                {
+                    if (parts.isEmpty())
+                    {
+                        state = State.COMPLETE;
+                        buffer = onlyBoundary.slice();
+                        last = true;
+                        break;
+                    }
+                    else
+                    {
+                        state = State.HEADERS;
+                        buffer = firstBoundary.slice();
+                        break;
+                    }
+                }
+                case HEADERS:
+                {
+                    Part part = parts.get(index);
+                    Request.Content content = part.content;
+                    subscription = content.subscribe(this, true);
+                    state = State.CONTENT;
+                    buffer = part.headers.slice();
+                    break;
+                }
+                case CONTENT:
+                {
+                    buffer = null;
+                    subscription.demand();
+                    break;
+                }
+                case MIDDLE_BOUNDARY:
+                {
+                    state = State.HEADERS;
+                    buffer = middleBoundary.slice();
+                    break;
+                }
+                case LAST_BOUNDARY:
+                {
+                    state = State.COMPLETE;
+                    buffer = lastBoundary.slice();
+                    last = true;
+                    break;
+                }
+                case COMPLETE:
+                {
+                    throw new EOFException("Demand after last content");
+                }
+                default:
+                {
+                    throw new IllegalStateException("Invalid state " + state);
+                }
+            }
+            return producer.produce(buffer, last, Callback.NOOP);
+        }
+
+        @Override
+        public void onContent(ByteBuffer buffer, boolean last, Callback callback)
+        {
+            if (last)
+            {
+                ++index;
+                if (index < parts.size())
+                    state = State.MIDDLE_BOUNDARY;
+                else
+                    state = State.LAST_BOUNDARY;
+            }
+            notifyContent(buffer, false, callback);
+        }
+
+        @Override
+        public void onFailure(Throwable failure)
+        {
+            if (subscription != null)
+                subscription.fail(failure);
+        }
+    }
+
+    private enum State
+    {
+        FIRST_BOUNDARY, HEADERS, CONTENT, MIDDLE_BOUNDARY, LAST_BOUNDARY, COMPLETE
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/OutputStreamContentProvider.java

@@ -72,7 +72,10 @@ import org.eclipse.jetty.util.Callback;
  *     output.write("some content".getBytes());
  * }
  * </pre>
+ *
+ * @deprecated use {@link OutputStreamRequestContent} instead
  */
+@Deprecated
 public class OutputStreamContentProvider implements AsyncContentProvider, Callback, Closeable
 {
     private final DeferredContentProvider deferred = new DeferredContentProvider();

jetty-client/src/main/java/org/eclipse/jetty/client/util/OutputStreamRequestContent.java

@@ -0,0 +1,125 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.concurrent.ExecutionException;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.util.FutureCallback;
+
+/**
+ * <p>A {@link Request.Content} that provides content asynchronously through an {@link OutputStream}
+ * similar to {@link AsyncRequestContent}.</p>
+ * <p>{@link OutputStreamRequestContent} can only be used in conjunction with
+ * {@link Request#send(Response.CompleteListener)} (and not with its blocking counterpart
+ * {@link Request#send()}) because it provides content asynchronously.</p>
+ * <p>Content must be provided by writing to the {@link #getOutputStream() output stream}
+ * that must be {@link OutputStream#close() closed} when all content has been provided.</p>
+ * <p>Example usage:</p>
+ * <pre>
+ * HttpClient httpClient = ...;
+ *
+ * // Use try-with-resources to autoclose the output stream.
+ * OutputStreamRequestContent content = new OutputStreamRequestContent();
+ * try (OutputStream output = content.getOutputStream())
+ * {
+ *     httpClient.newRequest("localhost", 8080)
+ *             .content(content)
+ *             .send(new Response.CompleteListener()
+ *             {
+ *                 &#64;Override
+ *                 public void onComplete(Result result)
+ *                 {
+ *                     // Your logic here
+ *                 }
+ *             });
+ *
+ *     // At a later time...
+ *     output.write("some content".getBytes());
+ *
+ *     // Even later...
+ *     output.write("more content".getBytes());
+ * } // Implicit call to output.close().
+ * </pre>
+ */
+public class OutputStreamRequestContent extends AsyncRequestContent
+{
+    private final AsyncOutputStream output;
+
+    public OutputStreamRequestContent()
+    {
+        this("application/octet-stream");
+    }
+
+    public OutputStreamRequestContent(String contentType)
+    {
+        super(contentType);
+        this.output = new AsyncOutputStream();
+    }
+
+    public OutputStream getOutputStream()
+    {
+        return output;
+    }
+
+    private class AsyncOutputStream extends OutputStream
+    {
+        @Override
+        public void write(int b) throws IOException
+        {
+            write(new byte[]{(byte)b}, 0, 1);
+        }
+
+        @Override
+        public void write(byte[] b, int off, int len) throws IOException
+        {
+            try
+            {
+                FutureCallback callback = new FutureCallback();
+                offer(ByteBuffer.wrap(b, off, len), callback);
+                callback.get();
+            }
+            catch (InterruptedException x)
+            {
+                throw new InterruptedIOException();
+            }
+            catch (ExecutionException x)
+            {
+                throw new IOException(x.getCause());
+            }
+        }
+
+        @Override
+        public void flush() throws IOException
+        {
+            OutputStreamRequestContent.this.flush();
+        }
+
+        @Override
+        public void close()
+        {
+            OutputStreamRequestContent.this.close();
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/PathContentProvider.java

@@ -43,7 +43,10 @@ import org.slf4j.LoggerFactory;
  * If a {@link ByteBufferPool} is provided via {@link #setByteBufferPool(ByteBufferPool)},
  * the buffer will be allocated from that pool, otherwise one buffer will be
  * allocated and used to read the file.</p>
+ *
+ * @deprecated use {@link PathRequestContent} instead.
  */
+@Deprecated
 public class PathContentProvider extends AbstractTypedContentProvider
 {
     private static final Logger LOG = LoggerFactory.getLogger(PathContentProvider.class);

jetty-client/src/main/java/org/eclipse/jetty/client/util/PathRequestContent.java

@@ -0,0 +1,177 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.file.AccessDeniedException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.io.ByteBufferPool;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.IO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>A {@link Request.Content} for files using JDK 7's {@code java.nio.file} APIs.</p>
+ * <p>It is possible to specify, at the constructor, a buffer size used to read
+ * content from the stream, by default 4096 bytes.
+ * If a {@link ByteBufferPool} is provided via {@link #setByteBufferPool(ByteBufferPool)},
+ * the buffer will be allocated from that pool, otherwise one buffer will be
+ * allocated and used to read the file.</p>
+ */
+public class PathRequestContent extends AbstractRequestContent
+{
+    private static final Logger LOG = LoggerFactory.getLogger(PathRequestContent.class);
+
+    private final Path filePath;
+    private final long fileSize;
+    private final int bufferSize;
+    private ByteBufferPool bufferPool;
+    private boolean useDirectByteBuffers = true;
+
+    public PathRequestContent(Path filePath) throws IOException
+    {
+        this(filePath, 4096);
+    }
+
+    public PathRequestContent(Path filePath, int bufferSize) throws IOException
+    {
+        this("application/octet-stream", filePath, bufferSize);
+    }
+
+    public PathRequestContent(String contentType, Path filePath) throws IOException
+    {
+        this(contentType, filePath, 4096);
+    }
+
+    public PathRequestContent(String contentType, Path filePath, int bufferSize) throws IOException
+    {
+        super(contentType);
+        if (!Files.isRegularFile(filePath))
+            throw new NoSuchFileException(filePath.toString());
+        if (!Files.isReadable(filePath))
+            throw new AccessDeniedException(filePath.toString());
+        this.filePath = filePath;
+        this.fileSize = Files.size(filePath);
+        this.bufferSize = bufferSize;
+    }
+
+    @Override
+    public long getLength()
+    {
+        return fileSize;
+    }
+
+    @Override
+    public boolean isReproducible()
+    {
+        return true;
+    }
+
+    public ByteBufferPool getByteBufferPool()
+    {
+        return bufferPool;
+    }
+
+    public void setByteBufferPool(ByteBufferPool byteBufferPool)
+    {
+        this.bufferPool = byteBufferPool;
+    }
+
+    public boolean isUseDirectByteBuffers()
+    {
+        return useDirectByteBuffers;
+    }
+
+    public void setUseDirectByteBuffers(boolean useDirectByteBuffers)
+    {
+        this.useDirectByteBuffers = useDirectByteBuffers;
+    }
+
+    @Override
+    protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
+    {
+        return new SubscriptionImpl(consumer, emitInitialContent);
+    }
+
+    private class SubscriptionImpl extends AbstractSubscription
+    {
+        private ReadableByteChannel channel;
+        private long readTotal;
+
+        private SubscriptionImpl(Consumer consumer, boolean emitInitialContent)
+        {
+            super(consumer, emitInitialContent);
+        }
+
+        @Override
+        protected boolean produceContent(Producer producer) throws IOException
+        {
+            ByteBuffer buffer;
+            boolean last;
+            if (channel == null)
+            {
+                channel = Files.newByteChannel(filePath, StandardOpenOption.READ);
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Opened file {}", filePath);
+            }
+
+            buffer = bufferPool == null
+                ? BufferUtil.allocate(bufferSize, isUseDirectByteBuffers())
+                : bufferPool.acquire(bufferSize, isUseDirectByteBuffers());
+
+            BufferUtil.clearToFill(buffer);
+            int read = channel.read(buffer);
+            BufferUtil.flipToFlush(buffer, 0);
+            if (LOG.isDebugEnabled())
+                LOG.debug("Read {} bytes from {}", read, filePath);
+            if (!channel.isOpen() && read < 0)
+                throw new EOFException("EOF reached for " + filePath);
+
+            if (read > 0)
+                readTotal += read;
+            last = readTotal == fileSize;
+            if (last)
+                IO.close(channel);
+            return producer.produce(buffer, last, Callback.from(() -> release(buffer)));
+        }
+
+        private void release(ByteBuffer buffer)
+        {
+            if (bufferPool != null)
+                bufferPool.release(buffer);
+        }
+
+        @Override
+        public void fail(Throwable failure)
+        {
+            super.fail(failure);
+            IO.close(channel);
+        }
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/util/StringContentProvider.java

@@ -28,7 +28,10 @@ import org.eclipse.jetty.client.api.ContentProvider;
  * <p>
  * It is possible to specify, at the constructor, an encoding used to convert
  * the string into bytes, by default UTF-8.
+ *
+ * @deprecated use {@link StringRequestContent} instead.
  */
+@Deprecated
 public class StringContentProvider extends BytesContentProvider
 {
     public StringContentProvider(String content)

jetty-client/src/main/java/org/eclipse/jetty/client/util/StringRequestContent.java

@@ -0,0 +1,52 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+import org.eclipse.jetty.client.api.Request;
+
+/**
+ * <p>A {@link Request.Content} for strings.</p>
+ * <p>It is possible to specify, at the constructor, an encoding used to convert
+ * the string into bytes, by default UTF-8.</p>
+ */
+public class StringRequestContent extends BytesRequestContent
+{
+    public StringRequestContent(String content)
+    {
+        this("text/plain;charset=UTF-8", content);
+    }
+
+    public StringRequestContent(String content, Charset encoding)
+    {
+        this("text/plain;charset=" + encoding.name(), content, encoding);
+    }
+
+    public StringRequestContent(String contentType, String content)
+    {
+        this(contentType, content, StandardCharsets.UTF_8);
+    }
+
+    public StringRequestContent(String contentType, String content, Charset encoding)
+    {
+        super(contentType, content.getBytes(encoding));
+    }
+}

jetty-client/src/test/java/org/eclipse/jetty/client/ClientConnectionCloseTest.java

@@ -29,8 +29,8 @@ import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.http.HttpConnectionOverHTTP;
-import org.eclipse.jetty.client.util.DeferredContentProvider;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
-import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.client.util.StringRequestContent;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpHeaderValue;
 import org.eclipse.jetty.http.HttpStatus;
@@ -87,7 +87,7 @@ public class ClientConnectionCloseTest extends AbstractHttpClientServerTest
         var request = client.newRequest(host, port)
             .scheme(scenario.getScheme())
             .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString())
-            .content(new StringContentProvider("0"))
+            .body(new StringRequestContent("0"))
             .onRequestSuccess(r ->
             {
                 HttpDestination destination = (HttpDestination)client.resolveDestination(r);
@@ -184,12 +184,12 @@ public class ClientConnectionCloseTest extends AbstractHttpClientServerTest
         String host = "localhost";
         int port = connector.getLocalPort();
 
-        DeferredContentProvider content = new DeferredContentProvider(ByteBuffer.allocate(8));
+        AsyncRequestContent content = new AsyncRequestContent(ByteBuffer.allocate(8));
         CountDownLatch resultLatch = new CountDownLatch(1);
         var request = client.newRequest(host, port)
             .scheme(scenario.getScheme())
             .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString())
-            .content(content)
+            .body(content)
             .idleTimeout(idleTimeout, TimeUnit.MILLISECONDS)
             .onRequestSuccess(r ->
             {

jetty-client/src/test/java/org/eclipse/jetty/client/ConnectionPoolTest.java

@@ -33,7 +33,7 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
-import org.eclipse.jetty.client.util.BytesContentProvider;
+import org.eclipse.jetty.client.util.BytesRequestContent;
 import org.eclipse.jetty.client.util.FutureResponseListener;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpMethod;
@@ -213,7 +213,7 @@ public class ConnectionPoolTest
                 break;
             case POST:
                 request.header(HttpHeader.CONTENT_LENGTH, String.valueOf(contentLength));
-                request.content(new BytesContentProvider(new byte[contentLength]));
+                request.body(new BytesRequestContent(new byte[contentLength]));
                 break;
             default:
                 throw new IllegalStateException();

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientAuthenticationTest.java

@@ -22,9 +22,7 @@ import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.nio.ByteBuffer;
-import java.util.Iterator;
 import java.util.List;
-import java.util.NoSuchElementException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -37,15 +35,15 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jetty.client.api.Authentication;
 import org.eclipse.jetty.client.api.Authentication.HeaderInfo;
 import org.eclipse.jetty.client.api.AuthenticationStore;
-import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Response.Listener;
 import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.client.util.AbstractAuthentication;
+import org.eclipse.jetty.client.util.AbstractRequestContent;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
 import org.eclipse.jetty.client.util.BasicAuthentication;
-import org.eclipse.jetty.client.util.DeferredContentProvider;
 import org.eclipse.jetty.client.util.DigestAuthentication;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpStatus;
@@ -60,6 +58,8 @@ import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
 import org.eclipse.jetty.util.Attributes;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.IO;
 import org.eclipse.jetty.util.URIUtil;
 import org.eclipse.jetty.util.security.Constraint;
@@ -460,7 +460,7 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
 
         CountDownLatch resultLatch = new CountDownLatch(1);
         byte[] data = new byte[]{'h', 'e', 'l', 'l', 'o'};
-        DeferredContentProvider content = new DeferredContentProvider(ByteBuffer.wrap(data))
+        AsyncRequestContent content = new AsyncRequestContent(ByteBuffer.wrap(data))
         {
             @Override
             public boolean isReproducible()
@@ -470,7 +470,7 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
         };
         Request request = client.newRequest(uri)
             .path("/secure")
-            .content(content);
+            .body(content);
         request.send(result ->
         {
             if (result.isSucceeded() && result.getResponse().getStatus() == HttpStatus.UNAUTHORIZED_401)
@@ -527,7 +527,7 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
         authenticationStore.addAuthentication(authentication);
 
         AtomicBoolean fail = new AtomicBoolean(true);
-        GeneratingContentProvider content = new GeneratingContentProvider(index ->
+        GeneratingRequestContent content = new GeneratingRequestContent(index ->
         {
             switch (index)
             {
@@ -546,9 +546,8 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
                         catch (InterruptedException ignored)
                         {
                         }
-
                         // Trigger request failure.
-                        throw new RuntimeException();
+                        throw new RuntimeException("explicitly_thrown_by_test");
                     }
                     else
                     {
@@ -563,7 +562,7 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
         client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .path("/secure")
-            .content(content)
+            .body(content)
             .onResponseSuccess(r -> authLatch.countDown())
             .send(result ->
             {
@@ -803,23 +802,16 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
         assertEquals(headerInfo.getParameter("nonce"), "1523430383=");
     }
 
-    private static class GeneratingContentProvider implements ContentProvider
+    private static class GeneratingRequestContent extends AbstractRequestContent
     {
-        private static final ByteBuffer DONE = ByteBuffer.allocate(0);
-
         private final IntFunction<ByteBuffer> generator;
 
-        private GeneratingContentProvider(IntFunction<ByteBuffer> generator)
+        private GeneratingRequestContent(IntFunction<ByteBuffer> generator)
         {
+            super("application/octet-stream");
             this.generator = generator;
         }
 
-        @Override
-        public long getLength()
-        {
-            return -1;
-        }
-
         @Override
         public boolean isReproducible()
         {
@@ -827,36 +819,32 @@ public class HttpClientAuthenticationTest extends AbstractHttpClientServerTest
         }
 
         @Override
-        public Iterator<ByteBuffer> iterator()
+        protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
         {
-            return new Iterator<ByteBuffer>()
+            return new SubscriptionImpl(consumer, emitInitialContent);
+        }
+
+        private class SubscriptionImpl extends AbstractSubscription
+        {
+            private int index;
+
+            public SubscriptionImpl(Consumer consumer, boolean emitInitialContent)
             {
-                private int index;
+                super(consumer, emitInitialContent);
-                public ByteBuffer current;
+            }
 
-                @Override
+            @Override
-                @SuppressWarnings("ReferenceEquality")
+            protected boolean produceContent(Producer producer)
-                public boolean hasNext()
+            {
+                ByteBuffer buffer = generator.apply(index++);
+                boolean last = false;
+                if (buffer == null)
                 {
-                    if (current == null)
+                    buffer = BufferUtil.EMPTY_BUFFER;
-                    {
+                    last = true;
-                        current = generator.apply(index++);
-                        if (current == null)
-                            current = DONE;
-                    }
-                    return current != DONE;
                 }
-
+                return producer.produce(buffer, last, Callback.NOOP);
-                @Override
+            }
-                public ByteBuffer next()
-                {
-                    ByteBuffer result = current;
-                    current = null;
-                    if (result == null)
-                        throw new NoSuchElementException();
-                    return result;
-                }
-            };
         }
     }
 }

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientFailureTest.java

@@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
 import org.eclipse.jetty.client.http.HttpConnectionOverHTTP;
-import org.eclipse.jetty.client.util.DeferredContentProvider;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.Server;
@@ -116,16 +116,16 @@ public class HttpClientFailureTest
         });
         client.start();
 
-        final CountDownLatch commitLatch = new CountDownLatch(1);
+        CountDownLatch commitLatch = new CountDownLatch(1);
-        final CountDownLatch completeLatch = new CountDownLatch(1);
+        CountDownLatch completeLatch = new CountDownLatch(1);
-        DeferredContentProvider content = new DeferredContentProvider();
+        AsyncRequestContent content = new AsyncRequestContent();
         client.newRequest("localhost", connector.getLocalPort())
             .onRequestCommit(request ->
             {
                 connectionRef.get().getEndPoint().close();
                 commitLatch.countDown();
             })
-            .content(content)
+            .body(content)
             .idleTimeout(2, TimeUnit.SECONDS)
             .send(result ->
             {

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientRedirectTest.java

@@ -36,7 +36,7 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
-import org.eclipse.jetty.client.util.ByteBufferContentProvider;
+import org.eclipse.jetty.client.util.ByteBufferRequestContent;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpStatus;
@@ -153,7 +153,7 @@ public class HttpClientRedirectTest extends AbstractHttpClientServerTest
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
             .path("/307/localhost/done")
-            .content(new ByteBufferContentProvider(ByteBuffer.wrap(data)))
+            .body(new ByteBufferRequestContent(ByteBuffer.wrap(data)))
             .timeout(5, TimeUnit.SECONDS)
             .send();
         assertNotNull(response);

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientSynchronizationTest.java

@@ -47,20 +47,22 @@ public class HttpClientSynchronizationTest extends AbstractHttpClientServerTest
         server.stop();
 
         int count = 10;
-        final CountDownLatch latch = new CountDownLatch(count);
+        CountDownLatch latch = new CountDownLatch(count);
         for (int i = 0; i < count; ++i)
         {
             Request request = client.newRequest("localhost", port)
-                .scheme(scenario.getScheme());
+                .scheme(scenario.getScheme())
+                .path("/" + i);
 
-            synchronized (this)
+            Object lock = this;
+            synchronized (lock)
             {
                 request.send(new Response.Listener.Adapter()
                 {
                     @Override
                     public void onFailure(Response response, Throwable failure)
                     {
-                        synchronized (HttpClientSynchronizationTest.this)
+                        synchronized (lock)
                         {
                             assertThat(failure, Matchers.instanceOf(ConnectException.class));
                             latch.countDown();
@@ -80,20 +82,22 @@ public class HttpClientSynchronizationTest extends AbstractHttpClientServerTest
         start(scenario, new EmptyServerHandler());
 
         int count = 10;
-        final CountDownLatch latch = new CountDownLatch(count);
+        CountDownLatch latch = new CountDownLatch(count);
         for (int i = 0; i < count; ++i)
         {
             Request request = client.newRequest("localhost", connector.getLocalPort())
-                .scheme(scenario.getScheme());
+                .scheme(scenario.getScheme())
+                .path("/" + i);
 
-            synchronized (this)
+            Object lock = this;
+            synchronized (lock)
             {
                 request.send(new Response.Listener.Adapter()
                 {
                     @Override
                     public void onComplete(Result result)
                     {
-                        synchronized (HttpClientSynchronizationTest.this)
+                        synchronized (lock)
                         {
                             assertFalse(result.isFailed());
                             latch.countDown();

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java

@@ -38,10 +38,8 @@ import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.NoSuchElementException;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Exchanger;
@@ -59,7 +57,6 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.client.api.Connection;
-import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Destination;
 import org.eclipse.jetty.client.api.Request;
@@ -67,11 +64,12 @@ import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
 import org.eclipse.jetty.client.http.HttpConnectionOverHTTP;
+import org.eclipse.jetty.client.util.AbstractRequestContent;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
 import org.eclipse.jetty.client.util.BufferingResponseListener;
-import org.eclipse.jetty.client.util.BytesContentProvider;
+import org.eclipse.jetty.client.util.BytesRequestContent;
-import org.eclipse.jetty.client.util.DeferredContentProvider;
 import org.eclipse.jetty.client.util.FutureResponseListener;
-import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.client.util.StringRequestContent;
 import org.eclipse.jetty.http.BadMessageException;
 import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpHeader;
@@ -231,7 +229,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         });
 
         String value1 = "\u20AC";
-        String paramValue1 = URLEncoder.encode(value1, "UTF-8");
+        String paramValue1 = URLEncoder.encode(value1, StandardCharsets.UTF_8);
         String query = paramName1 + "=" + paramValue1 + "&" + paramName2;
         ContentResponse response = client.GET(scenario.getScheme() + "://localhost:" + connector.getLocalPort() + "/?" + query);
 
@@ -268,9 +266,9 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         String value11 = "\u20AC";
         String value12 = "\u20AA";
         String value2 = "&";
-        String paramValue11 = URLEncoder.encode(value11, "UTF-8");
+        String paramValue11 = URLEncoder.encode(value11, StandardCharsets.UTF_8);
-        String paramValue12 = URLEncoder.encode(value12, "UTF-8");
+        String paramValue12 = URLEncoder.encode(value12, StandardCharsets.UTF_8);
-        String paramValue2 = URLEncoder.encode(value2, "UTF-8");
+        String paramValue2 = URLEncoder.encode(value2, StandardCharsets.UTF_8);
         String query = paramName1 + "=" + paramValue11 + "&" + paramName1 + "=" + paramValue12 + "&" + paramName2 + "=" + paramValue2;
         ContentResponse response = client.GET(scenario.getScheme() + "://localhost:" + connector.getLocalPort() + "/?" + query);
 
@@ -318,7 +316,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
     {
         String paramName = "a";
         String paramValue = "\u20AC";
-        String encodedParamValue = URLEncoder.encode(paramValue, "UTF-8");
+        String encodedParamValue = URLEncoder.encode(paramValue, StandardCharsets.UTF_8);
         start(scenario, new AbstractHandler()
         {
             @Override
@@ -372,7 +370,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
 
         ContentResponse response = client.POST(scenario.getScheme() + "://localhost:" + connector.getLocalPort() + "/?b=1")
             .param(paramName, paramValue)
-            .content(new BytesContentProvider(content))
+            .body(new BytesRequestContent(content))
             .timeout(5, TimeUnit.SECONDS)
             .send();
 
@@ -404,7 +402,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
                 if (!Arrays.equals(content, bytes))
                     request.abort(new Exception());
             })
-            .content(new BytesContentProvider(content))
+            .body(new BytesRequestContent(content))
             .timeout(5, TimeUnit.SECONDS)
             .send();
 
@@ -435,7 +433,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
                 buffer.get(bytes);
                 assertEquals(bytes[0], progress.getAndIncrement());
             })
-            .content(new BytesContentProvider(new byte[]{0}, new byte[]{1}, new byte[]{2}, new byte[]{3}, new byte[]{4}))
+            .body(new BytesRequestContent(new byte[]{0}, new byte[]{1}, new byte[]{2}, new byte[]{3}, new byte[]{4}))
             .timeout(5, TimeUnit.SECONDS)
             .send();
 
@@ -511,7 +509,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
 
         client.setMaxConnectionsPerDestination(1);
 
-        try (StacklessLogging stackless = new StacklessLogging(org.eclipse.jetty.server.HttpChannel.class))
+        try (StacklessLogging ignored = new StacklessLogging(org.eclipse.jetty.server.HttpChannel.class))
         {
             CountDownLatch latch = new CountDownLatch(2);
             client.newRequest("localhost", connector.getLocalPort())
@@ -630,36 +628,23 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         CountDownLatch latch = new CountDownLatch(1);
         client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
-            // The second ByteBuffer set to null will throw an exception
+            .body(new AbstractRequestContent("application/octet-stream")
-            .content(new ContentProvider()
             {
                 @Override
-                public long getLength()
+                protected Subscription newSubscription(Consumer consumer, boolean emitInitialContent)
                 {
-                    return -1;
+                    return new AbstractSubscription(consumer, emitInitialContent)
-                }
-
-                @Override
-                public Iterator<ByteBuffer> iterator()
-                {
-                    return new Iterator<>()
                     {
-                        @Override
+                        private int count;
-                        public boolean hasNext()
-                        {
-                            return true;
-                        }
 
                         @Override
-                        public ByteBuffer next()
+                        protected boolean produceContent(Producer producer) throws Exception
                         {
-                            throw new NoSuchElementException("explicitly_thrown_by_test");
+                            if (count == 2)
-                        }
+                                throw new IOException("explicitly_thrown_by_test");
-
+                            ByteBuffer buffer = BufferUtil.allocate(512);
-                        @Override
+                            ++count;
-                        public void remove()
+                            return producer.produce(buffer, false, Callback.NOOP);
-                        {
-                            throw new UnsupportedOperationException();
                         }
                     };
                 }
@@ -1244,7 +1229,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
             public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException
             {
                 // Send the headers at this point, then write the content
-                byte[] content = "TEST".getBytes("UTF-8");
+                byte[] content = "TEST".getBytes(StandardCharsets.UTF_8);
                 response.setContentLength(content.length);
                 response.flushBuffer();
                 response.getOutputStream().write(content);
@@ -1413,11 +1398,11 @@ public class HttpClientTest extends AbstractHttpClientServerTest
             }
         });
 
-        DeferredContentProvider content = new DeferredContentProvider(ByteBuffer.wrap(new byte[]{0}));
+        AsyncRequestContent content = new AsyncRequestContent(ByteBuffer.wrap(new byte[]{0}));
         Request request = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .version(version)
-            .content(content);
+            .body(content);
         FutureResponseListener listener = new FutureResponseListener(request);
         request.send(listener);
         // Wait some time to simulate a slow request.
@@ -1530,7 +1515,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         client = new HttpClient(new HttpClientTransportOverHTTP(clientConnector)
         {
             @Override
-            public org.eclipse.jetty.io.Connection newConnection(EndPoint endPoint, Map<String, Object> context) throws IOException
+            public org.eclipse.jetty.io.Connection newConnection(EndPoint endPoint, Map<String, Object> context)
             {
                 return new HttpConnectionOverHTTP(endPoint, context)
                 {
@@ -1658,7 +1643,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         assertCopyRequest(client.newRequest("http://example.com/some/url")
             .method(HttpMethod.HEAD)
             .version(HttpVersion.HTTP_2)
-            .content(new StringContentProvider("some string"))
+            .body(new StringRequestContent("some string"))
             .timeout(321, TimeUnit.SECONDS)
             .idleTimeout(2221, TimeUnit.SECONDS)
             .followRedirects(true)
@@ -1668,7 +1653,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         assertCopyRequest(client.newRequest("https://example.com")
             .method(HttpMethod.POST)
             .version(HttpVersion.HTTP_1_0)
-            .content(new StringContentProvider("some other string"))
+            .body(new StringRequestContent("some other string"))
             .timeout(123231, TimeUnit.SECONDS)
             .idleTimeout(232342, TimeUnit.SECONDS)
             .followRedirects(false)
@@ -1797,7 +1782,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         start(scenario, new AbstractHandler()
         {
             @Override
-            public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            public void handle(String target, org.eclipse.jetty.server.Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException
             {
                 baseRequest.setHandled(true);
                 ServletOutputStream output = response.getOutputStream();
@@ -1845,7 +1830,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         assertEquals(original.getURI(), copy.getURI());
         assertEquals(original.getMethod(), copy.getMethod());
         assertEquals(original.getVersion(), copy.getVersion());
-        assertEquals(original.getContent(), copy.getContent());
+        assertEquals(original.getBody(), copy.getBody());
         assertEquals(original.getIdleTimeout(), copy.getIdleTimeout());
         assertEquals(original.getTimeout(), copy.getTimeout());
         assertEquals(original.isFollowRedirects(), copy.isFollowRedirects());
@@ -1910,7 +1895,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
                 .scheme(scheme)
                 .method("POST")
                 .param("attempt", String.valueOf(retries))
-                .content(new StringContentProvider("0123456789ABCDEF"))
+                .body(new StringRequestContent("0123456789ABCDEF"))
                 .send(this);
         }
     }

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientUploadDuringServerShutdownTest.java

@@ -32,7 +32,7 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jetty.client.http.HttpChannelOverHTTP;
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
 import org.eclipse.jetty.client.http.HttpConnectionOverHTTP;
-import org.eclipse.jetty.client.util.BytesContentProvider;
+import org.eclipse.jetty.client.util.BytesRequestContent;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.Server;
@@ -116,7 +116,7 @@ public class HttpClientUploadDuringServerShutdownTest
                 {
                     int length = 16 * 1024 * 1024 + random.nextInt(16 * 1024 * 1024);
                     client.newRequest("localhost", 8888)
-                            .content(new BytesContentProvider(new byte[length]))
+                            .body(new BytesRequestContent(new byte[length]))
                             .send(result -> latch.countDown());
                     long sleep = 1 + random.nextInt(10);
                     TimeUnit.MILLISECONDS.sleep(sleep);

jetty-client/src/test/java/org/eclipse/jetty/client/HttpConnectionLifecycleTest.java

@@ -32,7 +32,7 @@ import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
-import org.eclipse.jetty.client.util.ByteBufferContentProvider;
+import org.eclipse.jetty.client.util.ByteBufferRequestContent;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.logging.StacklessLogging;
@@ -411,7 +411,7 @@ public class HttpConnectionLifecycleTest extends AbstractHttpClientServerTest
             CountDownLatch latch = new CountDownLatch(1);
             ByteBuffer buffer = ByteBuffer.allocate(16 * 1024 * 1024);
             Arrays.fill(buffer.array(), (byte)'x');
-            request.content(new ByteBufferContentProvider(buffer))
+            request.body(new ByteBufferRequestContent(buffer))
                 .send(new Response.Listener.Adapter()
                 {
                     @Override

jetty-client/src/test/java/org/eclipse/jetty/client/HttpRequestAbortTest.java

@@ -32,7 +32,7 @@ import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Result;
-import org.eclipse.jetty.client.util.ByteBufferContentProvider;
+import org.eclipse.jetty.client.util.ByteBufferRequestContent;
 import org.eclipse.jetty.logging.StacklessLogging;
 import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.eclipse.jetty.util.IO;
@@ -268,7 +268,7 @@ public class HttpRequestAbortTest extends AbstractHttpClientServerTest
             {
                 aborted.set(r.abort(cause));
                 latch.countDown();
-            }).content(new ByteBufferContentProvider(ByteBuffer.wrap(new byte[]{0}), ByteBuffer.wrap(new byte[]{1}))
+            }).body(new ByteBufferRequestContent(ByteBuffer.wrap(new byte[]{0}), ByteBuffer.wrap(new byte[]{1}))
             {
                 @Override
                 public long getLength()
@@ -323,7 +323,7 @@ public class HttpRequestAbortTest extends AbstractHttpClientServerTest
                 {
                     aborted.set(r.abort(cause));
                     latch.countDown();
-                }).content(new ByteBufferContentProvider(ByteBuffer.wrap(new byte[]{0}), ByteBuffer.wrap(new byte[]{1}))
+                }).body(new ByteBufferRequestContent(ByteBuffer.wrap(new byte[]{0}), ByteBuffer.wrap(new byte[]{1}))
                 {
                     @Override
                     public long getLength()

jetty-client/src/test/java/org/eclipse/jetty/client/HttpResponseAbortTest.java

@@ -24,11 +24,10 @@ import java.nio.ByteBuffer;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.eclipse.jetty.client.util.DeferredContentProvider;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -104,7 +103,7 @@ public class HttpResponseAbortTest extends AbstractHttpClientServerTest
         start(scenario, new AbstractHandler()
         {
             @Override
-            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
             {
                 try
                 {
@@ -141,7 +140,7 @@ public class HttpResponseAbortTest extends AbstractHttpClientServerTest
         start(scenario, new AbstractHandler()
         {
             @Override
-            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
             {
                 try
                 {
@@ -159,18 +158,18 @@ public class HttpResponseAbortTest extends AbstractHttpClientServerTest
             }
         });
 
-        final DeferredContentProvider contentProvider = new DeferredContentProvider(ByteBuffer.allocate(1));
+        AsyncRequestContent requestContent = new AsyncRequestContent(ByteBuffer.allocate(1));
-        final AtomicInteger completes = new AtomicInteger();
+        AtomicInteger completes = new AtomicInteger();
-        final CountDownLatch completeLatch = new CountDownLatch(1);
+        CountDownLatch completeLatch = new CountDownLatch(1);
         client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
-            .content(contentProvider)
+            .body(requestContent)
             .onResponseContent((response, content) ->
             {
                 try
                 {
                     response.abort(new Exception());
-                    contentProvider.close();
+                    requestContent.close();
                     // Delay to let the request side to finish its processing.
                     Thread.sleep(1000);
                 }

jetty-client/src/test/java/org/eclipse/jetty/client/NetworkTrafficListenerTest.java

@@ -0,0 +1,525 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.Socket;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectableChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
+import org.eclipse.jetty.client.util.FormRequestContent;
+import org.eclipse.jetty.client.util.StringRequestContent;
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpHeaderValue;
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.io.ClientConnector;
+import org.eclipse.jetty.io.EndPoint;
+import org.eclipse.jetty.io.ManagedSelector;
+import org.eclipse.jetty.io.NetworkTrafficListener;
+import org.eclipse.jetty.io.NetworkTrafficSocketChannelEndPoint;
+import org.eclipse.jetty.io.SelectorManager;
+import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.NetworkTrafficServerConnector;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Fields;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class NetworkTrafficListenerTest
+{
+    private static final String END_OF_CONTENT = "~";
+
+    private Server server;
+    private NetworkTrafficServerConnector connector;
+    private NetworkTrafficHttpClient client;
+
+    private void start(Handler handler) throws Exception
+    {
+        startServer(handler);
+        startClient();
+    }
+
+    private void startServer(Handler handler) throws Exception
+    {
+        server = new Server();
+        connector = new NetworkTrafficServerConnector(server);
+        connector.getConnectionFactory(HttpConfiguration.ConnectionFactory.class).getHttpConfiguration().setSendDateHeader(false);
+        connector.getConnectionFactory(HttpConfiguration.ConnectionFactory.class).getHttpConfiguration().setSendServerVersion(false);
+        server.addConnector(connector);
+        server.setHandler(handler);
+        server.start();
+    }
+
+    private void startClient() throws Exception
+    {
+        client = new NetworkTrafficHttpClient(new AtomicReference<>());
+        client.start();
+    }
+
+    @AfterEach
+    public void dispose() throws Exception
+    {
+        if (client != null)
+            client.stop();
+        if (server != null)
+            server.stop();
+    }
+
+    @Test
+    public void testOpenedClosedAreInvoked() throws Exception
+    {
+        startServer(null);
+
+        CountDownLatch openedLatch = new CountDownLatch(1);
+        CountDownLatch closedLatch = new CountDownLatch(1);
+        connector.setNetworkTrafficListener(new NetworkTrafficListener()
+        {
+            public volatile Socket socket;
+
+            @Override
+            public void opened(Socket socket)
+            {
+                this.socket = socket;
+                openedLatch.countDown();
+            }
+
+            @Override
+            public void closed(Socket socket)
+            {
+                if (this.socket == socket)
+                    closedLatch.countDown();
+            }
+        });
+        int port = connector.getLocalPort();
+
+        // Connect to the server
+        try (Socket ignored = new Socket("localhost", port))
+        {
+            assertTrue(openedLatch.await(10, TimeUnit.SECONDS));
+        }
+        assertTrue(closedLatch.await(10, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testTrafficWithNoResponseContentOnNonPersistentConnection() throws Exception
+    {
+        start(new AbstractHandler()
+        {
+            @Override
+            public void handle(String uri, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse)
+            {
+                request.setHandled(true);
+            }
+        });
+
+        AtomicReference<String> serverIncoming = new AtomicReference<>("");
+        CountDownLatch serverIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> serverOutgoing = new AtomicReference<>("");
+        CountDownLatch serverOutgoingLatch = new CountDownLatch(1);
+        connector.setNetworkTrafficListener(new NetworkTrafficListener()
+        {
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                serverIncoming.set(serverIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverIncomingLatch.countDown();
+            }
+
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                serverOutgoing.set(serverOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverOutgoingLatch.countDown();
+            }
+        });
+
+        AtomicReference<String> clientIncoming = new AtomicReference<>("");
+        CountDownLatch clientIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> clientOutgoing = new AtomicReference<>("");
+        CountDownLatch clientOutgoingLatch = new CountDownLatch(1);
+        client.listener.set(new NetworkTrafficListener()
+        {
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                clientOutgoing.set(clientOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientOutgoingLatch.countDown();
+            }
+
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                clientIncoming.set(clientIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientIncomingLatch.countDown();
+            }
+        });
+
+        ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
+            .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString())
+            .send();
+        assertEquals(HttpStatus.OK_200, response.getStatus());
+
+        assertTrue(clientOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(clientIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertEquals(clientOutgoing.get(), serverIncoming.get());
+        assertEquals(serverOutgoing.get(), clientIncoming.get());
+    }
+
+    @Test
+    public void testTrafficWithResponseContentOnPersistentConnection() throws Exception
+    {
+        String responseContent = "response_content" + END_OF_CONTENT;
+        start(new AbstractHandler()
+        {
+            @Override
+            public void handle(String uri, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException
+            {
+                request.setHandled(true);
+                ServletOutputStream output = servletResponse.getOutputStream();
+                output.write(responseContent.getBytes(StandardCharsets.UTF_8));
+            }
+        });
+
+        AtomicReference<String> serverIncoming = new AtomicReference<>("");
+        CountDownLatch serverIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> serverOutgoing = new AtomicReference<>("");
+        CountDownLatch serverOutgoingLatch = new CountDownLatch(1);
+        connector.setNetworkTrafficListener(new NetworkTrafficListener()
+        {
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                serverIncoming.set(serverIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverIncomingLatch.countDown();
+            }
+
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                serverOutgoing.set(serverOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverOutgoingLatch.countDown();
+            }
+        });
+
+        AtomicReference<String> clientIncoming = new AtomicReference<>("");
+        CountDownLatch clientIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> clientOutgoing = new AtomicReference<>("");
+        CountDownLatch clientOutgoingLatch = new CountDownLatch(1);
+        client.listener.set(new NetworkTrafficListener()
+        {
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                clientOutgoing.set(clientOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientOutgoingLatch.countDown();
+            }
+
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                clientIncoming.set(clientIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientIncomingLatch.countDown();
+            }
+        });
+
+        ContentResponse response = client.newRequest("localhost", connector.getLocalPort()).send();
+        assertEquals(HttpStatus.OK_200, response.getStatus());
+        assertEquals(responseContent, response.getContentAsString());
+
+        assertTrue(clientOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(clientIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertEquals(clientOutgoing.get(), serverIncoming.get());
+        assertEquals(serverOutgoing.get(), clientIncoming.get());
+    }
+
+    @Test
+    public void testTrafficWithResponseContentChunkedOnPersistentConnection() throws Exception
+    {
+        String responseContent = "response_content";
+        String responseChunk1 = responseContent.substring(0, responseContent.length() / 2);
+        String responseChunk2 = responseContent.substring(responseContent.length() / 2);
+        start(new AbstractHandler()
+        {
+            @Override
+            public void handle(String uri, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException
+            {
+                request.setHandled(true);
+                ServletOutputStream output = servletResponse.getOutputStream();
+                output.write(responseChunk1.getBytes(StandardCharsets.UTF_8));
+                output.flush();
+                output.write(responseChunk2.getBytes(StandardCharsets.UTF_8));
+                output.flush();
+            }
+        });
+
+        AtomicReference<String> serverIncoming = new AtomicReference<>("");
+        CountDownLatch serverIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> serverOutgoing = new AtomicReference<>("");
+        CountDownLatch serverOutgoingLatch = new CountDownLatch(1);
+        connector.setNetworkTrafficListener(new NetworkTrafficListener()
+        {
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                serverIncoming.set(serverIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverIncomingLatch.countDown();
+            }
+
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                serverOutgoing.set(serverOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                if (serverOutgoing.get().endsWith("\r\n0\r\n\r\n"))
+                    serverOutgoingLatch.countDown();
+            }
+        });
+
+        AtomicReference<String> clientIncoming = new AtomicReference<>("");
+        CountDownLatch clientIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> clientOutgoing = new AtomicReference<>("");
+        CountDownLatch clientOutgoingLatch = new CountDownLatch(1);
+        client.listener.set(new NetworkTrafficListener()
+        {
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                clientOutgoing.set(clientOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientOutgoingLatch.countDown();
+            }
+
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                clientIncoming.set(clientIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                if (clientIncoming.get().endsWith("\r\n0\r\n\r\n"))
+                    clientIncomingLatch.countDown();
+            }
+        });
+
+        ContentResponse response = client.newRequest("localhost", connector.getLocalPort()).send();
+        assertEquals(HttpStatus.OK_200, response.getStatus());
+
+        assertTrue(clientOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(clientIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertEquals(clientOutgoing.get(), serverIncoming.get());
+        assertEquals(serverOutgoing.get(), clientIncoming.get());
+    }
+
+    @Test
+    public void testTrafficWithRequestContentWithResponseRedirectOnPersistentConnection() throws Exception
+    {
+        String location = "/redirect";
+        start(new AbstractHandler()
+        {
+            @Override
+            public void handle(String uri, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException
+            {
+                request.setHandled(true);
+                servletResponse.sendRedirect(location);
+            }
+        });
+
+        AtomicReference<String> serverIncoming = new AtomicReference<>("");
+        CountDownLatch serverIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> serverOutgoing = new AtomicReference<>("");
+        CountDownLatch serverOutgoingLatch = new CountDownLatch(1);
+        connector.setNetworkTrafficListener(new NetworkTrafficListener()
+        {
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                serverIncoming.set(serverIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverIncomingLatch.countDown();
+            }
+
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                serverOutgoing.set(serverOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverOutgoingLatch.countDown();
+            }
+        });
+
+        AtomicReference<String> clientIncoming = new AtomicReference<>("");
+        CountDownLatch clientIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> clientOutgoing = new AtomicReference<>("");
+        CountDownLatch clientOutgoingLatch = new CountDownLatch(1);
+        client.listener.set(new NetworkTrafficListener()
+        {
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                clientOutgoing.set(clientOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientOutgoingLatch.countDown();
+            }
+
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                clientIncoming.set(clientIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientIncomingLatch.countDown();
+            }
+        });
+
+        client.setFollowRedirects(false);
+        Fields fields = new Fields();
+        fields.put("a", "1");
+        fields.put("b", "2");
+        ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
+            .body(new FormRequestContent(fields))
+            .send();
+        assertEquals(HttpStatus.FOUND_302, response.getStatus());
+
+        assertTrue(clientOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(serverOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(clientIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertEquals(clientOutgoing.get(), serverIncoming.get());
+        assertEquals(serverOutgoing.get(), clientIncoming.get());
+    }
+
+    @Test
+    public void testTrafficWithBigRequestContentOnPersistentConnection() throws Exception
+    {
+        start(new AbstractHandler()
+        {
+            @Override
+            public void handle(String uri, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException
+            {
+                // Read and discard the request body to make the test more
+                // reliable, otherwise there is a race between request body
+                // upload and response download
+                InputStream input = servletRequest.getInputStream();
+                byte[] buffer = new byte[4096];
+                while (true)
+                {
+                    int read = input.read(buffer);
+                    if (read < 0)
+                        break;
+                }
+                request.setHandled(true);
+            }
+        });
+
+        AtomicReference<String> serverIncoming = new AtomicReference<>("");
+        AtomicReference<String> serverOutgoing = new AtomicReference<>("");
+        CountDownLatch serverOutgoingLatch = new CountDownLatch(1);
+        connector.setNetworkTrafficListener(new NetworkTrafficListener()
+        {
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                serverIncoming.set(serverIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+            }
+
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                serverOutgoing.set(serverOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                serverOutgoingLatch.countDown();
+            }
+        });
+
+        AtomicReference<String> clientIncoming = new AtomicReference<>("");
+        CountDownLatch clientIncomingLatch = new CountDownLatch(1);
+        AtomicReference<String> clientOutgoing = new AtomicReference<>("");
+        client.listener.set(new NetworkTrafficListener()
+        {
+            @Override
+            public void outgoing(Socket socket, ByteBuffer bytes)
+            {
+                clientOutgoing.set(clientOutgoing.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+            }
+
+            @Override
+            public void incoming(Socket socket, ByteBuffer bytes)
+            {
+                clientIncoming.set(clientIncoming.get() + BufferUtil.toString(bytes, StandardCharsets.UTF_8));
+                clientIncomingLatch.countDown();
+            }
+        });
+
+        // Generate a large request content.
+        String requestContent = "0123456789ABCDEF";
+        for (int i = 0; i < 16; ++i)
+        {
+            requestContent += requestContent;
+        }
+
+        ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
+            .body(new StringRequestContent(requestContent))
+            .send();
+        assertEquals(HttpStatus.OK_200, response.getStatus());
+
+        assertTrue(serverOutgoingLatch.await(1, TimeUnit.SECONDS));
+        assertTrue(clientIncomingLatch.await(1, TimeUnit.SECONDS));
+        assertEquals(clientOutgoing.get(), serverIncoming.get());
+        assertTrue(clientOutgoing.get().length() > requestContent.length());
+        assertEquals(serverOutgoing.get(), clientIncoming.get());
+    }
+
+    private static class NetworkTrafficHttpClient extends HttpClient
+    {
+        private final AtomicReference<NetworkTrafficListener> listener;
+
+        private NetworkTrafficHttpClient(AtomicReference<NetworkTrafficListener> listener)
+        {
+            super(new HttpClientTransportOverHTTP(new ClientConnector()
+            {
+                @Override
+                protected SelectorManager newSelectorManager()
+                {
+                    return new ClientSelectorManager(getExecutor(), getScheduler(), getSelectors())
+                    {
+                        @Override
+                        protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey selectionKey)
+                        {
+                            return new NetworkTrafficSocketChannelEndPoint(channel, selector, selectionKey, getScheduler(), getIdleTimeout().toMillis(), listener.get());
+                        }
+                    };
+                }
+            }));
+            this.listener = listener;
+        }
+    }
+}

jetty-client/src/test/java/org/eclipse/jetty/client/api/Usage.java

@@ -32,12 +32,12 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
 import org.eclipse.jetty.client.util.BasicAuthentication;
-import org.eclipse.jetty.client.util.DeferredContentProvider;
 import org.eclipse.jetty.client.util.FutureResponseListener;
-import org.eclipse.jetty.client.util.InputStreamContentProvider;
+import org.eclipse.jetty.client.util.InputStreamRequestContent;
 import org.eclipse.jetty.client.util.InputStreamResponseListener;
-import org.eclipse.jetty.client.util.OutputStreamContentProvider;
+import org.eclipse.jetty.client.util.OutputStreamRequestContent;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.util.FuturePromise;
@@ -101,16 +101,12 @@ public class Usage
 
         client.newRequest("localhost", 8080)
             // Send asynchronously
-            .send(new Response.CompleteListener()
+            .send(result ->
             {
-                @Override
+                if (result.isSucceeded())
-                public void onComplete(Result result)
                 {
-                    if (result.isSucceeded())
+                    responseRef.set(result.getResponse());
-                    {
+                    latch.countDown();
-                        responseRef.set(result.getResponse());
-                        latch.countDown();
-                    }
                 }
             });
 
@@ -278,7 +274,7 @@ public class Usage
 
         ContentResponse response = client.newRequest("localhost", 8080)
             // Provide the content as InputStream
-            .content(new InputStreamContentProvider(input))
+            .body(new InputStreamRequestContent(input))
             .send();
 
         assertEquals(200, response.getStatus());
@@ -290,11 +286,11 @@ public class Usage
         HttpClient client = new HttpClient();
         client.start();
 
-        OutputStreamContentProvider content = new OutputStreamContentProvider();
+        OutputStreamRequestContent content = new OutputStreamRequestContent();
         try (OutputStream output = content.getOutputStream())
         {
             client.newRequest("localhost", 8080)
-                    .content(content)
+                    .body(content)
                     .send(result -> assertEquals(200, result.getResponse().getStatus()));
 
             output.write(new byte[1024]);
@@ -308,15 +304,15 @@ public class Usage
     public void testProxyUsage() throws Exception
     {
         // In proxies, we receive the headers but not the content, so we must be able to send the request with
-        // a lazy content provider that does not block request.send(...)
+        // a lazy request content that does not block request.send(...)
 
         HttpClient client = new HttpClient();
         client.start();
 
-        final AtomicBoolean sendContent = new AtomicBoolean(true);
+        AtomicBoolean sendContent = new AtomicBoolean(true);
-        DeferredContentProvider async = new DeferredContentProvider(ByteBuffer.wrap(new byte[]{0, 1, 2}));
+        AsyncRequestContent async = new AsyncRequestContent(ByteBuffer.wrap(new byte[]{0, 1, 2}));
         client.newRequest("localhost", 8080)
-            .content(async)
+            .body(async)
             .send(new Response.Listener.Adapter()
             {
                 @Override

jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpSenderOverHTTPTest.java

@@ -33,7 +33,7 @@ import org.eclipse.jetty.client.api.Connection;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
-import org.eclipse.jetty.client.util.ByteBufferContentProvider;
+import org.eclipse.jetty.client.util.ByteBufferRequestContent;
 import org.eclipse.jetty.io.ByteArrayEndPoint;
 import org.eclipse.jetty.util.Promise;
 import org.hamcrest.Matchers;
@@ -201,7 +201,7 @@ public class HttpSenderOverHTTPTest
         HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<Connection>());
         Request request = client.newRequest(URI.create("http://localhost/"));
         String content = "abcdef";
-        request.content(new ByteBufferContentProvider(ByteBuffer.wrap(content.getBytes(StandardCharsets.UTF_8))));
+        request.body(new ByteBufferRequestContent(ByteBuffer.wrap(content.getBytes(StandardCharsets.UTF_8))));
         final CountDownLatch headersLatch = new CountDownLatch(1);
         final CountDownLatch successLatch = new CountDownLatch(1);
         request.listener(new Request.Listener.Adapter()
@@ -237,7 +237,7 @@ public class HttpSenderOverHTTPTest
         Request request = client.newRequest(URI.create("http://localhost/"));
         String content1 = "0123456789";
         String content2 = "abcdef";
-        request.content(new ByteBufferContentProvider(ByteBuffer.wrap(content1.getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(content2.getBytes(StandardCharsets.UTF_8))));
+        request.body(new ByteBufferRequestContent(ByteBuffer.wrap(content1.getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(content2.getBytes(StandardCharsets.UTF_8))));
         final CountDownLatch headersLatch = new CountDownLatch(1);
         final CountDownLatch successLatch = new CountDownLatch(1);
         request.listener(new Request.Listener.Adapter()
@@ -273,7 +273,7 @@ public class HttpSenderOverHTTPTest
         Request request = client.newRequest(URI.create("http://localhost/"));
         String content1 = "0123456789";
         String content2 = "ABCDEF";
-        request.content(new ByteBufferContentProvider(ByteBuffer.wrap(content1.getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(content2.getBytes(StandardCharsets.UTF_8)))
+        request.body(new ByteBufferRequestContent(ByteBuffer.wrap(content1.getBytes(StandardCharsets.UTF_8)), ByteBuffer.wrap(content2.getBytes(StandardCharsets.UTF_8)))
         {
             @Override
             public long getLength()

jetty-client/src/test/java/org/eclipse/jetty/client/util/AsyncRequestContentTest.java

@@ -0,0 +1,151 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.util.Callback;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class AsyncRequestContentTest
+{
+    private ExecutorService executor;
+
+    @BeforeEach
+    public void prepare()
+    {
+        executor = Executors.newCachedThreadPool();
+    }
+
+    @AfterEach
+    public void dispose()
+    {
+        executor.shutdownNow();
+    }
+
+    @Test
+    public void testWhenEmptyFlushDoesNotBlock() throws Exception
+    {
+        AsyncRequestContent content = new AsyncRequestContent();
+
+        Future<?> task = executor.submit(() ->
+        {
+            content.flush();
+            return null;
+        });
+
+        assertTrue(await(task, 5000));
+    }
+
+    @Test
+    public void testOfferFlushDemandBlocksUntilSucceeded() throws Exception
+    {
+        AsyncRequestContent content = new AsyncRequestContent();
+        content.offer(ByteBuffer.allocate(1));
+
+        Future<?> task = executor.submit(() ->
+        {
+            content.flush();
+            return null;
+        });
+
+        // Wait until flush() blocks.
+        assertFalse(await(task, 500));
+
+        AtomicReference<Callback> callbackRef = new AtomicReference<>();
+        content.subscribe((buffer, last, callback) -> callbackRef.set(callback), true).demand();
+
+        // Flush should block until the callback is succeeded.
+        assertFalse(await(task, 500));
+
+        callbackRef.get().succeeded();
+
+        // Flush should return.
+        assertTrue(await(task, 5000));
+    }
+
+    @Test
+    public void testCloseFlushDoesNotBlock() throws Exception
+    {
+        AsyncRequestContent content = new AsyncRequestContent();
+        content.close();
+
+        Future<?> task = executor.submit(() ->
+        {
+            content.flush();
+            return null;
+        });
+
+        assertTrue(await(task, 5000));
+    }
+
+    @Test
+    public void testStallThenCloseProduces() throws Exception
+    {
+        AsyncRequestContent content = new AsyncRequestContent();
+
+        CountDownLatch latch = new CountDownLatch(1);
+        Request.Content.Subscription subscription = content.subscribe((buffer, last, callback) ->
+        {
+            callback.succeeded();
+            if (last)
+                latch.countDown();
+        }, true);
+
+        // Demand the initial content.
+        subscription.demand();
+
+        // Content must not be the last one.
+        assertFalse(latch.await(1, TimeUnit.SECONDS));
+
+        // Demand more content, now we are stalled.
+        subscription.demand();
+
+        // Close, we must be notified.
+        content.close();
+
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
+
+    private boolean await(Future<?> task, long time) throws Exception
+    {
+        try
+        {
+            task.get(time, TimeUnit.MILLISECONDS);
+            return true;
+        }
+        catch (TimeoutException x)
+        {
+            return false;
+        }
+    }
+}

jetty-client/src/test/java/org/eclipse/jetty/client/util/DeferredContentProviderTest.java

@@ -1,151 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-package org.eclipse.jetty.client.util;
-
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import org.eclipse.jetty.util.Callback;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-public class DeferredContentProviderTest
-{
-    private ExecutorService executor;
-
-    @BeforeEach
-    public void prepare() throws Exception
-    {
-        executor = Executors.newCachedThreadPool();
-    }
-
-    @AfterEach
-    public void dispose() throws Exception
-    {
-        executor.shutdownNow();
-    }
-
-    @Test
-    public void testWhenEmptyFlushDoesNotBlock() throws Exception
-    {
-        final DeferredContentProvider provider = new DeferredContentProvider();
-
-        Future<?> task = executor.submit(new Callable<Object>()
-        {
-            @Override
-            public Object call() throws Exception
-            {
-                provider.flush();
-                return null;
-            }
-        });
-
-        assertTrue(await(task, 5, TimeUnit.SECONDS));
-    }
-
-    @Test
-    public void testOfferFlushBlocksUntilSucceeded() throws Exception
-    {
-        final DeferredContentProvider provider = new DeferredContentProvider();
-        Iterator<ByteBuffer> iterator = provider.iterator();
-
-        provider.offer(ByteBuffer.allocate(0));
-
-        Future<?> task = executor.submit(new Callable<Object>()
-        {
-            @Override
-            public Object call() throws Exception
-            {
-                provider.flush();
-                return null;
-            }
-        });
-
-        // Wait until flush() blocks.
-        assertFalse(await(task, 1, TimeUnit.SECONDS));
-
-        // Consume the content and succeed the callback.
-        iterator.next();
-        ((Callback)iterator).succeeded();
-
-        // Flush should return.
-        assertTrue(await(task, 5, TimeUnit.SECONDS));
-    }
-
-    @Test
-    public void testCloseFlushDoesNotBlock() throws Exception
-    {
-        final DeferredContentProvider provider = new DeferredContentProvider();
-
-        provider.close();
-
-        Future<?> task = executor.submit(new Callable<Object>()
-        {
-            @Override
-            public Object call() throws Exception
-            {
-                provider.flush();
-                return null;
-            }
-        });
-
-        // Wait until flush() blocks.
-        assertTrue(await(task, 5, TimeUnit.SECONDS));
-    }
-
-    @Test
-    public void testCloseNextHasNextReturnsFalse() throws Exception
-    {
-        DeferredContentProvider provider = new DeferredContentProvider();
-        Iterator<ByteBuffer> iterator = provider.iterator();
-
-        provider.close();
-
-        assertFalse(iterator.hasNext());
-
-        assertThrows(NoSuchElementException.class, () -> iterator.next());
-
-        assertFalse(iterator.hasNext());
-    }
-
-    private boolean await(Future<?> task, long time, TimeUnit unit) throws Exception
-    {
-        try
-        {
-            task.get(time, unit);
-            return true;
-        }
-        catch (TimeoutException x)
-        {
-            return false;
-        }
-    }
-}

jetty-client/src/test/java/org/eclipse/jetty/client/util/InputStreamContentProviderTest.java

@@ -1,161 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-package org.eclipse.jetty.client.util;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.junit.jupiter.api.Test;
-
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-public class InputStreamContentProviderTest
-{
-    @Test
-    public void testHasNextFalseThenNext()
-    {
-        final AtomicBoolean closed = new AtomicBoolean();
-        InputStream stream = new InputStream()
-        {
-            @Override
-            public int read() throws IOException
-            {
-                return -1;
-            }
-
-            @Override
-            public void close() throws IOException
-            {
-                super.close();
-                closed.compareAndSet(false, true);
-            }
-        };
-
-        InputStreamContentProvider provider = new InputStreamContentProvider(stream);
-        Iterator<ByteBuffer> iterator = provider.iterator();
-
-        assertNotNull(iterator);
-        assertFalse(iterator.hasNext());
-
-        assertThrows(NoSuchElementException.class, () -> iterator.next());
-
-        assertFalse(iterator.hasNext());
-        assertTrue(closed.get());
-    }
-
-    @Test
-    public void testStreamWithContentThenNextThenNext()
-    {
-        final AtomicBoolean closed = new AtomicBoolean();
-        ByteArrayInputStream stream = new ByteArrayInputStream(new byte[]{1})
-        {
-            @Override
-            public void close() throws IOException
-            {
-                super.close();
-                closed.compareAndSet(false, true);
-            }
-        };
-
-        InputStreamContentProvider provider = new InputStreamContentProvider(stream);
-        Iterator<ByteBuffer> iterator = provider.iterator();
-
-        assertNotNull(iterator);
-
-        ByteBuffer buffer = iterator.next();
-
-        assertNotNull(buffer);
-
-        assertThrows(NoSuchElementException.class, () -> iterator.next());
-
-        assertFalse(iterator.hasNext());
-        assertTrue(closed.get());
-    }
-
-    @Test
-    public void testStreamWithExceptionThenNext()
-    {
-        final AtomicBoolean closed = new AtomicBoolean();
-        InputStream stream = new InputStream()
-        {
-            @Override
-            public int read() throws IOException
-            {
-                throw new IOException();
-            }
-
-            @Override
-            public void close() throws IOException
-            {
-                super.close();
-                closed.compareAndSet(false, true);
-            }
-        };
-
-        InputStreamContentProvider provider = new InputStreamContentProvider(stream);
-        Iterator<ByteBuffer> iterator = provider.iterator();
-
-        assertNotNull(iterator);
-
-        assertThrows(NoSuchElementException.class, () -> iterator.next());
-
-        assertFalse(iterator.hasNext());
-        assertTrue(closed.get());
-    }
-
-    @Test
-    public void testHasNextWithExceptionThenNext()
-    {
-        final AtomicBoolean closed = new AtomicBoolean();
-        InputStream stream = new InputStream()
-        {
-            @Override
-            public int read() throws IOException
-            {
-                throw new IOException();
-            }
-
-            @Override
-            public void close() throws IOException
-            {
-                super.close();
-                closed.compareAndSet(false, true);
-            }
-        };
-
-        InputStreamContentProvider provider = new InputStreamContentProvider(stream);
-        Iterator<ByteBuffer> iterator = provider.iterator();
-
-        assertNotNull(iterator);
-        assertTrue(iterator.hasNext());
-
-        assertThrows(NoSuchElementException.class, () -> iterator.next());
-
-        assertFalse(iterator.hasNext());
-        assertTrue(closed.get());
-    }
-}

jetty-client/src/test/java/org/eclipse/jetty/client/util/InputStreamContentTest.java

@@ -0,0 +1,266 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.eclipse.jetty.client.EmptyServerHandler;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.util.IO;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class InputStreamContentTest
+{
+    private Server server;
+    private ServerConnector connector;
+    private HttpClient client;
+
+    private void start(Handler handler) throws Exception
+    {
+        startServer(handler);
+        startClient();
+    }
+
+    private void startServer(Handler handler) throws Exception
+    {
+        QueuedThreadPool serverThreads = new QueuedThreadPool();
+        serverThreads.setName("server");
+        server = new Server(serverThreads);
+        connector = new ServerConnector(server, 1, 1);
+        server.addConnector(connector);
+        server.setHandler(handler);
+        server.start();
+    }
+
+    private void startClient() throws Exception
+    {
+        QueuedThreadPool clientThreads = new QueuedThreadPool();
+        clientThreads.setName("client");
+        client = new HttpClient();
+        client.setExecutor(clientThreads);
+        client.start();
+    }
+
+    @AfterEach
+    public void dispose() throws Exception
+    {
+        if (client != null)
+            client.stop();
+        if (server != null)
+            server.stop();
+    }
+
+    private static List<BiConsumer<Request, InputStream>> content()
+    {
+        return List.of(
+            (request, stream) -> request.body(new InputStreamRequestContent(stream)),
+            (request, stream) -> request.body(new InputStreamRequestContent(stream))
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("content")
+    public void testInputStreamEmpty(BiConsumer<Request, InputStream> setContent) throws Exception
+    {
+        CountDownLatch serverLatch = new CountDownLatch(1);
+        start(new EmptyServerHandler()
+        {
+            @Override
+            protected void service(String target, org.eclipse.jetty.server.Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException
+            {
+                serverLatch.countDown();
+                if (request.getInputStream().read() >= 0)
+                    throw new IOException();
+            }
+        });
+
+        CountDownLatch closeLatch = new CountDownLatch(1);
+        InputStream stream = new InputStream()
+        {
+            @Override
+            public int read()
+            {
+                return -1;
+            }
+
+            @Override
+            public void close() throws IOException
+            {
+                super.close();
+                closeLatch.countDown();
+            }
+        };
+
+        Request request = client.newRequest("localhost", connector.getLocalPort())
+            .timeout(5, TimeUnit.SECONDS);
+        setContent.accept(request, stream);
+        ContentResponse response = request.send();
+
+        assertTrue(serverLatch.await(5, TimeUnit.SECONDS));
+        assertEquals(response.getStatus(), HttpStatus.OK_200);
+        assertTrue(closeLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @MethodSource("content")
+    public void testInputStreamThrowing(BiConsumer<Request, InputStream> setContent) throws Exception
+    {
+        start(new EmptyServerHandler());
+
+        CountDownLatch closeLatch = new CountDownLatch(1);
+        InputStream stream = new InputStream()
+        {
+            @Override
+            public int read() throws IOException
+            {
+                throw new IOException();
+            }
+
+            @Override
+            public void close() throws IOException
+            {
+                super.close();
+                closeLatch.countDown();
+            }
+        };
+
+        Request request = client.newRequest("localhost", connector.getLocalPort())
+            .timeout(5, TimeUnit.SECONDS);
+        setContent.accept(request, stream);
+
+        assertThrows(ExecutionException.class, request::send);
+        assertTrue(closeLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @MethodSource("content")
+    public void testInputStreamThrowingAfterFirstRead(BiConsumer<Request, InputStream> setContent) throws Exception
+    {
+        byte singleByteContent = 0;
+        CountDownLatch serverLatch = new CountDownLatch(1);
+        start(new EmptyServerHandler()
+        {
+            @Override
+            protected void service(String target, org.eclipse.jetty.server.Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            {
+                assertEquals(singleByteContent, request.getInputStream().read());
+                serverLatch.countDown();
+            }
+        });
+
+        CountDownLatch closeLatch = new CountDownLatch(1);
+        InputStream stream = new InputStream()
+        {
+            private int reads;
+
+            @Override
+            public int read() throws IOException
+            {
+                if (++reads == 1)
+                    return singleByteContent;
+                throw new IOException();
+            }
+
+            @Override
+            public void close() throws IOException
+            {
+                super.close();
+                closeLatch.countDown();
+            }
+        };
+
+        Request request = client.newRequest("localhost", connector.getLocalPort())
+            .timeout(5, TimeUnit.SECONDS);
+        setContent.accept(request, stream);
+
+        assertThrows(ExecutionException.class, request::send);
+        assertTrue(closeLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @MethodSource("content")
+    public void testInputStreamWithSmallContent(BiConsumer<Request, InputStream> setContent) throws Exception
+    {
+        testInputStreamWithContent(setContent, new byte[1024]);
+    }
+
+    @ParameterizedTest
+    @MethodSource("content")
+    public void testInputStreamWithLargeContent(BiConsumer<Request, InputStream> setContent) throws Exception
+    {
+        testInputStreamWithContent(setContent, new byte[64 * 1024 * 1024]);
+    }
+
+    private void testInputStreamWithContent(BiConsumer<Request, InputStream> setContent, byte[] content) throws Exception
+    {
+        CountDownLatch serverLatch = new CountDownLatch(1);
+        start(new EmptyServerHandler()
+        {
+            @Override
+            protected void service(String target, org.eclipse.jetty.server.Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            {
+                serverLatch.countDown();
+                IO.copy(request.getInputStream(), IO.getNullStream());
+            }
+        });
+
+        CountDownLatch closeLatch = new CountDownLatch(1);
+        ByteArrayInputStream stream = new ByteArrayInputStream(content)
+        {
+            @Override
+            public void close() throws IOException
+            {
+                super.close();
+                closeLatch.countDown();
+            }
+        };
+
+        Request request = client.newRequest("localhost", connector.getLocalPort())
+            .timeout(5, TimeUnit.SECONDS);
+        setContent.accept(request, stream);
+        ContentResponse response = request.send();
+
+        assertTrue(serverLatch.await(5, TimeUnit.SECONDS));
+        assertEquals(response.getStatus(), HttpStatus.OK_200);
+        assertTrue(closeLatch.await(5, TimeUnit.SECONDS));
+    }
+}

jetty-client/src/test/java/org/eclipse/jetty/client/util/MultiPartContentTest.java

@@ -20,7 +20,6 @@ package org.eclipse.jetty.client.util;
 
 import java.io.BufferedWriter;
 import java.io.ByteArrayInputStream;
-import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -32,12 +31,10 @@ import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
 import javax.servlet.MultipartConfigElement;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
@@ -63,7 +60,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 // @checkstyle-disable-check : AvoidEscapedUnicodeCharactersCheck
-public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
+public class MultiPartContentTest extends AbstractHttpClientServerTest
 {
     @ParameterizedTest
     @ArgumentsSource(ScenarioProvider.class)
@@ -79,12 +76,12 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
         multiPart.close();
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send();
 
         assertEquals(200, response.getStatus());
@@ -109,13 +106,13 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
-        multiPart.addFieldPart(name, new StringContentProvider(value), null);
+        multiPart.addFieldPart(name, new StringRequestContent(value), null);
         multiPart.close();
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send();
 
         assertEquals(200, response.getStatus());
@@ -123,7 +120,7 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
 
     @ParameterizedTest
     @ArgumentsSource(ScenarioProvider.class)
-    public void testFieldWithOverridenContentType(Scenario scenario) throws Exception
+    public void testFieldWithOverriddenContentType(Scenario scenario) throws Exception
     {
         String name = "field";
         String value = "\u00e8";
@@ -146,16 +143,16 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
         HttpFields fields = new HttpFields();
         fields.put(HttpHeader.CONTENT_TYPE, "text/plain;charset=" + encoding.name());
-        BytesContentProvider content = new BytesContentProvider(value.getBytes(encoding));
+        BytesRequestContent content = new BytesRequestContent(value.getBytes(encoding));
         multiPart.addFieldPart(name, content, fields);
         multiPart.close();
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send();
 
         assertEquals(200, response.getStatus());
@@ -181,15 +178,15 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
-        DeferredContentProvider content = new DeferredContentProvider();
+        AsyncRequestContent content = new AsyncRequestContent("text/plain");
         multiPart.addFieldPart(name, content, null);
         multiPart.close();
         CountDownLatch responseLatch = new CountDownLatch(1);
         client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send(result ->
             {
                 assertTrue(result.isSucceeded(), supply(result.getFailure()));
@@ -233,8 +230,8 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
         });
 
         CountDownLatch closeLatch = new CountDownLatch(1);
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
-        InputStreamContentProvider content = new InputStreamContentProvider(new ByteArrayInputStream(data)
+        InputStreamRequestContent content = new InputStreamRequestContent(new ByteArrayInputStream(data)
         {
             @Override
             public void close() throws IOException
@@ -250,7 +247,7 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send();
 
         assertTrue(closeLatch.await(5, TimeUnit.SECONDS));
@@ -289,8 +286,8 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
-        PathContentProvider content = new PathContentProvider(contentType, tmpPath);
+        PathRequestContent content = new PathRequestContent(contentType, tmpPath);
         content.setByteBufferPool(client.getByteBufferPool());
         content.setUseDirectByteBuffers(client.isUseOutputDirectByteBuffers());
         multiPart.addFilePart(name, tmpPath.getFileName().toString(), content, null);
@@ -298,7 +295,7 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send();
 
         assertEquals(200, response.getStatus());
@@ -356,16 +353,16 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
         HttpFields fields = new HttpFields();
         fields.put(headerName, headerValue);
-        multiPart.addFieldPart(field, new StringContentProvider(value, encoding), fields);
+        multiPart.addFieldPart(field, new StringRequestContent(value, encoding), fields);
-        multiPart.addFilePart(fileField, tmpPath.getFileName().toString(), new PathContentProvider(tmpPath), null);
+        multiPart.addFilePart(fileField, tmpPath.getFileName().toString(), new PathRequestContent(tmpPath), null);
         multiPart.close();
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send();
 
         assertEquals(200, response.getStatus());
@@ -406,16 +403,18 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
             }
         });
 
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
+        MultiPartRequestContent multiPart = new MultiPartRequestContent();
-        DeferredContentProvider fieldContent = new DeferredContentProvider();
+        AsyncRequestContent fieldContent = new AsyncRequestContent();
         multiPart.addFieldPart("field", fieldContent, null);
-        DeferredContentProvider fileContent = new DeferredContentProvider();
+        AsyncRequestContent fileContent = new AsyncRequestContent();
         multiPart.addFilePart("file", "fileName", fileContent, null);
+        multiPart.close();
+
         CountDownLatch responseLatch = new CountDownLatch(1);
         client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(multiPart)
+            .body(multiPart)
             .send(result ->
             {
                 assertTrue(result.isSucceeded(), supply(result.getFailure()));
@@ -435,51 +434,9 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
         fieldContent.offer(encoding.encode(value));
         fieldContent.close();
 
-        multiPart.close();
-
         assertTrue(responseLatch.await(5, TimeUnit.SECONDS));
     }
 
-    @ParameterizedTest
-    @ArgumentsSource(ScenarioProvider.class)
-    public void testEachPartIsClosed(Scenario scenario) throws Exception
-    {
-        String name1 = "field1";
-        String value1 = "value1";
-        String name2 = "field2";
-        String value2 = "value2";
-        start(scenario, new AbstractMultiPartHandler()
-        {
-            @Override
-            protected void handle(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
-            {
-                Collection<Part> parts = request.getParts();
-                assertEquals(2, parts.size());
-                Iterator<Part> iterator = parts.iterator();
-                Part part1 = iterator.next();
-                assertEquals(name1, part1.getName());
-                assertEquals(value1, IO.toString(part1.getInputStream()));
-                Part part2 = iterator.next();
-                assertEquals(name2, part2.getName());
-                assertEquals(value2, IO.toString(part2.getInputStream()));
-            }
-        });
-
-        AtomicInteger closeCount = new AtomicInteger();
-        MultiPartContentProvider multiPart = new MultiPartContentProvider();
-        multiPart.addFieldPart(name1, new CloseableStringContentProvider(value1, closeCount::incrementAndGet), null);
-        multiPart.addFieldPart(name2, new CloseableStringContentProvider(value2, closeCount::incrementAndGet), null);
-        multiPart.close();
-        ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
-            .scheme(scenario.getScheme())
-            .method(HttpMethod.POST)
-            .content(multiPart)
-            .send();
-
-        assertEquals(200, response.getStatus());
-        assertEquals(2, closeCount.get());
-    }
-
     private abstract static class AbstractMultiPartHandler extends AbstractHandler
     {
         @Override
@@ -493,49 +450,4 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
 
         protected abstract void handle(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException;
     }
-
-    private static class CloseableStringContentProvider extends StringContentProvider
-    {
-        private final Runnable closeFn;
-
-        private CloseableStringContentProvider(String content, Runnable closeFn)
-        {
-            super(content);
-            this.closeFn = closeFn;
-        }
-
-        @Override
-        public Iterator<ByteBuffer> iterator()
-        {
-            return new CloseableIterator<>(super.iterator());
-        }
-
-        private class CloseableIterator<T> implements Iterator<T>, Closeable
-        {
-            private final Iterator<T> iterator;
-
-            public CloseableIterator(Iterator<T> iterator)
-            {
-                this.iterator = iterator;
-            }
-
-            @Override
-            public boolean hasNext()
-            {
-                return iterator.hasNext();
-            }
-
-            @Override
-            public T next()
-            {
-                return iterator.next();
-            }
-
-            @Override
-            public void close()
-            {
-                closeFn.run();
-            }
-        }
-    }
 }

jetty-client/src/test/java/org/eclipse/jetty/client/util/RequestContentBehaviorTest.java

@@ -0,0 +1,339 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.client.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.Fields;
+import org.eclipse.jetty.util.IO;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+public class RequestContentBehaviorTest
+{
+    private static Path emptyFile;
+    private static Path smallFile;
+
+    @BeforeAll
+    public static void prepare() throws IOException
+    {
+        Path testPath = MavenTestingUtils.getTargetTestingPath();
+        Files.createDirectories(testPath);
+        emptyFile = testPath.resolve("empty.txt");
+        Files.write(emptyFile, new byte[0]);
+        smallFile = testPath.resolve("small.txt");
+        byte[] bytes = new byte[64];
+        Arrays.fill(bytes, (byte)'#');
+        Files.write(smallFile, bytes);
+    }
+
+    @AfterAll
+    public static void dispose() throws IOException
+    {
+        if (smallFile != null)
+            Files.delete(smallFile);
+        if (emptyFile != null)
+            Files.delete(emptyFile);
+    }
+
+    public static List<Request.Content> emptyContents() throws IOException
+    {
+        return List.of(
+            new AsyncRequestContent()
+            {
+                {
+                    close();
+                }
+            },
+            new ByteBufferRequestContent(),
+            new BytesRequestContent(),
+            new FormRequestContent(new Fields()),
+            new InputStreamRequestContent(IO.getClosedStream()),
+            new MultiPartRequestContent()
+            {
+                {
+                    close();
+                }
+            },
+            new PathRequestContent(emptyFile),
+            new StringRequestContent("")
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("emptyContents")
+    public void testEmptyContentEmitInitialFirstDemand(Request.Content content) throws Exception
+    {
+        CountDownLatch latch = new CountDownLatch(1);
+        Request.Content.Subscription subscription = content.subscribe((buffer, last, callback) ->
+        {
+            if (last)
+                latch.countDown();
+        }, true);
+
+        subscription.demand();
+
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @MethodSource("emptyContents")
+    public void testEmptyContentDontEmitInitialFirstDemand(Request.Content content) throws Exception
+    {
+        AtomicBoolean initial = new AtomicBoolean(true);
+        AtomicReference<CountDownLatch> latch = new AtomicReference<>(new CountDownLatch(1));
+        Request.Content.Subscription subscription = content.subscribe((buffer, last, callback) ->
+        {
+            if (initial.get())
+            {
+                if (!last)
+                    latch.get().countDown();
+            }
+            else
+            {
+                if (last)
+                    latch.get().countDown();
+            }
+        }, false);
+
+        // Initial demand should have last=false.
+        subscription.demand();
+
+        assertTrue(latch.get().await(5, TimeUnit.SECONDS));
+
+        // More demand should have last=true.
+        initial.set(false);
+        latch.set(new CountDownLatch(1));
+        subscription.demand();
+
+        assertTrue(latch.get().await(5, TimeUnit.SECONDS));
+    }
+
+    public static List<Request.Content> smallContents() throws IOException
+    {
+        return List.of(
+            new AsyncRequestContent(ByteBuffer.allocate(64))
+            {
+                {
+                    close();
+                }
+            },
+            new ByteBufferRequestContent(ByteBuffer.allocate(64)),
+            new BytesRequestContent(new byte[64]),
+            new FormRequestContent(new Fields()
+            {
+                {
+                    add("foo", "bar");
+                }
+            }),
+            new InputStreamRequestContent(new ByteArrayInputStream(new byte[64])),
+            new MultiPartRequestContent()
+            {
+                {
+                    addFieldPart("field", new StringRequestContent("*".repeat(64)), null);
+                    close();
+                }
+            },
+            new PathRequestContent(smallFile),
+            new StringRequestContent("x".repeat(64))
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("smallContents")
+    public void testSmallContentEmitInitialFirstDemand(Request.Content content) throws Exception
+    {
+        AtomicBoolean initial = new AtomicBoolean(true);
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Request.Content.Subscription> subscriptionRef = new AtomicReference<>();
+        Request.Content.Subscription subscription = content.subscribe((buffer, last, callback) ->
+        {
+            if (initial.getAndSet(false))
+                assertTrue(buffer.hasRemaining());
+            if (last)
+                latch.countDown();
+            else
+                subscriptionRef.get().demand();
+        }, true);
+        subscriptionRef.set(subscription);
+
+        // Initial demand.
+        subscription.demand();
+
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @MethodSource("smallContents")
+    public void testSmallContentDontEmitInitialFirstDemand(Request.Content content) throws Exception
+    {
+        AtomicBoolean initial = new AtomicBoolean(true);
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Request.Content.Subscription> subscriptionRef = new AtomicReference<>();
+        Request.Content.Subscription subscription = content.subscribe((buffer, last, callback) ->
+        {
+            if (initial.getAndSet(false))
+            {
+                assertFalse(buffer.hasRemaining());
+                assertFalse(last);
+            }
+            if (last)
+                latch.countDown();
+            else
+                subscriptionRef.get().demand();
+        }, false);
+        subscriptionRef.set(subscription);
+
+        // Initial demand.
+        subscription.demand();
+
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @MethodSource("smallContents")
+    public void testSmallContentFailedAfterFirstDemand(Request.Content content)
+    {
+        Throwable testFailure = new Throwable("test_failure");
+
+        AtomicInteger notified = new AtomicInteger();
+        AtomicReference<Throwable> failureRef = new AtomicReference<>();
+        Request.Content.Subscription subscription = content.subscribe(new Request.Content.Consumer()
+        {
+            @Override
+            public void onContent(ByteBuffer buffer, boolean last, Callback callback)
+            {
+                notified.getAndIncrement();
+            }
+
+            @Override
+            public void onFailure(Throwable error)
+            {
+                testFailure.addSuppressed(new Throwable("suppressed"));
+                failureRef.compareAndSet(null, error);
+            }
+        }, false);
+
+        // Initial demand.
+        subscription.demand();
+
+        assertEquals(1, notified.get());
+
+        subscription.fail(testFailure);
+        subscription.demand();
+
+        assertEquals(1, notified.get());
+        Throwable failure = failureRef.get();
+        assertNotNull(failure);
+        assertSame(testFailure, failure);
+        assertEquals(1, failure.getSuppressed().length);
+    }
+
+    @ParameterizedTest
+    @MethodSource("smallContents")
+    public void testDemandAfterLastContentFails(Request.Content content) throws Exception
+    {
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Request.Content.Subscription> subscriptionRef = new AtomicReference<>();
+        AtomicReference<Throwable> failureRef = new AtomicReference<>();
+        Request.Content.Subscription subscription = content.subscribe(new Request.Content.Consumer()
+        {
+            @Override
+            public void onContent(ByteBuffer buffer, boolean last, Callback callback)
+            {
+                if (last)
+                    latch.countDown();
+                else
+                    subscriptionRef.get().demand();
+            }
+
+            @Override
+            public void onFailure(Throwable error)
+            {
+                error.addSuppressed(new Throwable("suppressed"));
+                failureRef.compareAndSet(null, error);
+            }
+        }, false);
+        subscriptionRef.set(subscription);
+
+        // Initial demand.
+        subscription.demand();
+
+        assertTrue(latch.await(5, TimeUnit.SECONDS));
+
+        // Demand more, should fail.
+        subscription.demand();
+
+        Throwable failure = failureRef.get();
+        assertNotNull(failure);
+        assertEquals(1, failure.getSuppressed().length);
+    }
+
+    @ParameterizedTest
+    @MethodSource("smallContents")
+    public void testReproducibleContentCanHaveMultipleSubscriptions(Request.Content content) throws Exception
+    {
+        assumeTrue(content.isReproducible());
+
+        CountDownLatch latch1 = new CountDownLatch(1);
+        Request.Content.Subscription subscription1 = content.subscribe((buffer, last, callback) ->
+        {
+            if (last)
+                latch1.countDown();
+        }, true);
+
+        CountDownLatch latch2 = new CountDownLatch(1);
+        Request.Content.Subscription subscription2 = content.subscribe((buffer, last, callback) ->
+        {
+            if (last)
+                latch2.countDown();
+        }, true);
+
+        // Initial demand.
+        subscription1.demand();
+        assertTrue(latch1.await(5, TimeUnit.SECONDS));
+
+        // Initial demand.
+        subscription2.demand();
+        assertTrue(latch2.await(5, TimeUnit.SECONDS));
+    }
+}

jetty-client/src/test/java/org/eclipse/jetty/client/util/SPNEGOAuthenticationTest.java

@@ -291,7 +291,7 @@ public class SPNEGOAuthenticationTest extends AbstractHttpClientServerTest
 
         requests.set(0);
         ByteArrayInputStream input = new ByteArrayInputStream("hello_world".getBytes(StandardCharsets.UTF_8));
-        request = client.newRequest(uri).method("POST").path("/secure").content(new InputStreamContentProvider(input));
+        request = client.newRequest(uri).method("POST").path("/secure").body(new InputStreamRequestContent(input));
         response = request.timeout(15, TimeUnit.SECONDS).send();
         assertEquals(200, response.getStatus());
         // Authentication expired, but POSTs are allowed.

jetty-client/src/test/java/org/eclipse/jetty/client/util/TypedContentProviderTest.java

@@ -108,7 +108,7 @@ public class TypedContentProviderTest extends AbstractHttpClientServerTest
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
-            .content(new FormContentProvider(fields))
+            .body(new FormRequestContent(fields))
             .header(HttpHeader.CONTENT_TYPE, contentType)
             .send();
 
@@ -135,7 +135,7 @@ public class TypedContentProviderTest extends AbstractHttpClientServerTest
 
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
-            .content(new StringContentProvider(null, content, StandardCharsets.UTF_8))
+            .body(new StringRequestContent(null, content, StandardCharsets.UTF_8))
             .send();
 
         assertEquals(200, response.getStatus());

jetty-documentation/pom.xml

@@ -49,9 +49,23 @@
   </properties>
 
   <dependencies>
+    <dependency>
+      <groupId>org.eclipse.jetty.toolchain</groupId>
+      <artifactId>jetty-servlet-api</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-io</artifactId>
+      <artifactId>jetty-client</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty.http2</groupId>
+      <artifactId>http2-http-client-transport</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty.fcgi</groupId>
+      <artifactId>fcgi-client</artifactId>
       <version>${project.version}</version>
     </dependency>
   </dependencies>
@@ -111,6 +125,13 @@
         <groupId>org.asciidoctor</groupId>
         <artifactId>asciidoctor-maven-plugin</artifactId>
         <version>${asciidoctor.maven.plugin.version}</version>
+        <dependencies>
+          <dependency>
+            <groupId>org.asciidoctor</groupId>
+            <artifactId>asciidoctorj-diagram</artifactId>
+            <version>2.0.1</version>
+          </dependency>
+        </dependencies>
         <configuration>
           <attributes>
             <JDURL>http://www.eclipse.org/jetty/javadoc/${javadoc.version}</JDURL>
@@ -125,7 +146,6 @@
             <MVNCENTRAL>http://central.maven.org/maven2</MVNCENTRAL>
             <VERSION>${project.version}</VERSION>
             <TIMESTAMP>${maven.build.timestamp}</TIMESTAMP>
-            <docbits>${basedir}/src/main/java</docbits>
           </attributes>
         </configuration>
         <executions>
@@ -169,6 +189,7 @@
               <sourceDirectory>${basedir}/src/main/asciidoc/contribution-guide</sourceDirectory>
               <sourceDocumentName>index.adoc</sourceDocumentName>
               <outputDirectory>${project.build.directory}/html/contribution-guide</outputDirectory>
+              <sourceHighlighter>coderay</sourceHighlighter>
             </configuration>
           </execution>
           <execution>
@@ -183,6 +204,10 @@
               <sourceDirectory>${basedir}/src/main/asciidoc/embedded-guide</sourceDirectory>
               <sourceDocumentName>index.adoc</sourceDocumentName>
               <outputDirectory>${project.build.directory}/html/embedded-guide</outputDirectory>
+              <sourceHighlighter>coderay</sourceHighlighter>
+              <requires>
+                <require>asciidoctor-diagram</require>
+              </requires>
             </configuration>
           </execution>
         </executions>

jetty-documentation/src/main/asciidoc/distribution-guide/annotations/using-annotations.adoc

@@ -121,9 +121,8 @@ As is the case with annotation scanning, the link:#using-extra-classpath-method[
 
 ____
 [NOTE]
-As of Jetty-9.4.4, unless the `web.xml` is version 3.0 or greater, only `ServletContainerInitializers` that are on the container classpath will be discovered.
+As of Jetty 10, Annotations will be discovered even for old versions of `web.xml` (2.5).
-Users wishing to use `ServletContainerInitializers` from within the webapp with older versions of `web.xml` must either upgrade their `web.xml` version, or call `WebAppContext.setConfigurationDiscovered(true)` either programmatically or in xml.
+Users wishing not to use annotations from the webapp classpath must call `WebAppContext.setConfigurationDiscovered(false)` either programmatically or in xml.
-Upgrading the `web.xml` version is preferable.
 ____
 
 ===== Controlling the order of ServletContainerInitializer invocation

jetty-documentation/src/main/asciidoc/distribution-guide/contexts/setting-form-size.adoc

@@ -19,57 +19,51 @@
 [[setting-form-size]]
 === Setting Max Form Size
 
-Jetty limits the amount of data that can post back from a browser or other client to the server.
+When a client issues a POST request with `Content-Type: application/x-www-form-urlencoded` there are 2 configurable limits imposed to help protect the server against denial of service attacks by malicious clients sending huge amounts of data.
-This helps protect the server against denial of service attacks by malicious clients sending huge amounts of data.
+
-The default maximum size Jetty permits is 200000 bytes.
+There is a maximum form size limit, and a maximum form keys limit.
-You can change this default for a particular webapp, for all webapps on a particular Server instance, or all webapps within the same JVM.
+
+The default maximum form size Jetty permits is 200000 bytes.
+The default maximum form keys Jetty permits is 1000 keys.
+
+You can change these defaults for a particular webapp, or all webapps within the same JVM.
 
 ==== For a Single Webapp
 
-The method to invoke is: `ContextHandler.setMaxFormContentSize(int maxSize);`
+The methods to invoke are:
+
+* `ContextHandler.setMaxFormContentSize(int maxSize);`
+* `ContextHandler.setMaxFormKeys(int max);`
 
 This can be done either in a context XML deployment descriptor external to the webapp, or in a `jetty-web.xml` file in the webapp's `WEB-INF` directory.
 
 In either case the syntax of the XML file is the same:
 
-[source, xml, subs="{sub-order}"]
+[source,xml,subs="{sub-order}"]
 ----
 <Configure class="org.eclipse.jetty.webapp.WebAppContext">
   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -->
   <!-- Max Form Size                                                   -->
   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -->
-  <Set name="maxFormContentSize">200000</Set>
+  <Set name="maxFormContentSize">400000</Set>
+  <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -->
+  <!-- Max Form Keys                                                   -->
+  <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -->
+  <Set name="maxFormKeys">400</Set>
 </Configure>
 ----
 
-==== For All Apps on a Server
-
-Set an attribute in `jetty.xml` on the Server instance for which you want to modify the maximum form content size:
-
-[source, xml, subs="{sub-order}"]
-----
-<Configure class="org.eclipse.jetty.server.Server">
-  <Call name="setAttribute">
-    <Arg>org.eclipse.jetty.server.Request.maxFormContentSize</Arg>
-    <Arg>200000</Arg>
-  </Call>
-</Configure>
-----
-
-____
-[IMPORTANT]
-It is important to remember that you should *not* modify the XML files in your `$JETTY_HOME`.
-If you do for some reason feel you want to change the way an XML file operates, it is best to make a copy of it in your `$JETTY_BASE` in an `/etc` directory.
-Jetty will always look first to the `$JETTY_BASE` for configuration.
-____
-
 ==== For All Apps in the JVM
 
-Use the system property `org.eclipse.jetty.server.Request.maxFormContentSize`.
+Use the system properties:
+
+* `org.eclipse.jetty.server.Request.maxFormContentSize` - the maximum size of Form Content allowed
+* `org.eclipse.jetty.server.Request.maxFormKeys` - the maximum number of Form Keys allowed
+
 This can be set on the command line or in the `$JETTY_BASE\start.ini` or any `$JETTY_BASE\start.d\*.ini` link:#startup-modules[module ini file.]
 Using `$JETTY_BASE\start.d\server.ini` as an example:
 
-[source, console, subs="{sub-order}"]
+[source,console,subs="{sub-order}"]
 ----
 # ---------------------------------------
 # Module: server
@@ -83,5 +77,6 @@ Using `$JETTY_BASE\start.d\server.ini` as an example:
 
 ...
 
--Dorg.eclipse.jetty.server.Request.maxFormContentSize=200000
+-Dorg.eclipse.jetty.server.Request.maxFormContentSize=400000
+-Dorg.eclipse.jetty.server.Request.maxFormKeys=400
 ----

jetty-documentation/src/main/asciidoc/distribution-guide/http2/introduction.adoc

@@ -50,5 +50,5 @@ The Jetty HTTP/2 implementation consists of the following sub-projects (each pro
 2.  `http2-hpack`: Contains the HTTP/2 HPACK implementation for HTTP header compression.
 3.  `http2-server`: Provides the server-side implementation of HTTP/2.
 4.  `http2-client`: Provides the implementation of HTTP/2 client with a low level HTTP/2 API, dealing with HTTP/2 streams, frames, etc.
-5.  `http2-http-client-transport`: Provides the implementation of the HTTP/2 transport for `HttpClient` (see xref:http-client[]).
+5.  `http2-http-client-transport`: Provides the implementation of the HTTP/2 transport for `HttpClient` (see xref:client-http[this section]).
 Applications can use the higher level API provided by `HttpClient` to send HTTP requests and receive HTTP responses, and the HTTP/2 transport will take care of converting them in HTTP/2 format (see also https://webtide.com/http2-support-for-httpclient/[this blog entry]).

jetty-documentation/src/main/asciidoc/embedded-guide/.asciidoctorconfig

@@ -0,0 +1,3 @@
+// Asciidoctor IDE configuration file.
+// See https://github.com/asciidoctor/asciidoctor-intellij-plugin/wiki/Support-project-specific-configurations
+:doc_code: ../../java

jetty-documentation/src/main/asciidoc/embedded-guide/client/client-concepts.adoc

@@ -1,149 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-[[client-concepts]]
-=== Client Libraries Concepts
-
-The Jetty client libraries implement a network client speaking different protocols
-such as HTTP/1.1, HTTP/2, WebSocket and FastCGI.
-
-It is possible to implement your own custom protocol on top of the Jetty client
-libraries.
-
-NOTE: TODO: perhaps add a section about this.
-
-There are conceptually three layers that compose the Jetty client libraries, from
-more abstract to more concrete:
-
-. The API layer, that exposes semantic APIs to applications so that they can write
-code such as "GET me the resource at this URI"
-. The protocol layer, where the API request is converted into the appropriate
-protocol bytes, for example encrypted HTTP/2
-. The infrastructure layer, that handles the low level I/O and deals with network,
-buffer, threads, etc.
-
-Let's look at these layers starting from the more concrete (and low level) one
-and build up to the more abstract layer.
-
-[[client-concepts-infrastructure]]
-==== Client Libraries Infrastructure Layer
-
-The Jetty client libraries use the common I/O design described in
-link:#io-arch[this section].
-The main client-side component is the
-link:{JDURL}/org/eclipse/jetty/io/ClientConnector.html[`ClientConnector`].
-
-The `ClientConnector` primarily wraps the
-link:{JDURL}/org/eclipse/jetty/io/SelectorManager.html[`SelectorManager`]
-and aggregates other four components: the thread pool (in form of an `Executor`),
-the `Scheduler`, the `ByteBufferPool` and the `SslContextFactory.Client`.
-
-The `ClientConnector` is where you want to set those components after you
-have configured them.
-If you don't explicitly set those components on the `ClientConnector`, then
-appropriate defaults will be chosen when the `ClientConnector` starts.
-
-The simplest example that creates and starts a `ClientConnector`:
-
-[source,java,indent=0]
-----
-include::{docbits}/embedded/client/ClientConnectorDocSnippets.java[tags=simplest]
-----
-
-A more typical example:
-
-[source,java,indent=0]
-----
-include::{docbits}/embedded/client/ClientConnectorDocSnippets.java[tags=typical]
-----
-
-A more advanced example that customizes the `ClientConnector` by overriding
-factory methods:
-
-[source,java,indent=0]
-----
-include::{docbits}/embedded/client/ClientConnectorDocSnippets.java[tags=advanced]
-----
-
-Since `ClientConnector` is the component that handles the low-level network, it
-is also the component where you want to configure the parameters that control
-how it should handle the low-level network.
-
-The most common parameters are:
-
-* `ClientConnector.selectors`: the number of ``java.nio.Selector``s components
-(defaults to `1`) that are present to handle the ``SocketChannel``s opened by
-the `ClientConnector`. You typically want to increase the number of selectors
-only for those use cases where each selector should handle more than few hundreds
-_concurrent_ socket events.
-For example, one selector typically runs well for `250` _concurrent_ socket
-events; as a rule of thumb, you can multiply that number by `10` to obtain the
-number of opened sockets a selector can handle (`2500`), based on the assumption
-that not all the `2500` sockets will be active _at the same time_.
-* `ClientConnector.idleTimeout`: the duration of time after which
-`ClientConnector` closes a socket due to inactivity (defaults to `30` seconds).
-This is an important parameter to configure, and you typically want the client
-idle timeout to be shorter than the server idle timeout, to avoid race
-conditions where the client attempts to use a socket just before the client-side
-idle timeout expires, but the server-side idle timeout has already expired and
-the is already closing the socket.
-* `ClientConnector.connectBlocking`: whether the operation of connecting a
-socket to the server (i.e. `SocketChannel.connect(SocketAddress)`) must be a
-blocking or a non-blocking operation (defaults to `false`).
-For `localhost` or same datacenter hosts you want to set this parameter to
-`true` because DNS resolution will be immediate (and likely never fail).
-For generic Internet hosts (e.g. when you are implementing a web spider) you
-want to set this parameter to `false`.
-* `ClientConnector.connectTimeout`: the duration of time after which
-`ClientConnector` aborts a connection attempt to the server (defaults to `5`
-seconds).
-This time includes the DNS lookup time _and_ the TCP connect time.
-
-Please refer to the `ClientConnector`
-link:{JDURL}/org/eclipse/jetty/io/ClientConnector.html[javadocs]
-for the complete list of configurable parameters.
-
-Once the `ClientConnector` is configured and started, it can be used to connect
-to the server via `ClientConnector.connect(SocketAddress, Map<String, Object>)`
-which in turn will call `SocketChannel.connect(SocketAddress)`.
-
-
-// TODO: from down here, moved to io-arch.adoc
-
-When establishing a TCP connection to a server, applications need to tell
-`ClientConnector` how to create the `Connection` for that particular
-TCP connection.
-This is done via a
-link:{JDURL}/org/eclipse/jetty/io/ClientConnectionFactory.html[`ClientConnectionFactory`].
-that must be passed in the context `Map` as follows:
-
-[source,java,indent=0]
-----
-include::{docbits}/embedded/client/ClientConnectorDocSnippets.java[tags=connect]
-----
-
-
-TODO: expand on what is the API to use, what parameters the context Map must
-have, and basically how we can write a generic network client with it.
-
-[[client-concepts-protocol]]
-==== Client Libraries Protocol Layer
-
-The protocol layer builds on top of the infrastructure layer to generate the
-bytes to be written to the network and to parse the bytes received from the
-network.

jetty-documentation/src/main/asciidoc/embedded-guide/client/client-io-arch.adoc

@@ -0,0 +1,196 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-client-io-arch]]
+=== Client Libraries Architecture
+
+The Jetty client libraries provide the basic components and APIs to implement
+a network client.
+
+They build on the common link:#io-arch[Jetty I/O Architecture] and provide client
+specific concepts (such as establishing a connection to a server).
+
+There are conceptually two layers that compose the Jetty client libraries:
+
+. xref:eg-client-io-arch-network[The network layer], that handles the low level
+I/O and deals with buffers, threads, etc.
+. xref:eg-client-io-arch-protocol[The protocol layer], that handles the parsing
+of bytes read from the network and the generation of bytes to write to the
+network.
+
+[[eg-client-io-arch-network]]
+==== Client Libraries Network Layer
+
+The Jetty client libraries use the common I/O design described in
+link:#io-arch[this section].
+The main client-side component is the
+link:{JDURL}/org/eclipse/jetty/io/ClientConnector.html[`ClientConnector`].
+
+The `ClientConnector` primarily wraps the
+link:{JDURL}/org/eclipse/jetty/io/SelectorManager.html[`SelectorManager`]
+and aggregates other four components:
+
+* a thread pool (in form of an `java.util.concurrent.Executor`)
+* a scheduler (in form of `org.eclipse.jetty.util.thread.Scheduler`)
+* a byte buffer pool (in form of `org.eclipse.jetty.io.ByteBufferPool`)
+* a TLS factory (in form of `org.eclipse.jetty.util.ssl.SslContextFactory.Client`)
+
+The `ClientConnector` is where you want to set those components after you
+have configured them.
+If you don't explicitly set those components on the `ClientConnector`, then
+appropriate defaults will be chosen when the `ClientConnector` starts.
+
+The simplest example that creates and starts a `ClientConnector` is the
+following:
+
+[source,java,indent=0]
+----
+include::../{doc_code}/embedded/client/ClientConnectorDocs.java[tags=simplest]
+----
+
+A more typical example:
+
+[source,java,indent=0]
+----
+include::../{doc_code}/embedded/client/ClientConnectorDocs.java[tags=typical]
+----
+
+A more advanced example that customizes the `ClientConnector` by overriding
+some of its methods:
+
+[source,java,indent=0]
+----
+include::../{doc_code}/embedded/client/ClientConnectorDocs.java[tags=advanced]
+----
+
+Since `ClientConnector` is the component that handles the low-level network, it
+is also the component where you want to configure the low-level network configuration.
+
+The most common parameters are:
+
+* `ClientConnector.selectors`: the number of ``java.nio.Selector``s components
+(defaults to `1`) that are present to handle the ``SocketChannel``s opened by
+the `ClientConnector`. You typically want to increase the number of selectors
+only for those use cases where each selector should handle more than few hundreds
+_concurrent_ socket events.
+For example, one selector typically runs well for `250` _concurrent_ socket
+events; as a rule of thumb, you can multiply that number by `10` to obtain the
+number of opened sockets a selector can handle (`2500`), based on the assumption
+that not all the `2500` sockets will be active _at the same time_.
+* `ClientConnector.idleTimeout`: the duration of time after which
+`ClientConnector` closes a socket due to inactivity (defaults to `30` seconds).
+This is an important parameter to configure, and you typically want the client
+idle timeout to be shorter than the server idle timeout, to avoid race
+conditions where the client attempts to use a socket just before the client-side
+idle timeout expires, but the server-side idle timeout has already expired and
+the is already closing the socket.
+* `ClientConnector.connectBlocking`: whether the operation of connecting a
+socket to the server (i.e. `SocketChannel.connect(SocketAddress)`) must be a
+blocking or a non-blocking operation (defaults to `false`).
+For `localhost` or same datacenter hosts you want to set this parameter to
+`true` because DNS resolution will be immediate (and likely never fail).
+For generic Internet hosts (e.g. when you are implementing a web spider) you
+want to set this parameter to `false`.
+* `ClientConnector.connectTimeout`: the duration of time after which
+`ClientConnector` aborts a connection attempt to the server (defaults to `5`
+seconds).
+This time includes the DNS lookup time _and_ the TCP connect time.
+
+Please refer to the `ClientConnector`
+link:{JDURL}/org/eclipse/jetty/io/ClientConnector.html[javadocs]
+for the complete list of configurable parameters.
+
+[[eg-client-io-arch-protocol]]
+==== Client Libraries Protocol Layer
+
+The protocol layer builds on top of the network layer to generate the
+bytes to be written to the network and to parse the bytes read from the
+network.
+
+Recall from link:#io-arch-connection[this section] that Jetty uses the
+`Connection` abstraction to produce and interpret the network bytes.
+
+On the client side, a `ClientConnectionFactory` implementation is the
+component that creates `Connection` instances based on the protocol that
+the client wants to "speak" with the server.
+
+Applications use `ClientConnector.connect(SocketAddress, Map<String, Object>)`
+to establish a TCP connection to the server, and must tell
+`ClientConnector` how to create the `Connection` for that particular
+TCP connection, and how to notify back the application when the connection
+creation succeeds or fails.
+
+This is done by passing a
+link:{JDURL}/org/eclipse/jetty/io/ClientConnectionFactory.html[`ClientConnectionFactory`]
+(that creates `Connection` instances) and a
+link:{JDURL}/org/eclipse/jetty/util/Promise.html[`Promise`] (that is notified
+of connection creation success or failure) in the context `Map` as follows:
+
+[source,java,indent=0]
+----
+include::../{doc_code}/embedded/client/ClientConnectorDocs.java[tags=connect]
+----
+
+When a `Connection` is created successfully, its `onOpen()` method is invoked,
+and then the promise is completed successfully.
+
+It is now possible to write a super-simple `telnet` client that reads and writes
+string lines:
+
+[source,java,indent=0]
+----
+include::../{doc_code}/embedded/client/ClientConnectorDocs.java[tags=telnet]
+----
+
+Note how a very basic "telnet" API that applications could use is implemented
+in the form of the `onLine(Consumer<String>)` for the non-blocking receiving
+side and `writeLine(String, Callback)` for the non-blocking sending side.
+Note also how the `onFillable()` method implements some basic "parsing"
+by looking up the `\n` character in the buffer.
+
+NOTE: The "telnet" client above looks like a super-simple HTTP client because
+HTTP/1.0 can be seen as a line-based protocol. HTTP/1.0 was used just as an
+example, but we could have used any other line-based protocol such as
+link:https://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol[SMTP],
+provided that the server was able to understand it.
+
+This is very similar to what the Jetty client implementation does for real
+network protocols.
+Real network protocols are of course more complicated and so is the implementation
+code that handles them, but the general ideas are similar.
+
+The Jetty client implementation provides a number of `ClientConnectionFactory`
+implementations that can be composed to produce and interpret the network bytes.
+
+For example, it is simple to modify the above example to use the TLS protocol
+so that you will be able to connect to the server on port `443`, typically
+reserved for the encrypted HTTP protocol.
+
+The differences between the clear-text version and the TLS encrypted version
+are minimal:
+
+[source,java,indent=0]
+----
+include::../{doc_code}/embedded/client/ClientConnectorDocs.java[tags=tlsTelnet]
+----
+
+The differences with the clear-text version are only:
+
+* Change the port from `80` to `443`.
+* Wrap the `ClientConnectionFactory` with `SslClientConnectionFactory`.
+* Unwrap the `SslConnection` to access `TelnetConnection`.

jetty-documentation/src/main/asciidoc/embedded-guide/client/client.adoc

@@ -16,20 +16,32 @@
 // ========================================================================
 //
 
-[[client]]
+[[eg-client]]
-== Jetty Client Libraries
+== Client Libraries
 
-The Eclipse Jetty Project provides not only server-side libraries so that you
+The Eclipse Jetty Project provides also provides client-side libraries
-can embed a server in your code, but it also provides client-side libraries
+that allow you to embed a client in your applications.
-that allow you to embed a client - for example a HTTP client invoking a third
+A typical example is a client application that needs to contact a third party
-party HTTP service - in your application.
+service via HTTP (for example a REST service).
+Another example is a proxy application that receives HTTP requests and
+forwards them as FCGI requests to a PHP application such as WordPress,
+or receives HTTP/1.1 requests and converts them to HTTP/2.
+Yet another example is a client application that needs to received events
+from a WebSocket server.
 
 The client libraries are designed to be non-blocking and offer both synchronous
 and asynchronous APIs and come with a large number of configuration options.
 
-There are primarily two client libraries:
+These are the available client libraries:
 
-* link:#client-http[The HTTP client library]
+* xref:eg-client-http[The HTTP Client Library]
-* link:#client-websocket[The WebSocket client library]
+* xref:eg-client-http2[The HTTP/2 Client Library]
+* xref:eg-client-websocket[The WebSocket client library]
 
-include::client-concepts.adoc[]
+If you are interested in the low-level details of how the Eclipse Jetty
+client libraries work, or are interested in writing a custom protocol,
+look at the xref:eg-client-io-arch[Client I/O Architecture].
+
+include::http/client-http.adoc[]
+include::http2/client-http2.adoc[]
+include::client-io-arch.adoc[]

jetty-documentation/src/main/asciidoc/embedded-guide/client/http/client-http-api.adoc

@@ -0,0 +1,297 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-client-http-api]]
+=== HttpClient API Usage
+
+`HttpClient` provides two types of APIs: a blocking API and a non-blocking API.
+
+[[eg-client-http-blocking]]
+==== HttpClient Blocking APIs
+
+The simpler way to perform a HTTP request is the following:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=simpleBlockingGet]
+----
+
+The method `HttpClient.GET(...)` performs a HTTP `GET` request to the given URI and returns a `ContentResponse` when the request/response conversation completes successfully.
+
+The `ContentResponse` object contains the HTTP response information: status code, headers and possibly content.
+The content length is limited by default to 2 MiB; for larger content see xref:client-http-content-response[].
+
+If you want to customize the request, for example by issuing a `HEAD` request instead of a `GET`, and simulating a browser user agent, you can do it in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=headFluent]
+----
+
+This is a shorthand for:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=headNonFluent]
+----
+
+You first create a request object using `httpClient.newRequest(...)`, and then you customize it using the fluent API style (that is, a chained invocation of methods on the request object).
+When the request object is customized, you call `request.send()` that produces the `ContentResponse` when the request/response conversation is complete.
+
+Simple `POST` requests also have a shortcut method:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=postFluent]
+----
+
+The `POST` parameter values added via the `param()` method are automatically URL-encoded.
+
+Jetty's `HttpClient` automatically follows redirects, so it handles the typical web pattern http://en.wikipedia.org/wiki/Post/Redirect/Get[POST/Redirect/GET], and the response object contains the content of the response of the `GET` request.
+Following redirects is a feature that you can enable/disable on a per-request basis or globally.
+
+File uploads also require one line, and make use of `java.nio.file` classes:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=fileFluent]
+----
+
+It is possible to impose a total timeout for the request/response conversation using the `Request.timeout(...)` method as follows:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=totalTimeout]
+----
+
+In the example above, when the 5 seconds expire, the request is aborted and a `java.util.concurrent.TimeoutException` is thrown.
+
+[[eg-client-http-non-blocking]]
+==== HttpClient Non-Blocking APIs
+
+So far we have shown how to use Jetty HTTP client in a blocking style - that is, the thread that issues the request blocks until the request/response conversation is complete.
+
+This section will look at Jetty's `HttpClient` non-blocking, asynchronous APIs that are perfectly suited for large content downloads, for parallel processing of requests/responses and in cases where performance and efficient thread and resource utilization is a key factor.
+
+The asynchronous APIs rely heavily on listeners that are invoked at various stages of request and response processing.
+These listeners are implemented by applications and may perform any kind of logic.
+The implementation invokes these listeners in the same thread that is used to process the request or response.
+Therefore, if the application code in these listeners takes a long time to execute, the request or response processing is delayed until the listener returns.
+
+If you need to execute application code that takes long time inside a listener, you must spawn your own thread.
+
+Request and response processing are executed by two different threads and therefore may happen concurrently.
+A typical example of this concurrent processing is an echo server, where a large upload may be concurrent with the large download echoed back.
+As a side note, remember that responses may be processed and completed _before_ requests; a typical example is a large upload that triggers a quick response - for example an error - by the server: the response may arrive and be completed while the request content is still being uploaded.
+
+The application thread that calls `Request.send(Response.CompleteListener)` performs the processing of the request until either the request is fully processed or until it would block on I/O, then it returns (and therefore never blocks).
+If it would block on I/O, the thread asks the I/O system to emit an event when the I/O will be ready to continue, then returns.
+When such an event is fired, a thread taken from the `HttpClient` thread pool will resume the processing of the request.
+
+Response are processed from the I/O thread that fires the event that bytes are ready to be read.
+Response processing continues until either the response is fully processed or until it would block for I/O.
+If it would block for I/O, the thread asks the I/O system to emit an event when the I/O will be ready to continue, then returns.
+When such an event is fired, a thread taken from the `HttpClient` thread pool will resume the processing of the response.
+
+When the request and the response are both fully processed, the thread that finished the last processing (usually the thread that processes the response, but may also be the thread that processes the request - if the request takes more time than the response to be processed) is used to dequeue the next request for the same destination and processes it.
+
+A simple non-blocking `GET` request that discards the response content can be written in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=simpleNonBlocking]
+----
+
+Method `Request.send(Response.CompleteListener)` returns `void` and does not block; the `Response.CompleteListener` lambda provided as a parameter is notified when the request/response conversation is complete, and the `Result` parameter allows you to access the request and response objects as well as failures, if any.
+
+You can impose a total timeout for the request/response conversation in the same way used by the synchronous API:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=nonBlockingTotalTimeout]
+----
+
+The example above will impose a total timeout of 3 seconds on the request/response conversation.
+
+The HTTP client APIs use listeners extensively to provide hooks for all possible request and response events:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=listeners]
+----
+
+This makes Jetty HTTP client suitable for HTTP load testing because, for example, you can accurately time every step of the request/response conversation (thus knowing where the request/response time is really spent).
+
+Have a look at the link:{JDURL}/org/eclipse/jetty/client/api/Request.Listener.html[`Request.Listener`] class to know about request events, and to the link:{JDURL}/org/eclipse/jetty/client/api/Response.Listener.html[`Response.Listener`] class to know about response events.
+
+[[eg-client-http-content]]
+==== HttpClient Content Handling
+
+[[eg-client-http-content-request]]
+===== Request Content Handling
+
+Jetty's `HttpClient` provides a number of utility classes off the shelf to handle request content.
+
+You can provide request content as `String`, `byte[]`, `ByteBuffer`, `java.nio.file.Path`, `InputStream`, and provide your own implementation of `org.eclipse.jetty.client.api.Request.Content`.
+Here’s an example that provides the request content using `java.nio.file.Paths`:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=pathRequestContent]
+----
+
+Alternatively, you can use `FileInputStream` via the `InputStreamRequestContent` utility class:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=inputStreamRequestContent]
+----
+
+Since `InputStream` is blocking, then also the send of the request will block if the input stream blocks, even in case of usage of the non-blocking `HttpClient` APIs.
+
+If you have already read the content in memory, you can pass it as a `byte[]` (or a `String`) using the `BytesRequestContent` (or `StringRequestContent`) utility class:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=bytesStringRequestContent]
+----
+
+If the request content is not immediately available, but your application will be notified of the content to send, you can use `AsyncRequestContent` in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=asyncRequestContent]
+----
+
+While the request content is awaited and consequently uploaded by the client application, the server may be able to respond (at least with the response headers) completely asynchronously.
+In this case, `Response.Listener` callbacks will be invoked before the request is fully sent.
+This allows fine-grained control of the request/response conversation: for example the server may reject contents that are too big, send a response to the client, which in turn may stop the content upload.
+
+Another way to provide request content is by using an `OutputStreamRequestContent`,
+which allows applications to write request content when it is available to the `OutputStream` provided by `OutputStreamRequestContent`:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=outputStreamRequestContent]
+----
+
+[[eg-client-http-content-response]]
+===== Response Content Handling
+
+Jetty's `HttpClient` allows applications to handle response content in different ways.
+
+You can buffer the response content in memory; this is done when using the xref:client-http-blocking[blocking APIs] and the content is buffered within a `ContentResponse` up to 2 MiB.
+
+If you want to control the length of the response content (for example limiting to values smaller than the default of 2 MiB), then you can use a `org.eclipse.jetty.client.util.FutureResponseListener` in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=futureResponseListener]
+----
+
+If the response content length is exceeded, the response will be aborted, and an exception will be thrown by method `get(...)`.
+
+You can buffer the response content in memory also using the xref:client-http-non-blocking[non-blocking APIs], via the `BufferingResponseListener` utility class:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=bufferingResponseListener]
+----
+
+If you want to avoid buffering, you can wait for the response and then stream the content using the `InputStreamResponseListener` utility class:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=inputStreamResponseListener]
+----
+
+Finally, let's look at the advanced usage of the response content handling.
+
+The response content is provided by the `HttpClient` implementation to application
+listeners following a reactive model similar to that of `java.util.concurrent.Flow`.
+
+The listener that follows this model is `Response.DemandedContentListener`.
+
+After the response headers have been processed by the `HttpClient` implementation,
+`Response.DemandedContentListener.onBeforeContent(response, demand)` is
+invoked. This allows the application to control whether to demand the first
+content or not. The default implementation of this method calls `demand.accept(1)`,
+which demands one chunk of content to the implementation.
+The implementation will deliver the chunk of content as soon as it is available.
+
+The chunks of content are delivered to the application by invoking
+`Response.DemandedContentListener.onContent(response, demand, buffer, callback)`.
+Applications implement this method to process the content bytes in the `buffer`.
+Succeeding the `callback` signals to the implementation that the application
+has consumed the `buffer` so that the implementation can dispose/recycle the
+`buffer`. Failing the `callback` signals to the implementation to fail the
+response (no more content will be delivered, and the _response failed_ event
+will be emitted).
+
+IMPORTANT: Succeeding the `callback` must be done only after the `buffer`
+bytes have been consumed. When the `callback` is succeeded, the `HttpClient`
+implementation may reuse the `buffer` and overwrite the bytes with different
+bytes; if the application looks at the `buffer` _after_ having succeeded
+the `callback` is may see other, unrelated, bytes.
+
+The application uses the `demand` object to demand more content chunks.
+Applications will typically demand for just one more content via
+`demand.accept(1)`, but may decide to demand for more via `demand.accept(2)`
+or demand "infinitely" once via `demand.accept(Long.MAX_VALUE)`.
+Applications that demand for more than 1 chunk of content must be prepared
+to receive all the content that they have demanded.
+
+Demanding for content and consuming the content are orthogonal activities.
+
+An application can demand "infinitely" and store aside the pairs
+`(buffer, callback)` to consume them later.
+If not done carefully, this may lead to excessive memory consumption, since
+the ``buffer``s are not consumed.
+Succeeding the ``callback``s will result in the ``buffer``s to be
+disposed/recycled and may be performed at any time.
+
+An application can also demand one chunk of content, consume it (by
+succeeding the associated `callback`) and then _not_ demand for more content
+until a later time.
+
+Subclass `Response.AsyncContentListener` overrides the behavior of
+`Response.DemandedContentListener`; when an application implementing its
+`onContent(response, buffer, callback)` succeeds the `callback`, it
+will have _both_ the effect of disposing/recycling the `buffer` _and_ the
+effect of demanding one more chunk of content.
+
+Subclass `Response.ContentListener` overrides the behavior of
+`Response.AsyncContentListener`; when an application implementing its
+`onContent(response, buffer)` returns from the method itself, it will
+_both_ the effect of disposing/recycling the `buffer` _and_ the effect
+of demanding one more chunk of content.
+
+Previous examples of response content handling were inefficient because they
+involved copying the `buffer` bytes, either to accumulate them aside so that
+the application could use them when the request was completed, or because
+they were provided to an API such as `InputStream` that made use of `byte[]`
+(and therefore a copy from `ByteBuffer` to `byte[]` is necessary).
+
+An application that implements a forwarder between two servers can be
+implemented efficiently by handling the response content without copying
+the `buffer` bytes as in the following example:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=demandedContentListener]
+----

jetty-documentation/src/main/asciidoc/embedded-guide/client/http/client-http-authentication.adoc

@@ -0,0 +1,101 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-client-http-authentication]]
+=== HttpClient Authentication Support
+
+Jetty's `HttpClient` supports the `BASIC` and `DIGEST` authentication
+mechanisms defined by link:https://tools.ietf.org/html/rfc7235[RFC 7235],
+as well as the SPNEGO authentication mechanism defined in
+link:https://tools.ietf.org/html/rfc4559[RFC 4559].
+
+The HTTP _conversation_ - the sequence of related HTTP requests - for a
+request that needs authentication is the following:
+
+[plantuml]
+----
+skinparam backgroundColor transparent
+skinparam monochrome true
+skinparam shadowing false
+
+participant Application
+participant HttpClient
+participant Server
+
+Application -> Server : GET /path
+Server -> HttpClient : 401 + WWW-Authenticate
+HttpClient -> Server : GET + Authentication
+Server -> Application : 200 OK
+----
+
+Upon receiving a HTTP 401 response code, `HttpClient` looks at the
+`WWW-Authenticate` response header (the server _challenge_) and then tries to
+match configured authentication credentials to produce an `Authentication`
+header that contains the authentication credentials to access the resource.
+
+You can configure authentication credentials in the `HttpClient` instance as
+follows:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=addAuthentication]
+----
+
+``Authentication``s are matched against the server challenge first by
+mechanism (e.g. `BASIC` or `DIGEST`), then by realm and then by URI.
+
+If an `Authentication` match is found, the application does not receive events
+related to the HTTP 401 response. These events are handled internally by
+`HttpClient` which produces another (internal) request similar to the original
+request but with an additional `Authorization` header.
+
+If the authentication is successful, the server responds with a HTTP 200 and
+`HttpClient` caches the `Authentication.Result` so that subsequent requests
+for a matching URI will not incur in the additional rountrip caused by the
+HTTP 401 response.
+
+It is possible to clear ``Authentication.Result``s in order to force
+authentication again:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=clearResults]
+----
+
+Authentication results may be preempted to avoid the additional roundtrip
+due to the server challenge in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=preemptedResult]
+----
+
+In this way, requests for the given URI are enriched immediately with the
+`Authorization` header, and the server should respond with HTTP 200 (and the
+resource content) rather than with the 401 and the challenge.
+
+It is also possible to preempt the authentication for a single request only,
+in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=requestPreemptedResult]
+----
+
+See also the xref:eg-client-http-proxy-authentication[proxy authentication section]
+for further information about how authentication works with HTTP proxies.

jetty-documentation/src/main/asciidoc/embedded-guide/client/http/client-http-configuration.adoc

@@ -0,0 +1,86 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-client-http-configuration]]
+=== HttpClient Configuration
+
+`HttpClient` has a quite large number of configuration parameters.
+Please refer to the `HttpClient`
+link:{JDURL}/org/eclipse/jetty/client/HttpClient.html[javadocs]
+for the complete list of configurable parameters.
+The most common parameters are:
+
+* `HttpClient.idleTimeout`: same as `ClientConnector.idleTimeout`
+described in xref:eg-client-io-arch-network[this section].
+* `HttpClient.connectBlocking`: same as `ClientConnector.connectBlocking`
+described in xref:eg-client-io-arch-network[this section].
+* `HttpClient.connectTimeout`: same as `ClientConnector.connectTimeout`
+described in xref:eg-client-io-arch-network[this section].
+* `HttpClient.maxConnectionsPerDestination`: the max number of TCP
+connections that are opened for a particular destination (defaults to 64).
+* `HttpClient.maxRequestsQueuedPerDestination`: the max number of requests
+queued (defaults to 1024).
+
+[[eg-client-http-configuration-tls]]
+==== HttpClient TLS Configuration
+
+`HttpClient` supports HTTPS requests out-of-the-box like a browser does.
+
+The support for HTTPS request is provided by a `SslContextFactory.Client`,
+typically configured in the `ClientConnector`.
+If not explicitly configured, the `ClientConnector` will allocate a default
+one when started.
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=tlsExplicit]
+----
+
+The default `SslContextFactory.Client` verifies the certificate sent by the
+server by verifying the certificate chain.
+This means that requests to public websites that have a valid certificates
+(such as ``https://google.com``) will work out-of-the-box.
+
+However, requests made to sites (typically ``localhost``) that have invalid
+(for example, expired or with a wrong host) or self-signed certificates will
+fail (like they will in a browser).
+
+Certificate validation is performed at two levels: at the TLS implementation
+level (in the JDK) and - optionally - at the application level.
+
+By default, certificate validation at the TLS level is enabled, while
+certificate validation at the application level is disabled.
+
+You can configure the `SslContextFactory.Client` to skip certificate validation
+at the TLS level:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=tlsNoValidation]
+----
+
+You can enable certificate validation at the application level:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=tlsAppValidation]
+----
+
+Please refer to the `SslContextFactory.Client`
+link:{JDURL}/org/eclipse/jetty/util/ssl/SslContextFactory.Client.html[javadocs]
+for the complete list of configurable parameters.

jetty-documentation/src/main/asciidoc/embedded-guide/client/http/client-http-cookie.adoc

@@ -16,104 +16,88 @@
 // ========================================================================
 //
 
-[[http-client-cookie]]
+[[eg-client-http-cookie]]
-=== Cookies Support
+=== HttpClient Cookie Support
 
-Jetty HTTP client supports cookies out of the box.
+Jetty's `HttpClient` supports cookies out of the box.
 The `HttpClient` instance receives cookies from HTTP responses and stores them in a `java.net.CookieStore`, a class that is part of the JDK.
 When new requests are made, the cookie store is consulted and if there are matching cookies (that is, cookies that are not expired and that match domain and path of the request) then they are added to the requests.
 
 Applications can programmatically access the cookie store to find the cookies that have been set:
 
-[source, java, subs="{sub-order}"]
+[source,java,indent=0]
 ----
-CookieStore cookieStore = httpClient.getCookieStore();
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=getCookies]
-List<HttpCookie> cookies = cookieStore.get(URI.create("http://domain.com/path"));
 ----
 
 Applications can also programmatically set cookies as if they were returned from a HTTP response:
 
-[source, java, subs="{sub-order}"]
+[source,java,indent=0]
 ----
-CookieStore cookieStore = httpClient.getCookieStore();
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=setCookie]
-HttpCookie cookie = new HttpCookie("foo", "bar");
-cookie.setDomain("domain.com");
-cookie.setPath("/");
-cookie.setMaxAge(TimeUnit.DAYS.toSeconds(1));
-cookieStore.add(URI.create("http://domain.com"), cookie);
 ----
 
-Cookies may be added only for a particular request:
+Cookies may be added explicitly only for a particular request:
 
-[source, java, subs="{sub-order}"]
+[source,java,indent=0]
 ----
-ContentResponse response = httpClient.newRequest("http://domain.com/path")
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=requestCookie]
-        .cookie(new HttpCookie("foo", "bar"))
-        .send();
 ----
 
 You can remove cookies that you do not want to be sent in future HTTP requests:
 
-[source, java, subs="{sub-order}"]
+[source,java,indent=0]
 ----
-CookieStore cookieStore = httpClient.getCookieStore();
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=removeCookie]
-URI uri = URI.create("http://domain.com");
-List<HttpCookie> cookies = cookieStore.get(uri);
-for (HttpCookie cookie : cookies)
-    cookieStore.remove(uri, cookie);
 ----
 
-If you want to totally disable cookie handling, you can install a `HttpCookieStore.Empty` instance in this way:
+If you want to totally disable cookie handling, you can install a
+`HttpCookieStore.Empty`. This must be done when `HttpClient` is used in a
+proxy application, in this way:
 
-[source, java, subs="{sub-order}"]
+[source,java,indent=0]
 ----
-httpClient.setCookieStore(new HttpCookieStore.Empty());
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=emptyCookieStore]
 ----
 
 You can enable cookie filtering by installing a cookie store that performs the filtering logic in this way:
 
-[source, java, subs="{sub-order}"]
+[source,java,indent=0]
 ----
-httpClient.setCookieStore(new GoogleOnlyCookieStore());
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=filteringCookieStore]
-
-public class GoogleOnlyCookieStore extends HttpCookieStore
-{
-    @Override
-    public void add(URI uri, HttpCookie cookie)
-    {
-        if (uri.getHost().endsWith("google.com"))
-            super.add(uri, cookie);
-    }
-}
 ----
 
 The example above will retain only cookies that come from the `google.com` domain or sub-domains.
 
+// TODO: move this section to server-side
 ==== Special Characters in Cookies
 Jetty is compliant with link:https://tools.ietf.org/html/rfc6265[RFC6265], and as such care must be taken when setting a cookie value that includes special characters such as `;`.
 
 Previously, Version=1 cookies defined in link:https://tools.ietf.org/html/rfc2109[RFC2109] (and continued in link:https://tools.ietf.org/html/rfc2965[RFC2965]) allowed for special/reserved characters to be enclosed within double quotes when declared in a `Set-Cookie` response header:
 
-[source, java, subs="{sub-order}"]
+[source,subs="{sub-order}"]
 ----
 Set-Cookie: foo="bar;baz";Version=1;Path="/secur"
 ----
 
-This was added to the HTTP Response header as follows:
+This was added to the HTTP Response as follows:
 
-[source, java, subs="{sub-order}"]
+[source,java,subs="{sub-order}"]
 ----
-Cookie cookie = new Cookie("foo", "bar;baz");
+protected void service(HttpServletRequest request, HttpServletResponse response)
-cookie.setPath("/secur");
+{
-response.addCookie(cookie);
+    javax.servlet.http.Cookie cookie = new Cookie("foo", "bar;baz");
+    cookie.setPath("/secur");
+    response.addCookie(cookie);
+}
 ----
 
 The introduction of RFC6265 has rendered this approach no longer possible; users are now required to encode cookie values that use these special characters.
 This can be done utilizing `javax.servlet.http.Cookie` as follows:
 
-[source, java, subs="{sub-order}"]
+[source,java,subs="{sub-order}"]
 ----
-Cookie cookie = new Cookie("foo", URLEncoder.encode("bar;baz", "utf-8"));
+javax.servlet.http.Cookie cookie = new Cookie("foo", URLEncoder.encode("bar;baz", "UTF-8"));
 ----
 
 Jetty validates all cookie names and values being added to the `HttpServletResponse` via the `addCookie(Cookie)` method.

jetty-documentation/src/main/asciidoc/embedded-guide/client/http/client-http-intro.adoc

@@ -0,0 +1,253 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-client-http-intro]]
+=== HttpClient Introduction
+
+The Jetty HTTP client module provides easy-to-use APIs and utility classes to perform HTTP (or HTTPS) requests.
+
+Jetty's HTTP client is non-blocking and asynchronous.
+It offers an asynchronous API that never blocks for I/O, making it very efficient in thread utilization and well suited for high performance scenarios such as load testing or parallel computation.
+
+However, when all you need to do is to perform a `GET` request to a resource, Jetty's HTTP client offers also a synchronous API; a programming interface
+where the thread that issued the request blocks until the request/response conversation is complete.
+
+Jetty's HTTP client supports xref:#eg-client-http-transport[different transports]: HTTP/1.1, FastCGI and HTTP/2.
+This means that the semantic of a HTTP request (that is, " `GET` me the resource `/index.html` ") can be carried over the network in different formats.
+The most common and default format is HTTP/1.1.
+That said, Jetty's HTTP client can carry the same request using the FastCGI format or the HTTP/2 format.
+
+The FastCGI transport is heavily used in Jetty's link:#fastcgi[FastCGI support] that allows Jetty to work as a reverse proxy to PHP (exactly like Apache or Nginx do) and therefore be able to serve - for example - WordPress websites.
+
+The HTTP/2 transport allows Jetty's HTTP client to perform requests using HTTP/2 to HTTP/2 enabled web sites, see also Jetty's link:#http2[HTTP/2 support].
+
+Out of the box features that you get with the Jetty HTTP client include:
+
+* Redirect support - redirect codes such as 302 or 303 are automatically followed.
+* Cookies support - cookies sent by servers are stored and sent back to servers in matching requests.
+* Authentication support - HTTP "Basic" and "Digest" authentications are supported, others are pluggable.
+* Forward proxy support - HTTP proxying and SOCKS4 proxying.
+
+[[eg-client-http-start]]
+==== Starting HttpClient
+
+The Jetty artifact that provides the main HTTP client implementation is `jetty-client`.
+The Maven artifact coordinates are the following:
+
+[source,xml,subs="{sub-order}"]
+----
+<dependency>
+  <groupId>org.eclipse.jetty</groupId>
+  <artifactId>jetty-client</artifactId>
+  <version>{version}</version>
+</dependency>
+----
+
+The main class is named `org.eclipse.jetty.client.HttpClient`.
+
+You can think of a `HttpClient` instance as a browser instance.
+Like a browser it can make requests to different domains, it manages redirects, cookies and authentication, you can configure it with a proxy, and
+it provides you with the responses to the requests you make.
+
+In order to use `HttpClient`, you must instantiate it, configure it, and then start it:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=start]
+----
+
+You may create multiple instances of `HttpClient`, but typically one instance is enough for an application.
+There are several reasons for having multiple `HttpClient` instances including, but not limited to:
+
+* You want to specify different configuration parameters (for example, one instance is configured with a forward proxy while another is not).
+* You want the two instances to behave like two different browsers and hence have different cookies, different authentication credentials, etc.
+* You want to use link:#eg-client-http-transport[different transports].
+
+Like browsers, HTTPS requests are supported out-of-the-box, as long as the server
+provides a valid certificate.
+In case the server does not provide a valid certificate (or in case it is self-signed)
+you want to customize ``HttpClient``'s TLS configuration as described in
+xref:eg-client-http-configuration-tls[this section].
+
+[[eg-client-http-stop]]
+==== Stopping HttpClient
+
+It is recommended that when your application stops, you also stop the `HttpClient` instance (or instances) that you are using.
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=stop]
+----
+
+Stopping `HttpClient` makes sure that the memory it holds (for example, authentication credentials, cookies, etc.) is released, and that the thread pool and scheduler are properly stopped allowing all threads used by `HttpClient` to exit.
+
+[[eg-client-http-arch]]
+==== HttpClient Architecture
+
+A `HttpClient` instance can be thought as a browser instance, and it manages the
+following components:
+
+* a `CookieStore` (see xref:eg-client-http-cookie[this section]).
+* a `AuthenticationStore` (see xref:eg-client-http-authentication[this section]).
+* a `ProxyConfiguration` (see xref:eg-client-http-proxy[this section]).
+* a set of _destinations_.
+
+A _destination_ is the client-side component that represent an _origin_ on
+a server, and manages a queue of requests for that origin, and a
+xref:eg-client-http-connection-pool[pool of connections] to that origin.
+
+An _origin_ may be simply thought as the tuple `(scheme, host, port)` and it
+is where the client connects to in order to communicate with the server.
+However, this is not enough.
+
+If you use `HttpClient` to write a proxy you may have different clients that
+want to contact the same server.
+In this case, you may not want to use the same proxy-to-server connection to
+proxy requests for both clients, for example for authentication reasons: the
+server may associate the connection with authentication credentials and you
+do not want to use the same connection for two different users that have
+different credentials.
+Instead, you want to use different connections for different clients and
+this can be achieved by "tagging" a destination with a tag object that
+represents the remote client (for example, it could be the remote client IP
+address).
+
+Two origins with the same `(scheme, host, port)` but different `tag`
+create two different destinations and therefore two different connection pools.
+However, also this is not enough.
+
+It is possible that a server speaks different protocols on the same `port`.
+A connection may start by speaking one protocol, for example HTTP/1.1, but
+then be upgraded to speak a different protocol, for example HTTP/2.
+After a connection has been upgraded to a second protocol, it cannot speak
+the first protocol anymore, so it can only be used to communicate using
+the second protocol.
+
+Two origins with the same `(scheme, host, port)` but different
+`protocol` create two different destinations and therefore two different
+connection pools.
+
+Therefore an origin is identified by the tuple
+`(scheme, host, port, tag, protocol)`.
+
+[[eg-client-http-connection-pool]]
+==== HttpClient Connection Pooling
+
+A destination manages a `org.eclipse.jetty.client.ConnectionPool`, where
+connections to a particular origin are pooled for performance reasons:
+opening a connection is a costly operation and it's better to reuse them
+for multiple requests.
+
+NOTE: Remember that to select a specific destination you must select a
+specific origin, and that an origin is identified by the tuple
+`(scheme, host, port, tag, protocol)`, so you can have multiple destinations
+for the same `host` and `port`.
+
+You can access the `ConnectionPool` in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=getConnectionPool]
+----
+
+Jetty's client library provides the following `ConnectionPool` implementations:
+
+* `DuplexConnectionPool`, historically the first implementation, only used by
+the HTTP/1.1 transport.
+* `MultiplexConnectionPool`, the generic implementation valid for any transport
+where connections are reused with a MRU (most recently used) algorithm (that is,
+the connections most recently returned to the connection pool are the more
+likely to be used again).
+* `RoundRobinConnectionPool`, similar to `MultiplexConnectionPool` but where
+connections are reused with a round-robin algorithm.
+
+The `ConnectionPool` implementation can be customized for each destination in
+by setting a `ConnectionPool.Factory` on the `HttpClientTransport`:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tags=setConnectionPool]
+----
+
+[[eg-client-http-request-processing]]
+==== HttpClient Request Processing
+
+[plantuml]
+----
+skinparam backgroundColor transparent
+skinparam monochrome true
+skinparam shadowing false
+
+participant Application
+participant Request
+participant HttpClient
+participant Destination
+participant ConnectionPool
+participant Connection
+
+Application -> HttpClient : newRequest()
+HttpClient -> Request **
+Application -> Request : send()
+Request -> HttpClient : send()
+HttpClient -> Destination ** : get or create
+Destination -> ConnectionPool ** : create
+HttpClient -> Destination : send(Request)
+Destination -> Destination : enqueue(Request)
+Destination -> ConnectionPool : acquire()
+ConnectionPool -> Connection ** : create
+Destination -> Destination : dequeue(Request)
+Destination -> Connection : send(Request)
+----
+
+When a request is sent, an origin is computed from the request; `HttpClient`
+uses that origin to find (or create if it does not exist) the correspondent
+destination.
+The request is then queued onto the destination, and this causes the
+destination to ask its connection pool for a free connection.
+If a connection is available, it is returned, otherwise a new connection is
+created.
+Once the destination has obtained the connection, it dequeues the request
+and sends it over the connection.
+
+The first request to a destination triggers the opening of the first
+connection.
+A second request with the same origin sent _after_ the first request/response
+cycle is completed will reuse the same connection.
+A second request with the same origin sent _concurrently_ with the first
+request will cause the opening of a second connection.
+The configuration parameter `HttpClient.maxConnectionsPerDestination`
+(see also the xref:eg-client-http-configuration[configuration section]) controls
+the max number of connections that can be opened for a destination.
+
+NOTE: If opening connections to a given origin takes a long time, then
+requests for that origin will queue up in the corresponding destination.
+
+Each connection can handle a limited number of concurrent requests.
+For HTTP/1.1, this number is always `1`: there can only be one outstanding
+request for each connection.
+For HTTP/2 this number is determined by the server `max_concurrent_stream`
+setting (typically around `100`, i.e. there can be up to `100` outstanding
+requests for every connection).
+
+When a destination has maxed out its number of connections, and all
+connections have maxed out their number of outstanding requests, more requests
+sent to that destination will be queued.
+When the request queue is full, the request will be failed.
+The configuration parameter `HttpClient.maxRequestsQueuedPerDestination`
+(see also the xref:eg-client-http-configuration[configuration section]) controls
+the max number of requests that can be queued for a destination.

jetty-documentation/src/main/asciidoc/embedded-guide/client/http/client-http-proxy.adoc

@@ -0,0 +1,92 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-client-http-proxy]]
+=== HttpClient Proxy Support
+
+Jetty's `HttpClient` can be configured to use proxies to connect to destinations.
+
+Two types of proxies are available out of the box: a HTTP proxy (provided by
+class `org.eclipse.jetty.client.HttpProxy`) and a SOCKS 4 proxy (provided by
+class `org.eclipse.jetty.client.Socks4Proxy`).
+Other implementations may be written by subclassing `ProxyConfiguration.Proxy`.
+
+The following is a typical configuration:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=proxy]
+----
+
+You specify the proxy host and proxy port, and optionally also the addresses
+that you do not want to be proxied, and then add the proxy configuration on
+the `ProxyConfiguration` instance.
+
+Configured in this way, `HttpClient` makes requests to the HTTP proxy (for
+plain-text HTTP requests) or establishes a tunnel via HTTP `CONNECT` (for
+encrypted HTTPS requests).
+
+Proxying is supported for both HTTP/1.1 and HTTP/2.
+
+[[eg-client-http-proxy-authentication]]
+==== Proxy Authentication Support
+
+Jetty's `HttpClient` supports proxy authentication in the same way it supports
+xref:eg-client-http-authentication[server authentication].
+
+In the example below, the proxy requires `BASIC` authentication, but the server
+requires `DIGEST` authentication, and therefore:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=proxyAuthentication]
+----
+
+The HTTP conversation for successful authentications on both the proxy and the
+server is the following:
+
+[plantuml]
+----
+skinparam backgroundColor transparent
+skinparam monochrome true
+skinparam shadowing false
+
+participant Application
+participant HttpClient
+participant Proxy
+participant Server
+
+Application -> Proxy : GET /path
+Proxy -> HttpClient : 407 + Proxy-Authenticate
+HttpClient -> Proxy : GET /path + Proxy-Authorization
+Proxy -> Server : GET /path
+Server -> Proxy : 401 + WWW-Authenticate
+Proxy -> HttpClient : 401 + WWW-Authenticate
+HttpClient -> Proxy : GET /path + Proxy-Authorization + Authorization
+Proxy -> Server : GET /path + Authorization
+Server -> Proxy : 200 OK
+Proxy -> HttpClient : 200 OK
+HttpClient -> Application : 200 OK
+----
+
+The application does not receive events related to the responses with code 407
+and 401 since they are handled internally by `HttpClient`.
+
+Similarly to the xref:eg-client-http-authentication[authentication section], the
+proxy authentication result and the server authentication result can be
+preempted to avoid, respectively, the 407 and 401 roundtrips.

jetty-documentation/src/main/asciidoc/embedded-guide/client/http/client-http-transport.adoc

@@ -0,0 +1,208 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-client-http-transport]]
+=== HttpClient Pluggable Transports
+
+Jetty's `HttpClient` can be configured to use different transports to carry the
+semantic of HTTP requests and responses.
+
+This means that the intention of a client to request resource `/index.html`
+using the `GET` method can be carried over the network in different formats.
+
+A `HttpClient` transport is the component that is in charge of converting a
+high-level, semantic, HTTP requests such as "`GET` resource ``/index.html``"
+into the specific format understood by the server (for example, HTTP/2), and to
+convert the server response from the specific format (HTTP/2) into high-level,
+semantic objects that can be used by applications.
+
+The most common protocol format is HTTP/1.1, a textual protocol with lines
+separated by `\r\n`:
+
+[source,screen,subs="{sub-order}"]
+----
+GET /index.html HTTP/1.1\r\n
+Host: domain.com\r\n
+...
+\r\n
+----
+
+However, the same request can be made using FastCGI, a binary protocol:
+
+[source,screen,subs="{sub-order}"]
+----
+x01 x01 x00 x01 x00 x08 x00 x00
+x00 x01 x01 x00 x00 x00 x00 x00
+x01 x04 x00 x01 xLL xLL x00 x00
+x0C x0B  D   O   C   U   M   E
+ N   T   _   U   R   I   /   i
+ n   d   e   x   .   h   t   m
+ l
+...
+----
+
+Similarly, HTTP/2 is a binary protocol that transports the same information
+in a yet different format.
+
+A protocol may be _negotiated_ between client and server. A request for a
+resource may be sent using one protocol (for example, HTTP/1.1), but the
+response may arrive in a different protocol (for example, HTTP/2).
+
+`HttpClient` supports 3 static transports, each speaking only one protocol:
+xref:eg-client-http-transport-http11[HTTP/1.1],
+xref:eg-client-http-transport-http2[HTTP/2] and
+xref:eg-client-http-transport-fcgi[FastCGI],
+all of them with 2 variants: clear-text and TLS encrypted.
+
+`HttpClient` also supports one
+xref:eg-client-http-transport-dynamic[dynamic transport],
+that can speak different protocols and can select the right protocol by
+negotiating it with the server or by explicit indication from applications.
+
+Applications are typically not aware of the actual protocol being used.
+This allows them to write their logic against a high-level API that hides the
+details of the specific protocol being used over the network.
+
+[[eg-client-http-transport-http11]]
+==== HTTP/1.1 Transport
+
+HTTP/1.1 is the default transport.
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=defaultTransport]
+----
+
+If you want to customize the HTTP/1.1 transport, you can explicitly configure
+it in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=http11Transport]
+----
+
+[[eg-client-http-transport-http2]]
+==== HTTP/2 Transport
+
+The HTTP/2 transport can be configured in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=http2Transport]
+----
+
+`HTTP2Client` is the lower-level client that provides an API based on HTTP/2
+concepts such as _sessions_, _streams_ and _frames_ that are specific to HTTP/2.
+See xref:eg-client-http2[the HTTP/2 client section] for more information.
+
+`HttpClientTransportOverHTTP2` uses `HTTP2Client` to format high-level semantic
+HTTP requests (like "GET resource /index.html") into the HTTP/2 specific format.
+
+[[eg-client-http-transport-fcgi]]
+==== FastCGI Transport
+
+The FastCGI transport can be configured in this way:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=fcgiTransport]
+----
+
+In order to make requests using the FastCGI transport, you need to have a
+FastCGI server such as https://en.wikipedia.org/wiki/PHP#PHPFPM[PHP-FPM]
+(see also http://php.net/manual/en/install.fpm.php).
+
+The FastCGI transport is primarily used by Jetty's link:#fastcgi[FastCGI support]
+to serve PHP pages (WordPress for example).
+
+[[eg-client-http-transport-dynamic]]
+==== Dynamic Transport
+
+The static transports work well if you know in advance the protocol you want
+to speak with the server, or if the server only supports one protocol (such
+as FastCGI).
+
+With the advent of HTTP/2, however, servers are now able to support multiple
+protocols, at least both HTTP/1.1 and HTTP/2.
+
+The HTTP/2 protocol is typically negotiated between client and server.
+This negotiation can happen via ALPN, a TLS extension that allows the client
+to tell the server the list of protocol that the client supports, so that the
+server can pick one of the client supported protocols that also the server
+supports; or via HTTP/1.1 upgrade by means of the `Upgrade` header.
+
+Applications can configure the dynamic transport with one or more
+_application_ protocols such as HTTP/1.1 or HTTP/2. The implementation will
+take care of using TLS for HTTPS URIs, using ALPN, negotiating protocols,
+upgrading from one protocol to another, etc.
+
+By default, the dynamic transport only speaks HTTP/1.1:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=dynamicDefault]
+----
+
+The dynamic transport can be configured with just one protocol, making it
+equivalent to the corresponding static transport:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=dynamicOneProtocol]
+----
+
+The dynamic transport, however, has been implemented to support multiple
+transports, in particular both HTTP/1.1 and HTTP/2:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=dynamicH1H2]
+----
+
+IMPORTANT: The order in which the protocols are specified to
+`HttpClientTransportDynamic` indicates what is the client preference.
+If the protocol is negotiated via ALPN, it is the server that decides what is
+the protocol to use for the communication, regardless of the client preference.
+If the protocol is not negotiated, the client preference is honored.
+
+Provided that the server supports both HTTP/1.1 and HTTP/2 clear-text, client
+applications can explicitly hint the version they want to use:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http/HTTPClientDocs.java[tag=dynamicClearText]
+----
+
+In case of TLS encrypted communication using the HTTPS scheme, things are a
+little more complicated.
+
+If the client application explicitly specifies the HTTP version, then ALPN
+is not used on the client. By specifying the HTTP version explicitly, the
+client application has prior-knowledge of what HTTP version the server
+supports, and therefore ALPN is not needed.
+If the server does not support the HTTP version chosen by the client, then
+the communication will fail.
+
+If the client application does not explicitly specify the HTTP version,
+then ALPN will be used on the client.
+If the server also supports ALPN, then the protocol will be negotiated via
+ALPN and the server will choose the protocol to use.
+If the server does not support ALPN, the client will try to use the first
+protocol configured in `HttpClientTransportDynamic`, and the communication
+may succeed or fail depending on whether the server supports the protocol
+chosen by the client.

jetty-documentation/src/main/asciidoc/embedded-guide/client/http/client-http.adoc

@@ -16,5 +16,14 @@
 // ========================================================================
 //
 
-[[client-http]]
+[[eg-client-http]]
 === HTTP Client
+
+include::client-http-intro.adoc[]
+include::client-http-api.adoc[]
+include::client-http-configuration.adoc[]
+include::client-http-cookie.adoc[]
+include::client-http-authentication.adoc[]
+include::client-http-proxy.adoc[]
+include::client-http-transport.adoc[]
+

jetty-documentation/src/main/asciidoc/embedded-guide/client/http2/client-http2.adoc

@@ -0,0 +1,322 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-client-http2]]
+=== HTTP/2 Client Library
+
+In the vast majority of cases, client applications should use the generic,
+high-level, xref:eg-client-http[HTTP client library] that also provides
+HTTP/2 support via the pluggable
+xref:eg-client-http-transport-http2[HTTP/2 transport] or the
+xref:eg-client-http-transport-dynamic[dynamic transport].
+
+The high-level HTTP library supports cookies, authentication, redirection,
+connection pooling and a number of other features that are absent in the
+low-level HTTP/2 library.
+
+The HTTP/2 client library has been designed for those applications that need
+low-level access to HTTP/2 features such as _sessions_, _streams_ and
+_frames_, and this is quite a rare use case.
+
+[[eg-client-http2-intro]]
+==== Introducing HTTP2Client
+
+The Maven artifact coordinates for the HTTP/2 client library are the following:
+
+[source,xml,subs="{sub-order}"]
+----
+<dependency>
+  <groupId>org.eclipse.jetty.http2</groupId>
+  <artifactId>http2-client</artifactId>
+  <version>{version}</version>
+</dependency>
+----
+
+The main class is named `org.eclipse.jetty.http2.client.HTTP2Client`, and
+must be created, configured and started before use:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http2/HTTP2ClientDocs.java[tags=start]
+----
+
+When your application stops, or otherwise does not need `HTTP2Client` anymore,
+it should stop the `HTTP2Client` instance (or instances) that were started:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http2/HTTP2ClientDocs.java[tags=stop]
+----
+
+`HTTP2Client` allows client applications to connect to a HTTP/2 server.
+A _session_ represents a single TCP connection to a HTTP/2 server and is defined
+by class `org.eclipse.jetty.http2.api.Session`.
+A _session_ typically has a long life - once the TCP connection is established,
+it remains open until it is not used anymore (and therefore it is closed by
+the idle timeout mechanism), until a fatal error occurs (for example, a network
+failure), or if one of the peers decides unilaterally to close the TCP
+connection.
+
+HTTP/2 is a multiplexed protocol: it allows multiple HTTP/2 requests to be sent
+on the same TCP connection.
+Each request/response cycle is represented by a _stream_.
+Therefore, a single _session_ manages multiple concurrent _streams_.
+A _stream_ has typically a very short life compared to the _session_: a
+_stream_ only exists for the duration of the request/response cycle and then
+disappears.
+
+[[eg-client-http2-flow-control]]
+===== HTTP/2 Flow Control
+
+The HTTP/2 protocol is _flow controlled_ (see
+link:https://tools.ietf.org/html/rfc7540#section-5.2[the specification]).
+This means that a sender and a receiver maintain a _flow control window_ that
+tracks the number of data bytes sent and received, respectively.
+When a sender sends data bytes, it reduces its flow control window. When a
+receiver receives data bytes, it also reduces its flow control window, and
+then passes the received data bytes to the application.
+The application consumes the data bytes and tells back the receiver that it
+has consumed the data bytes.
+The receiver then enlarges the flow control window, and arranges to send a
+message to the sender with the number of bytes consumed, so that the sender
+can enlarge its flow control window.
+
+A sender can send data bytes up to its whole flow control window, then it must
+stop sending until it receives a message from the receiver that the data bytes
+have been consumed, which enlarges the flow control window, which allows the
+sender to send more data bytes.
+
+HTTP/2 defines _two_ flow control windows: one for each _session_, and one
+for each _stream_. Let's see with an example how they interact, assuming that
+in this example the session flow control window is 120 bytes and the stream
+flow control window is 100 bytes.
+
+The sender opens a session, and then opens `stream_1` on that session, and
+sends `80` data bytes.
+At this point the session flow control window is `40` bytes (`120 - 80`), and
+``stream_1``'s flow control window is `20` bytes (`100 - 80`).
+The sender now opens `stream_2` on the same session and sends `40` data bytes.
+At this point, the session flow control window is `0` bytes (`40 - 40`),
+while ``stream_2``'s flow control window is `60` (`100 - 40`).
+Since now the session flow control window is `0`, the sender cannot send more
+data bytes, neither on `stream_1` nor on `stream_2` despite both have their
+stream flow control windows greater than `0`.
+
+The receiver consumes ``stream_2``'s `40` data bytes and sends a message to
+the sender with this information.
+At this point, the session flow control window is `40` (`0 + 40`),
+``stream_1``'s flow control window is still `20` and ``stream_2``'s flow
+control window is `100` (`60 + 40`).
+If the sender opens `stream_3` and would like to send 50 data bytes, it would
+only be able to send `40` because that is the maximum allowed by the session
+flow control window at this point.
+
+It is therefore very important that applications notify the fact that they
+have consumed data bytes as soon as possible, so that the implementation
+(the receiver) can send a message to the sender (in the form of a
+`WINDOW_UPDATE` frame) with the information to enlarge the flow control
+window, therefore reducing the possibility that sender stalls due to the flow
+control windows being reduced to `0`.
+This is discussed in details in xref:eg-client-http2-response[this section].
+
+
+
+
+[[eg-client-http2-connect]]
+==== Connecting to the Server
+
+The first thing an application should do is to connect to the server and
+obtain a `Session`.
+The following example connects to the server on a clear-text port:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http2/HTTP2ClientDocs.java[tags=clearTextConnect]
+----
+
+The following example connects to the server on an encrypted port:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http2/HTTP2ClientDocs.java[tags=encryptedConnect]
+----
+
+IMPORTANT: Applications must know in advance whether they want to connect to a
+clear-text or encrypted port, and pass the `SslContextFactory` parameter
+accordingly to the `connect(...)` method.
+
+[[eg-client-http2-configure]]
+===== Configuring the Session
+
+The `connect(...)` method takes a `Session.Listener` parameter.
+This listener's `onPreface(...)` method is invoked just before establishing the
+connection to the server to gather the client configuration to send to the
+server. Client applications can override this method to change the default
+configuration:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http2/HTTP2ClientDocs.java[tags=configure]
+----
+
+The `Session.Listener` is notified of session events originated by the server
+such as receiving a `SETTINGS` frame from the server, or the server closing
+the connection, or the client timing out the connection due to idleness.
+Please refer to the `Session.Listener`
+link:{JDURL}/org/eclipse/jetty/http2/api/Session.Listener.html[javadocs] for
+the complete list of events.
+
+Once a `Session` has been established, the communication with the server happens
+by exchanging _frames_, as specified in the
+link:https://tools.ietf.org/html/rfc7540#section-4[HTTP/2 specification].
+
+[[eg-client-http2-request]]
+==== Sending a Request
+
+Sending an HTTP request to the server, and receiving a response, creates a
+_stream_ that encapsulates the exchange of HTTP/2 frames that compose the
+request and the response.
+
+In order to send a HTTP request to the server, the client must send a
+`HEADERS` frame.
+`HEADERS` frames carry the request method, the request URI and the request
+headers.
+Sending the `HEADERS` frame opens the `Stream`:
+
+[source,java,indent=0,subs={sub-order}]
+----
+include::../../{doc_code}/embedded/client/http2/HTTP2ClientDocs.java[tags=newStream]
+----
+
+Note how `Session.newStream(...)` takes a `Stream.Listener` parameter.
+This listener is notified of stream events originated by the server such as
+receiving `HEADERS` or `DATA` frames that are part of the response, discussed
+in more details in the xref:eg-client-http2-response[section below].
+Please refer to the `Stream.Listener`
+link:{JDURL}/org/eclipse/jetty/http2/api/Stream.Listener.html[javadocs] for
+the complete list of events.
+
+HTTP requests may have content, which is sent using the `Stream` APIs:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http2/HTTP2ClientDocs.java[tags=newStreamWithData]
+----
+
+IMPORTANT: When sending two `DATA` frames consecutively, the second call to
+`Stream.data(...)` must be done only when the first is completed, or a
+`WritePendingException` will be thrown.
+Use the `Callback` APIs or `CompletableFuture` APIs to ensure that the second
+`Stream.data(...)` call is performed when the first completed successfully.
+
+[[eg-client-http2-response]]
+==== Receiving a Response
+
+Response events are delivered to the `Stream.Listener` passed to
+`Session.newStream(...)`.
+
+A HTTP response is typically composed of a `HEADERS` frame containing the HTTP
+status code and the response headers, and optionally one or more `DATA` frames
+containing the response content bytes.
+
+The HTTP/2 protocol also supports response trailers (that is, headers that are
+sent after the response content) that also are sent using a `HEADERS` frame.
+
+A client application can therefore receive the HTTP/2 frames sent by the server
+by implementing the relevant methods in `Stream.Listener`:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http2/HTTP2ClientDocs.java[tags=responseListener]
+----
+
+NOTE: Returning from the `onData(...)` method implicitly demands for
+more `DATA` frames (unless the one just delivered was the last).
+Additional `DATA` frames may be delivered immediately if they are available
+or later, asynchronously, when they arrive.
+
+Client applications that consume the content buffer within `onData(...)`
+(for example, writing it to a file, or copying the bytes to another storage)
+should succeed the callback as soon as they have consumed the content buffer.
+This allows the implementation to reuse the buffer, reducing the memory
+requirements needed to handle the response content.
+
+Alternatively, a client application may store away _both_ the buffer and the
+callback to consume the buffer bytes later.
+
+IMPORTANT: Completing the `Callback` is very important not only to allow the
+implementation to reuse the buffer, but also tells the implementation to
+enlarge the stream and session flow control windows so that the server will
+be able to send more `DATA` frames without stalling.
+
+Client applications can also precisely control _when_ to demand more `DATA`
+frames, by implementing the `onDataDemanded(...)` method instead of
+`onData(...)`:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http2/HTTP2ClientDocs.java[tags=responseDataDemanded]
+----
+
+IMPORTANT: Applications that implement `onDataDemanded(...)` must remember
+to call `Stream.demand(...)`. If they don't, the implementation will not
+deliver `DATA` frames and the application will stall threadlessly until an
+idle timeout fires to close the stream or the session.
+
+[[eg-client-http2-reset]]
+==== Resetting a Request or Response
+
+In HTTP/2, clients and servers have the ability to tell to the other peer that
+they are not interested anymore in either the request or the response, using a
+`RST_STREAM` frame.
+
+The `HTTP2Client` APIs allow client applications to send and receive this
+"reset" frame:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http2/HTTP2ClientDocs.java[tags=reset]
+----
+
+[[eg-client-http2-push]]
+==== Receiving HTTP/2 Pushes
+
+HTTP/2 servers have the ability to push resources related to a primary
+resource.
+When a HTTP/2 server pushes a resource, it send to the client a `PUSH_PROMISE`
+frame that contains the request URI and headers that a client would use to
+request explicitly that resource.
+
+Client applications can be configured to tell the server to never push
+resources, see xref:eg-client-http2-configure[this section].
+
+Client applications can listen to the push events, and act accordingly:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http2/HTTP2ClientDocs.java[tags=push]
+----
+
+If a client application does not want to handle a particular HTTP/2 push, it
+can just reset the pushed stream to tell the server to stop sending bytes for
+the pushed stream:
+
+[source,java,indent=0]
+----
+include::../../{doc_code}/embedded/client/http2/HTTP2ClientDocs.java[tags=pushReset]
+----

jetty-documentation/src/main/asciidoc/embedded-guide/index.adoc

@@ -58,6 +58,7 @@ endif::[]
 // suppress automatic id generation
 :sectids!:
 
+include::.asciidoctorconfig[]
 include::client/client.adoc[]
 include::server/server.adoc[]
 include::io-arch.adoc[]

jetty-documentation/src/main/asciidoc/embedded-guide/io-arch.adoc

@@ -40,26 +40,26 @@ NOTE: TODO: add image
 to a server and by a network server when accepting connections from network
 clients.
 In both cases the `SocketChannel` instance is passed to `SelectorManager`
-(and to `ManagedSelector` and eventually to `java.nio.channels.Selector`)
+(which passes it to `ManagedSelector` and eventually to
-to be registered for use within Jetty.
+`java.nio.channels.Selector`) to be registered for use within Jetty.
 
-It is therefore possible for an application to create the `SocketChannel`
+It is possible for an application to create the `SocketChannel`
 instances outside Jetty, even perform some initial network traffic also
 outside Jetty (for example for authentication purposes), and then pass the
 `SocketChannel` instance to `SelectorManager` for use within Jetty.
 
-This example shows how to connect to a server:
+This example shows how a client can connect to a server:
 
 [source,java,indent=0]
 ----
-include::{docbits}/embedded/SelectorManagerDocSnippets.java[tags=connect]
+include::{doc_code}/embedded/SelectorManagerDocs.java[tags=connect]
 ----
 
-This example shows how to accept a client connection:
+This example shows how a server accepts a client connection:
 
 [source,java,indent=0]
 ----
-include::{docbits}/embedded/SelectorManagerDocSnippets.java[tags=accept]
+include::{doc_code}/embedded/SelectorManagerDocs.java[tags=accept]
 ----
 
 [[io-arch-endpoint-connection]]
@@ -101,9 +101,10 @@ The `EndPoint` and `Connection` pairs can be chained, for example in case of
 encrypted communication using the TLS protocol.
 There is an `EndPoint` and `Connection` TLS pair where the `EndPoint` reads the
 encrypted bytes from the network and the `Connection` decrypts them; next in the
-chain there is an `EndPoint` and `Connection` pair where the `EndPoint` provides
+chain there is an `EndPoint` and `Connection` pair where the `EndPoint` "reads"
-decrypted bytes and the `Connection` deserializes them into specific protocol
+decrypted bytes (provided by the previous `Connection`) and the `Connection`
-objects (for example a HTTP/1.1 request object).
+deserializes them into specific protocol objects (for example HTTP/2 frame
+objects).
 
 Certain protocols, such as WebSocket, start the communication with the server
 using one protocol (e.g. HTTP/1.1), but then change the communication to use
@@ -145,7 +146,7 @@ The Jetty I/O library use Java NIO to handle I/O, so that I/O is non-blocking.
 At the Java NIO level, in order to be notified when a `SocketChannel` has data
 to be read, the `SelectionKey.OP_READ` flag must be set.
 
-In the Jetty I/O library, you can call `AbstractEndPoint.fillInterested(Callback)`
+In the Jetty I/O library, you can call `EndPoint.fillInterested(Callback)`
 to declare interest in the "read" (or "fill") event, and the `Callback` parameter
 is the object that is notified when such event occurs.
 
@@ -155,12 +156,110 @@ is therefore writable again, the `SelectionKey.OP_WRITE` flag must be set.
 
 In the Jetty I/O library, you can call `EndPoint.write(Callback, ByteBuffer...)`
 to write the ``ByteBuffer``s and the `Callback` parameter is the object that is
-notified when the whole write is finished(i.e. _all_ ``ByteBuffer``s have been
+notified when the whole write is finished (i.e. _all_ ``ByteBuffer``s have been
-fully written).
+fully written, even if they are delayed by TCP congestion/uncongestion).
+
+The `EndPoint` APIs abstract out the Java NIO details by providing non-blocking
+APIs based on `Callback` objects for I/O operations.
+The `EndPoint` APIs are typically called by `Connection` implementations, see
+link:#io-arch-connection[this section].
 
 [[io-arch-connection]]
-=== Jetty I/O: Implementing `Connection`
+=== Jetty I/O: `Connection`
 
-Implementing a `Connection` is how you deserialize incoming bytes into objects
+`Connection` is the abstraction that deserializes incoming bytes into objects,
-that can be used by more abstract layers, for example a HTTP request object or
+for example a HTTP request object or a WebSocket frame object, that can be used
-a WebSocket frame object.
+by more abstract layers.
+
+`Connection` instances have two lifecycle methods:
+
+* `Connection.onOpen()`, invoked when the `Connection` is associated with the
+`EndPoint`
+* `Connection.onClose(Throwable)`, invoked when the `Connection` is disassociated
+from the `EndPoint`, where the `Throwable` parameter indicates whether the
+disassociation was normal (when the parameter is `null`) or was due to an error
+(when the parameter is not `null`)
+
+When a `Connection` is first created, it is not registered for any Java NIO
+event.
+It is therefore typical to implement `onOpen()` to call
+`EndPoint.fillInterested(Callback)` so that the `Connection` declares interest
+for read events and it is invoked (via the `Callback`) when the read event
+happens.
+
+Abstract class `AbstractConnection` partially implements `Connection` and
+provides simpler APIs. The example below shows a typical implementation that
+extends `AbstractConnection`:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/SelectorManagerDocs.java[tags=connection]
+----
+
+[[io-arch-echo]]
+=== Jetty I/O: Network Echo
+
+With the concepts above it is now possible to write a simple, fully non-blocking,
+`Connection` implementation that simply echoes the bytes that it reads back
+to the other peer.
+
+A naive, but wrong, implementation may be the following:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/SelectorManagerDocs.java[tags=echo-wrong]
+----
+
+WARNING: The implementation above is wrong and leads to `StackOverflowError`.
+
+The problem with this implementation is that if the writes always complete
+synchronously (i.e. without being delayed by TCP congestion), you end up with
+this sequence of calls:
+
+----
+Connection.onFillable()
+  EndPoint.write()
+    Callback.succeeded()
+      Connection.onFillable()
+        EndPoint.write()
+          Callback.succeeded()
+          ...
+----
+
+which leads to `StackOverflowError`.
+
+This is a typical side effect of asynchronous programming using non-blocking
+APIs, and happens in the Jetty I/O library as well.
+
+NOTE: The callback is invoked synchronously for efficiency reasons.
+Submitting the invocation of the callback to an `Executor` to be invoked in
+a different thread would cause a context switch and make simple writes
+extremely inefficient.
+
+A correct implementation is the following:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/SelectorManagerDocs.java[tags=echo-correct]
+----
+
+The correct implementation performs consecutive reads in a loop (rather than
+recursively), but _only_ if the correspondent write is completed successfully.
+
+In order to detect whether the write is completed, a concurrent state machine
+is used. This is necessary because the notification of the completion of the
+write may happen in a different thread, while the original writing thread
+may still be changing the state.
+
+The original writing thread starts moves the state from `IDLE` to `WRITING`,
+then issues the actual `write()` call.
+The original writing thread then assumes that the `write()` did not complete
+and tries to move to the `PENDING` state just after the `write()`.
+If it fails to move from the `WRITING` state to the `PENDING` state, it means
+that the write was completed.
+Otherwise, the write is now `PENDING` and waiting for the callback to be
+notified of the completion at a later time.
+When the callback is notified of the `write()` completion, it checks whether
+the `write()` was `PENDING`, and if it was it resumes reading.
+
+NOTE: TODO: Introduce IteratingCallback?

jetty-documentation/src/main/asciidoc/embedded-guide/server/clients/http/http-client-api.adoc

@@ -1,381 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-[[http-client-api]]
-=== API Usage
-
-[[http-client-blocking]]
-==== Blocking APIs
-
-The simple way to perform a HTTP request is the following:
-
-[source, java, subs="{sub-order}"]
-----
-ContentResponse response = httpClient.GET("http://domain.com/path?query");
-----
-
-The method `HttpClient.GET(...)` performs a HTTP `GET` request to the given URI and returns a `ContentResponse` when the request/response conversation completes successfully.
-
-The `ContentResponse` object contains the HTTP response information: status code, headers and possibly content.
-The content length is limited by default to 2 MiB; for larger content see xref:http-client-response-content[].
-
-If you want to customize the request, for example by issuing a `HEAD` request instead of a `GET`, and simulating a browser user agent, you can do it in this way:
-
-[source, java, subs="{sub-order}"]
-----
-ContentResponse response = httpClient.newRequest("http://domain.com/path?query")
-        .method(HttpMethod.HEAD)
-        .agent("Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0")
-        .send();
-----
-
-This is a shorthand for:
-
-[source, java, subs="{sub-order}"]
-----
-Request request = httpClient.newRequest("http://domain.com/path?query");
-request.method(HttpMethod.HEAD);
-request.agent("Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0");
-ContentResponse response = request.send();
-----
-
-You first create a request object using `httpClient.newRequest(...)`, and then you customize it using the fluent API style (that is, a chained invocation of methods on the request object).
-When the request object is customized, you call `request.send()` that produces the `ContentResponse` when the request/response conversation is complete.
-
-Simple `POST` requests also have a shortcut method:
-
-[source, java, subs="{sub-order}"]
-----
-ContentResponse response = httpClient.POST("http://domain.com/entity/1")
-        .param("p", "value")
-        .send();
-----
-
-The `POST` parameter values added via the `param()` method are automatically URL-encoded.
-
-Jetty's HTTP client automatically follows redirects, so it handles the typical web pattern http://en.wikipedia.org/wiki/Post/Redirect/Get[POST/Redirect/GET], and the response object contains the content of the response of the `GET` request.
-Following redirects is a feature that you can enable/disable on a per-request basis or globally.
-
-File uploads also require one line, and make use of JDK 7′s `java.nio.file` classes:
-
-[source, java, subs="{sub-order}"]
-----
-ContentResponse response = httpClient.newRequest("http://domain.com/upload")
-        .method(HttpMethod.POST)
-        .file(Paths.get("file_to_upload.txt"), "text/plain")
-        .send();
-----
-
-It is possible to impose a total timeout for the request/response conversation using the `Request.timeout(...)` method as follows:
-
-[source, java, subs="{sub-order}"]
-----
-ContentResponse response = httpClient.newRequest("http://domain.com/path?query")
-        .timeout(5, TimeUnit.SECONDS)
-        .send();
-----
-
-In the example above, when the 5 seconds expire, the request is aborted and a `java.util.concurrent.TimeoutException` is thrown.
-
-[[http-client-async]]
-==== Non-Blocking APIs
-
-So far we have shown how to use Jetty HTTP client in a blocking style - that is, the thread that issues the request blocks until the request/response conversation is complete.
-
-This section will look at Jetty's HTTP client non-blocking, asynchronous APIs that are perfectly suited for large content downloads, for parallel processing of requests/responses and in cases where performance and efficient thread and resource utilization is a key factor.
-
-The asynchronous APIs rely heavily on listeners that are invoked at various stages of request and response processing.
-These listeners are implemented by applications and may perform any kind of logic.
-The implementation invokes these listeners in the same thread that is used to process the request or response.
-Therefore, if the application code in these listeners takes a long time to execute, the request or response processing is delayed until the listener returns.
-
-If you need to execute application code that takes long time inside a listener, you must spawn your own thread and remember to deep copy any data provided by the listener that you will need in your code, because when the listener returns the data it provides may be recycled/cleared/destroyed.
-
-Request and response processing are executed by two different threads and therefore may happen concurrently.
-A typical example of this concurrent processing is an echo server, where a large upload may be concurrent with the large download echoed back.
-As a side note, remember that responses may be processed and completed _before_ requests; a typical example is a large upload that triggers a quick response - for example an error - by the server: the response may arrive and be completed while the request content is still being uploaded.
-
-The application thread that calls `Request.send(Response.CompleteListener)` performs the processing of the request until either the request is fully processed or until it would block on I/O, then it returns (and therefore never blocks).
-If it would block on I/O, the thread asks the I/O system to emit an event when the I/O will be ready to continue, then returns.
-When such an event is fired, a thread taken from the `HttpClient` thread pool will resume the processing of the request.
-
-Response are processed from the I/O thread that fires the event that bytes are ready to be read.
-Response processing continues until either the response is fully processed or until it would block for I/O.
-If it would block for I/O, the thread asks the I/O system to emit an event when the I/O will be ready to continue, then returns.
-When such an event is fired, a thread taken from the `HttpClient` thread pool will resume the processing of the response.
-
-When the request and the response are both fully processed, the thread that finished the last processing (usually the thread that processes the response, but may also be the thread that processes the request - if the request takes more time than the response to be processed) is used to de-queue the next request for the same destination and processes it.
-
-A simple asynchronous `GET` request that discards the response content can be written in this way:
-
-[source, java, subs="{sub-order}"]
-----
-httpClient.newRequest("http://domain.com/path")
-        .send(new Response.CompleteListener()
-        {
-            @Override
-            public void onComplete(Result result)
-            {
-                // Your logic here
-            }
-        });
-----
-
-Method `Request.send(Response.CompleteListener)` returns `void` and does not block; the `Response.CompleteListener` provided as a parameter is notified when the request/response conversation is complete, and the `Result` parameter allows you to access the response object.
-
-You can write the same code using JDK 8′s lambda expressions:
-
-[source, java, subs="{sub-order}"]
-----
-httpClient.newRequest("http://domain.com/path")
-        .send(result -> { /* Your logic here */ });
-----
-
-You can impose a total timeout for the request/response conversation in the same way used by the synchronous API:
-
-[source, java, subs="{sub-order}"]
-----
-httpClient.newRequest("http://domain.com/path")
-        .timeout(3, TimeUnit.SECONDS)
-        .send(result -> { /* Your logic here */ });
-----
-
-The example above will impose a total timeout of 3 seconds on the request/response conversation.
-
-The HTTP client APIs use listeners extensively to provide hooks for all possible request and response events, and with JDK 8′s lambda expressions they are even more fun to use:
-
-[source, java, subs="{sub-order}"]
-----
-httpClient.newRequest("http://domain.com/path")
-        // Add request hooks
-        .onRequestQueued(request -> { ... })
-        .onRequestBegin(request -> { ... })
-        ... // More request hooks available
-
-        // Add response hooks
-        .onResponseBegin(response -> { ... })
-        .onResponseHeaders(response -> { ... })
-        .onResponseContent((response, buffer) -> { ... })
-        ... // More response hooks available
-
-        .send(result -> { ... });
-----
-
-This makes Jetty HTTP client suitable for HTTP load testing because, for example, you can accurately time every step of the request/response conversation (thus knowing where the request/response time is really spent).
-
-Have a look at the link:{JDURL}/org/eclipse/jetty/client/api/Request.Listener.html[`Request.Listener`] class to know about request events, and to the link:{JDURL}/org/eclipse/jetty/client/api/Response.Listener.html[`Response.Listener`] class to know about response events.
-
-[[http-client-content]]
-==== Content Handling
-
-[[http-client-request-content]]
-===== Request Content Handling
-
-Jetty's HTTP client provides a number of utility classes off the shelf to handle request content.
-
-You can provide request content as `String`, `byte[]`, `ByteBuffer`, `java.nio.file.Path`, `InputStream`, and provide your own implementation of `org.eclipse.jetty.client.api.ContentProvider`.
-Here’s an example that provides the request content using `java.nio.file.Paths`:
-
-[source, java, subs="{sub-order}"]
-----
-ContentResponse response = httpClient.newRequest("http://domain.com/upload")
-        .method(HttpMethod.POST)
-        .file(Paths.get("file_to_upload.txt"), "text/plain")
-        .send();
-----
-
-This is equivalent to using the `PathContentProvider` utility class:
-
-[source, java, subs="{sub-order}"]
-----
-ContentResponse response = httpClient.newRequest("http://domain.com/upload")
-        .method(HttpMethod.POST)
-        .content(new PathContentProvider(Paths.get("file_to_upload.txt")), "text/plain")
-        .send();
-----
-
-Alternatively, you can use `FileInputStream` via the `InputStreamContentProvider` utility class:
-
-[source, java, subs="{sub-order}"]
-----
-ContentResponse response = httpClient.newRequest("http://domain.com/upload")
-        .method(HttpMethod.POST)
-        .content(new InputStreamContentProvider(new FileInputStream("file_to_upload.txt")), "text/plain")
-        .send();
-----
-
-Since `InputStream` is blocking, then also the send of the request will block if the input stream blocks, even in case of usage of the asynchronous `HttpClient` APIs.
-
-If you have already read the content in memory, you can pass it as a `byte[]` using the `BytesContentProvider` utility class:
-
-[source, java, subs="{sub-order}"]
-----
-byte[] bytes = ...;
-ContentResponse response = httpClient.newRequest("http://domain.com/upload")
-        .method(HttpMethod.POST)
-        .content(new BytesContentProvider(bytes), "text/plain")
-        .send();
-----
-
-If the request content is not immediately available, but your application will be notified of the content to send, you can use `DeferredContentProvider` in this way:
-
-[source, java, subs="{sub-order}"]
-----
-DeferredContentProvider content = new DeferredContentProvider();
-httpClient.newRequest("http://domain.com/upload")
-        .method(HttpMethod.POST)
-        .content(content)
-        .send(new Response.CompleteListener()
-        {
-            @Override
-            public void onComplete(Result result)
-            {
-                // Your logic here
-            }
-        });
-
-// Content not available yet here
-
-...
-
-// An event happens, now content is available
-byte[] bytes = ...;
-content.offer(ByteBuffer.wrap(bytes));
-
-...
-
-// All content has arrived
-content.close();
-----
-
-While the request content is awaited and consequently uploaded by the client application, the server may be able to respond (at least with the response headers) completely asynchronously.
-In this case, `Response.Listener` callbacks will be invoked before the request is fully sent.
-This allows fine-grained control of the request/response conversation: for example the server may reject contents that are too big, send a response to the client, which in turn may stop the content upload.
-
-Another way to provide request content is by using an `OutputStreamContentProvider`,
-which allows applications to write request content when it is available to the `OutputStream` provided by `OutputStreamContentProvider`:
-
-[source, java, subs="{sub-order}"]
-----
-OutputStreamContentProvider content = new OutputStreamContentProvider();
-
-// Use try-with-resources to close the OutputStream when all content is written
-try (OutputStream output = content.getOutputStream())
-{
-    client.newRequest("localhost", 8080)
-            .method(HttpMethod.POST)
-            .content(content)
-            .send(new Response.CompleteListener()
-            {
-                @Override
-                public void onComplete(Result result)
-                {
-                    // Your logic here
-                }
-            });
-
-    ...
-
-    // Write content
-    writeContent(output);
-}
-// End of try-with-resource, output.close() called automatically to signal end of content
-----
-
-[[http-client-response-content]]
-===== Response Content Handling
-
-Jetty HTTP client allows applications to handle response content in different ways.
-
-The first way is to buffer the response content in memory; this is done when using the blocking APIs (see xref:http-client-blocking[]) and the content is buffered within a `ContentResponse` up to 2 MiB.
-
-If you want to control the length of the response content (for example limiting to values smaller than the default of 2 MiB), then you can use a `org.eclipse.jetty.client.util.FutureResponseListener` in this way:
-
-[source, java, subs="{sub-order}"]
-----
-Request request = httpClient.newRequest("http://domain.com/path");
-
-// Limit response content buffer to 512 KiB
-FutureResponseListener listener = new FutureResponseListener(request, 512 * 1024);
-
-request.send(listener);
-
-ContentResponse response = listener.get(5, TimeUnit.SECONDS);
-----
-
-If the response content length is exceeded, the response will be aborted, and an exception will be thrown by method `get()`.
-
-If you are using the asynchronous APIs (see xref:http-client-async[]), you can use the `BufferingResponseListener` utility class:
-
-[source, java, subs="{sub-order}"]
-----
-httpClient.newRequest("http://domain.com/path")
-        // Buffer response content up to 8 MiB
-        .send(new BufferingResponseListener(8 * 1024 * 1024)
-        {
-            @Override
-            public void onComplete(Result result)
-            {
-                if (!result.isFailed())
-                {
-                    byte[] responseContent = getContent();
-                    // Your logic here
-                }
-            }
-        });
-----
-
-The second way is the most efficient (because it avoids content copies) and allows you to specify a `Response.ContentListener`, or a subclass, to handle the content as soon as it arrives.
-In the example below, `Response.Listener.Adapter` is a class that implements both `Response.ContentListener` and `Response.CompleteListener` and can be passed to `Request.send()`.
-Jetty's HTTP client will invoke the `onContent()` method zero or more times (until there is content), and finally invoke the `onComplete()` method.
-
-[source, java, subs="{sub-order}"]
-----
-httpClient .newRequest("http://domain.com/path")
-        .send(new Response.Listener.Adapter()
-        {
-            @Override
-            public void onContent(Response response, ByteBuffer buffer)
-            {
-                // Your logic here
-            }
-        });
-----
-
-The third way allows you to wait for the response and then stream the content using the `InputStreamResponseListener` utility class:
-
-[source, java, subs="{sub-order}"]
-----
-
-InputStreamResponseListener listener = new InputStreamResponseListener();
-httpClient.newRequest("http://domain.com/path")
-        .send(listener);
-
-// Wait for the response headers to arrive
-Response response = listener.get(5, TimeUnit.SECONDS);
-
-// Look at the response
-if (response.getStatus() == HttpStatus.OK_200)
-{
-    // Use try-with-resources to close input stream.
-    try (InputStream responseContent = listener.getInputStream())
-    {
-        // Your logic here
-    }
-}
-----

jetty-documentation/src/main/asciidoc/embedded-guide/server/clients/http/http-client-authentication.adoc

@@ -1,91 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-[[http-client-authentication]]
-=== Authentication Support
-
-Jetty's HTTP client supports the `BASIC` and `DIGEST` authentication mechanisms defined by link:https://tools.ietf.org/html/rfc7235[RFC 7235].
-
-You can configure authentication credentials in the HTTP client instance as follows:
-
-[source, java, subs="{sub-order}"]
-----
-URI uri = new URI("http://domain.com/secure");
-String realm = "MyRealm";
-String user = "username";
-String pass = "password";
-
-// Add authentication credentials
-AuthenticationStore auth = httpClient.getAuthenticationStore();
-auth.addAuthentication(new BasicAuthentication(uri, realm, user, pass));
-
-ContentResponse response = httpClient
-        .newRequest(uri)
-        .send()
-        .get(5, TimeUnit.SECONDS);
-----
-
-Jetty's HTTP client tests authentication credentials against the challenge(s) the server issues (see our section here on link:#configuring-security-secure-passwords[secure password obfuscation]), and if they match it automatically sends the right authentication headers to the server for authentication.
-If the authentication is successful, it caches the result and reuses it for subsequent requests for the same domain and matching URIs.
-
-The HTTP conversation for a successful match is the following:
-
-----
-Application  HttpClient                     Server
-     |           |                             |
-     |--- GET ---|------------ GET ----------->|
-     |           |                             |
-     |           |<-- 401 + WWW-Authenticate --|
-     |           |                             |
-     |           |--- GET + Authentication --->|
-     |           |                             |
-     |<-- 200 ---|------------ 200 ------------|
-----
-
-The application does not receive events related to the response with code 401, they are handled internally by `HttpClient` which produces a request similar to the original but with the correct `Authorization` header, and then relays the response with code 200 to the application.
-
-Successful authentications are cached, but it is possible to clear them in order to force authentication again:
-
-[source, java, subs="{sub-order}"]
-----
-httpClient.getAuthenticationStore().clearAuthenticationResults();
-----
-
-Authentications may be preempted to avoid the additional roundtrip due to the server challenge in this way:
-
-[source, java, subs="{sub-order}"]
-----
-AuthenticationStore auth = httpClient.getAuthenticationStore();
-URI uri = URI.create("http://domain.com/secure");
-auth.addAuthenticationResult(new BasicAuthentication.BasicResult(uri, "username", "password"));
-----
-
-In this way, requests for the given URI are enriched by `HttpClient` immediately with the `Authorization` header, and the server should respond with a 200 and the resource content rather than with the 401 and the challenge.
-
-It is also possible to preempt the authentication for a single request only, in this way:
-
-[source, java, subs="{sub-order}"]
-----
-URI uri = URI.create("http://domain.com/secure");
-Authentication.Result authn = new BasicAuthentication.BasicResult(uri, "username", "password")
-Request request = httpClient.newRequest(uri);
-authn.apply(request);
-request.send();
-----
-
-See also the link:#http-client-proxy-authentication[proxy authentication section] for further information about how authentication works with HTTP proxies.

jetty-documentation/src/main/asciidoc/embedded-guide/server/clients/http/http-client-intro.adoc

@@ -1,105 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-[[http-client-intro]]
-=== Introduction
-
-The Jetty HTTP client module provides easy-to-use APIs and utility classes to perform HTTP (or HTTPS) requests.
-
-Jetty's HTTP client is non-blocking and asynchronous.
-It offers an asynchronous API that never blocks for I/O, making it very efficient in thread utilization and well suited for high performance scenarios such as load testing or parallel computation.
-
-However, when all you need to do is to perform a `GET` request to a resource, Jetty's HTTP client offers also a synchronous API; a programming interface
-where the thread that issued the request blocks until the request/response conversation is complete.
-
-Jetty's HTTP client supports different link:#http-client-transport[transports]: HTTP/1.1, FastCGI and HTTP/2.
-This means that the semantic of a HTTP request (that is, " `GET` me the resource `/index.html` ") can be carried over the network in different formats.
-The most common and default format is HTTP/1.1.
-That said, Jetty's HTTP client can carry the same request using the FastCGI format or the new HTTP/2 format.
-
-The FastCGI transport is heavily used in Jetty's link:#fastcgi[FastCGI support] that allows Jetty to work as a reverse proxy to PHP (exactly like Apache or Nginx do) and therefore be able to serve - for example - WordPress websites.
-
-The HTTP/2 transport allows Jetty's HTTP client to perform requests using HTTP/2 to HTTP/2 enabled web sites, see also Jetty's link:#http2[HTTP/2 support].
-
-Out of the box features that you get with the Jetty HTTP client include:
-
-* Redirect support - redirect codes such as 302 or 303 are automatically followed.
-* Cookies support - cookies sent by servers are stored and sent back to servers in matching requests.
-* Authentication support - HTTP "Basic" and "Digest" authentications are supported, others are pluggable.
-* Forward proxy support - HTTP proxying and SOCKS4 proxying.
-
-[[http-client-init]]
-==== Starting HttpClient
-
-The main class is named `org.eclipse.jetty.client.HttpClient`.
-
-You can think of a `HttpClient` instance as a browser instance.
-Like a browser it can make requests to different domains, it manages redirects, cookies and authentication, you can configure it with a proxy, and
-it provides you with the responses to the requests you make.
-
-In order to use `HttpClient`, you must instantiate it, configure it, and then start it:
-
-[source, java, subs="{sub-order}"]
-----
-// Instantiate HttpClient
-HttpClient httpClient = new HttpClient();
-
-// Configure HttpClient, for example:
-httpClient.setFollowRedirects(false);
-
-// Start HttpClient
-httpClient.start();
-----
-
-You may create multiple instances of `HttpClient`, but typically one instance is enough for an application.
-There are several reasons for having multiple `HttpClient` instances including, but not limited to:
-
-* You want to specify different configuration parameters (for example, one instance is configured with a forward proxy while another is not)
-* You want the two instances to behave like two different browsers and hence have different cookies, different authentication credentials...etc.
-* You want to use different transports
-
-When you create a `HttpClient` instance using the parameterless constructor, you will only be able to perform plain HTTP requests and you will not be able to perform HTTPS requests.
-
-In order to perform HTTPS requests, you should create first a link:{JDURL}/org/eclipse/jetty/util/ssl/SslContextFactory.Client.html[`SslContextFactory.Client`], configure it, and pass it to the `HttpClient` constructor.
-When created with a `SslContextFactory`, the `HttpClient` will be able to perform both HTTP and HTTPS requests to any domain.
-
-[source, java, subs="{sub-order}"]
-----
-// Instantiate and configure the SslContextFactory
-SslContextFactory.Client sslContextFactory = new SslContextFactory.Client();
-
-// Instantiate HttpClient with the SslContextFactory
-HttpClient httpClient = new HttpClient(sslContextFactory);
-
-// Configure HttpClient, for example:
-httpClient.setFollowRedirects(false);
-
-// Start HttpClient
-httpClient.start();
-----
-
-==== Stopping HttpClient
-
-It is recommended that when your application stops, you also stop the `HttpClient` instance (or instances) that you are using.
-
-[source, java, subs="{sub-order}"]
-----
-httpClient.stop();
-----
-
-Stopping `HttpClient` makes sure that the memory it holds (for example, authentication credentials, cookies, etc.) is released, and that the thread pool and scheduler are properly stopped allowing all threads used by `HttpClient` to exit.

jetty-documentation/src/main/asciidoc/embedded-guide/server/clients/http/http-client-proxy.adoc

@@ -1,103 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-[[http-client-proxy]]
-=== Proxy Support
-
-Jetty's HTTP client can be configured to use proxies to connect to destinations.
-
-Two types of proxies are available out of the box: a HTTP proxy (provided by class `org.eclipse.jetty.client.HttpProxy`) and a SOCKS 4 proxy (provided by class `org.eclipse.jetty.client.Socks4Proxy`).
-Other implementations may be written by subclassing `ProxyConfiguration.Proxy`.
-
-The following is a typical configuration:
-
-[source, java, subs="{sub-order}"]
-----
-ProxyConfiguration proxyConfig = httpClient.getProxyConfiguration();
-HttpProxy proxy = new HttpProxy("proxyHost", proxyPort);
-
-// Do not proxy requests for localhost:8080
-proxy.getExcludedAddresses().add("localhost:8080");
-
-// add the new proxy to the list of proxies already registered
-proxyConfig.getProxies().add(proxy);
-
-ContentResponse response = httpClient.GET(uri);
-----
-
-You specify the proxy host and port, and optionally also the addresses that you do not want to be proxied, and then add the proxy configuration on the `ProxyConfiguration` instance.
-
-Configured in this way, `HttpClient` makes requests to the HTTP proxy (for plain-text HTTP requests) or establishes a tunnel via `HTTP CONNECT` (for encrypted HTTPS requests).
-
-[[http-client-proxy-authentication]]
-==== Proxy Authentication Support
-
-Jetty's HTTP client support proxy authentication in the same way it supports link:#http-client-authentication[server authentication].
-
-In the example below, the proxy requires Basic authentication, but the server requires Digest authentication, and therefore:
-
-[source, java, subs="{sub-order}"]
-----
-URI proxyURI = new URI("http://proxy.net:8080");
-URI serverURI = new URI("http://domain.com/secure");
-
-AuthenticationStore auth = httpClient.getAuthenticationStore();
-
-// Proxy credentials.
-auth.addAuthentication(new BasicAuthentication(proxyURI, "ProxyRealm", "proxyUser", "proxyPass"));
-
-// Server credentials.
-auth.addAuthentication(new DigestAuthentication(serverURI, "ServerRealm", "serverUser", "serverPass"));
-
-// Proxy configuration.
-ProxyConfiguration proxyConfig = httpClient.getProxyConfiguration();
-HttpProxy proxy = new HttpProxy("proxy.net", 8080);
-proxyConfig.getProxies().add(proxy);
-
-ContentResponse response = httpClient.newRequest(serverURI)
-        .send()
-        .get(5, TimeUnit.SECONDS);
-----
-
-The HTTP conversation for successful authentications on both the proxy and the server is the following:
-
-----
-Application  HttpClient                         Proxy                    Server
-     |           |                                |                         |
-     |--- GET -->|------------- GET ------------->|                         |
-     |           |                                |                         |
-     |           |<----- 407 + Proxy-Authn -------|                         |
-     |           |                                |                         |
-     |           |------ GET + Proxy-Authz ------>|                         |
-     |           |                                |                         |
-     |           |                                |---------- GET --------->|
-     |           |                                |                         |
-     |           |                                |<--- 401 + WWW-Authn ----|
-     |           |                                |                         |
-     |           |<------ 401 + WWW-Authn --------|                         |
-     |           |                                |                         |
-     |           |-- GET + Proxy-Authz + Authz -->|                         |
-     |           |                                |                         |
-     |           |                                |------ GET + Authz ----->|
-     |           |                                |                         |
-     |<-- 200 ---|<------------ 200 --------------|<--------- 200 ----------|
-----
-
-The application does not receive events related to the responses with code 407 and 401 since they are handled internally by `HttpClient`.
-
-Similarly to the link:#http-client-authentication[authentication section], the proxy authentication result and the server authentication result can be preempted to avoid, respectively, the 407 and 401 roundtrips.

jetty-documentation/src/main/asciidoc/embedded-guide/server/clients/http/http-client-transport.adoc

@@ -1,115 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-[[http-client-transport]]
-=== Pluggable Transports
-
-Jetty's HTTP client can be configured to use different transports to carry the semantic of HTTP requests and responses.
-
-This means that the intention of a client to request resource `/index.html` using the `GET` method can be carried over the network in different formats.
-
-A HTTP client transport is the component that is in charge of converting a high-level, semantic, HTTP requests such as "GET resource /index.html" into the specific format understood by the server (for example, HTTP/2), and to convert the server response from the specific format (HTTP/2) into high-level, semantic objects that can be used by applications.
-
-In this way, applications are not aware of the actual protocol being used.
-This allows them to write their logic against a high-level API that hides the details of the specific protocol being used over the network.
-
-The most common protocol format is HTTP/1.1, a text-based protocol with lines separated by `\r\n`:
-
-[source, screen, subs="{sub-order}"]
-----
-GET /index.html HTTP/1.1\r\n
-Host: domain.com\r\n
-...
-\r\n
-----
-
-However, the same request can be made using FastCGI, a binary protocol:
-
-[source, screen, subs="{sub-order}"]
-----
-x01 x01 x00 x01 x00 x08 x00 x00
-x00 x01 x01 x00 x00 x00 x00 x00
-x01 x04 x00 x01 xLL xLL x00 x00
-x0C x0B  D   O   C   U   M   E
- N   T   _   U   R   I   /   i
- n   d   e   x   .   h   t   m
- l
-...
-----
-
-Similarly, HTTP/2 is a binary protocol that transports the same information in a yet different format.
-
-==== HTTP/1.1 Transport
-
-HTTP/1.1 is the default transport.
-
-[source, java, subs="{sub-order}"]
-----
-// No transport specified, using default.
-HttpClient client = new HttpClient();
-client.start();
-----
-
-If you want to customize the HTTP/1.1 transport, you can explicitly configure `HttpClient` in this way:
-
-[source, java, subs="{sub-order}"]
-----
-int selectors = 1;
-HttpClientTransportOverHTTP transport = new HttpClientTransportOverHTTP(selectors);
-
-HttpClient client = new HttpClient(transport, null);
-client.start();
-----
-
-The example above allows you to customize the number of NIO selectors that `HttpClient` will be using.
-
-==== HTTP/2 Transport
-
-The HTTP/2 transport can be configured in this way:
-
-[source, java, subs="{sub-order}"]
-----
-HTTP2Client h2Client = new HTTP2Client();
-h2Client.setSelectors(1);
-HttpClientTransportOverHTTP2 transport = new HttpClientTransportOverHTTP2(h2Client);
-
-HttpClient client = new HttpClient(transport, null);
-client.start();
-----
-
-`HTTP2Client` is the lower-level client that provides an API based on HTTP/2 concepts such as _sessions_, _streams_ and _frames_ that are specific to HTTP/2.
-
-`HttpClientTransportOverHTTP2` uses `HTTP2Client` to format high-level semantic HTTP requests ("GET resource /index.html") into the HTTP/2 specific format.
-
-==== FastCGI Transport
-
-The FastCGI transport can be configured in this way:
-
-[source, java, subs="{sub-order}"]
-----
-int selectors = 1;
-String scriptRoot = "/var/www/wordpress";
-HttpClientTransportOverFCGI transport = new HttpClientTransportOverFCGI(selectors, false, scriptRoot);
-
-HttpClient client = new HttpClient(transport, null);
-client.start();
-----
-
-In order to make requests using the FastCGI transport, you need to have a FastCGI server such as https://en.wikipedia.org/wiki/PHP#PHPFPM[PHP-FPM] (see also http://php.net/manual/en/install.fpm.php).
-
-The FastCGI transport is primarily used by Jetty's link:#fastcgi[FastCGI support] to serve PHP pages (WordPress for example).

jetty-documentation/src/main/asciidoc/embedded-guide/server/embedding/examples/embedded-minimal-servlet.adoc

@@ -22,7 +22,7 @@
 This example shows the bare minimum required for deploying a servlet into Jetty.
 Note that this is strictly a servlet, not a servlet in the context of a web application, that example comes later.
 This is purely just a servlet deployed and mounted on a context and able to process requests.
-This example is excellent for situations where you have a simple servlet that you need to unit test, just mount it on a context and issue requests using your favorite http client library (like our Jetty client found in xref:http-client[]).
+This example is excellent for situations where you have a simple servlet that you need to unit test, just mount it on a context and issue requests using your favorite http client library (like our Jetty client found in xref:client-http[]).
 
 [source, java, subs="{sub-order}"]
 ----

jetty-documentation/src/main/asciidoc/embedded-guide/server/embedding/examples/embedded-one-webapp.adoc

@@ -21,7 +21,7 @@
 
 This example shows how to deploy a simple webapp with an embedded instance of Jetty.
 This is useful when you want to manage the lifecycle of a server programmatically, either within a production application or as a simple way to deploying and debugging a full scale application deployment.
-In many ways it is easier then traditional deployment since you control the classpath yourself, making this easy to wire up in a test case in Maven  and issue requests using your favorite http client library (like our Jetty client found in xref:http-client[]).
+In many ways it is easier then traditional deployment since you control the classpath yourself, making this easy to wire up in a test case in Maven  and issue requests using your favorite http client library (like our Jetty client found in xref:client-http[]).
 
 [source, java, subs="{sub-order}"]
 ----

jetty-documentation/src/main/asciidoc/quickstart-guide/upgrading/upgrading-9.4-to.10.0.adoc

@@ -23,9 +23,14 @@ It is not comprehensive, but covers many of the major changes included in the re
 
 ==== Required Java Version
 
-Jetty 10 requires, at a minimum, Java 9 to function.
+Jetty 10 requires, at a minimum, Java 11 to function.
 Items such as the Java Platform Module System (JPMS), which Jetty 10 supports, are not available in earlier versions of Java.
 
+==== ServletContainerInitializers
+
+As of Jetty 10, Annotations will be discovered even for old versions of `web.xml` (2.5).
+Users wishing not to use annotations from the webapp classpath with older versions of `web.xml` must call `WebAppContext.setConfigurationDiscovered(false)` either programmatically or in xml.
+
 ==== Removed Classes
 
 //TODO - Insert major removed/refactored classes from Jetty-9.x.x to Jetty-10.0.x

jetty-documentation/src/main/java/embedded/SelectorManagerDocs.java

@@ -0,0 +1,270 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package embedded;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.nio.channels.WritePendingException;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.eclipse.jetty.io.AbstractConnection;
+import org.eclipse.jetty.io.EndPoint;
+import org.eclipse.jetty.io.SelectorManager;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+
+public class SelectorManagerDocs
+{
+    // tag::connect[]
+    public void connect(SelectorManager selectorManager, Map<String, Object> context) throws IOException
+    {
+        String host = "host";
+        int port = 8080;
+
+        // Create an unconnected SocketChannel.
+        SocketChannel socketChannel = SocketChannel.open();
+        socketChannel.configureBlocking(false);
+
+        // Connect and register to Jetty.
+        if (socketChannel.connect(new InetSocketAddress(host, port)))
+            selectorManager.accept(socketChannel, context);
+        else
+            selectorManager.connect(socketChannel, context);
+    }
+    // end::connect[]
+
+    // tag::accept[]
+    public void accept(ServerSocketChannel acceptor, SelectorManager selectorManager) throws IOException
+    {
+        // Wait until a client connects.
+        SocketChannel socketChannel = acceptor.accept();
+        socketChannel.configureBlocking(false);
+
+        // Accept and register to Jetty.
+        Object attachment = null;
+        selectorManager.accept(socketChannel, attachment);
+    }
+    // end::accept[]
+
+    public void connection()
+    {
+        // tag::connection[]
+        // Extend AbstractConnection to inherit basic implementation.
+        class MyConnection extends AbstractConnection
+        {
+            public MyConnection(EndPoint endPoint, Executor executor)
+            {
+                super(endPoint, executor);
+            }
+
+            @Override
+            public void onOpen()
+            {
+                super.onOpen();
+
+                // Declare interest for fill events.
+                fillInterested();
+            }
+
+            @Override
+            public void onFillable()
+            {
+                // Called when a fill event happens.
+            }
+        }
+        // end::connection[]
+    }
+
+    public void echoWrong()
+    {
+        // tag::echo-wrong[]
+        class WrongEchoConnection extends AbstractConnection implements Callback
+        {
+            public WrongEchoConnection(EndPoint endPoint, Executor executor)
+            {
+                super(endPoint, executor);
+            }
+
+            @Override
+            public void onOpen()
+            {
+                super.onOpen();
+
+                // Declare interest for fill events.
+                fillInterested();
+            }
+
+            @Override
+            public void onFillable()
+            {
+                try
+                {
+                    ByteBuffer buffer = BufferUtil.allocate(1024);
+                    int filled = getEndPoint().fill(buffer);
+                    if (filled > 0)
+                    {
+                        // Filled some bytes, echo them back.
+                        getEndPoint().write(this, buffer);
+                    }
+                    else if (filled == 0)
+                    {
+                        // No more bytes to fill, declare
+                        // again interest for fill events.
+                        fillInterested();
+                    }
+                    else
+                    {
+                        // The other peer closed the
+                        // connection, close it back.
+                        getEndPoint().close();
+                    }
+                }
+                catch (Exception x)
+                {
+                    getEndPoint().close(x);
+                }
+            }
+
+            @Override
+            public void succeeded()
+            {
+                // The write is complete, fill again.
+                onFillable();
+            }
+
+            @Override
+            public void failed(Throwable x)
+            {
+                getEndPoint().close(x);
+            }
+        }
+        // end::echo-wrong[]
+    }
+
+    public void echoCorrect()
+    {
+        // tag::echo-correct[]
+        class EchoConnection extends AbstractConnection implements Callback
+        {
+            public static final int IDLE = 0;
+            public static final int WRITING = 1;
+            public static final int PENDING = 2;
+
+            private final AtomicInteger state = new AtomicInteger();
+
+            public EchoConnection(EndPoint endp, Executor executor)
+            {
+                super(endp, executor);
+            }
+
+            @Override
+            public void onOpen()
+            {
+                super.onOpen();
+
+                // Declare interest for fill events.
+                fillInterested();
+            }
+
+            @Override
+            public void onFillable()
+            {
+                try
+                {
+                    ByteBuffer buffer = BufferUtil.allocate(1024);
+                    while (true)
+                    {
+                        int filled = getEndPoint().fill(buffer);
+                        if (filled > 0)
+                        {
+                            // We have filled some bytes, echo them back.
+                            if (write(buffer))
+                            {
+                                // If the write completed, continue to fill.
+                                continue;
+                            }
+                            else
+                            {
+                                // The write is pending, return to wait for completion.
+                                return;
+                            }
+                        }
+                        else if (filled == 0)
+                        {
+                            // No more bytes to read, declare
+                            // again interest for fill events.
+                            fillInterested();
+                            return;
+                        }
+                        else
+                        {
+                            // The other peer closed the connection.
+                            close();
+                            return;
+                        }
+                    }
+                }
+                catch (Throwable x)
+                {
+                    getEndPoint().close(x);
+                }
+            }
+
+            private boolean write(ByteBuffer buffer)
+            {
+                // Check if we are writing concurrently.
+                if (!state.compareAndSet(IDLE, WRITING))
+                    throw new WritePendingException();
+
+                // Write the buffer using "this" as a callback.
+                getEndPoint().write(this, buffer);
+
+                // Check if the write is already completed.
+                boolean writeIsPending = state.compareAndSet(WRITING, PENDING);
+
+                // Return true if the write was completed.
+                return !writeIsPending;
+            }
+
+            @Override
+            public void succeeded()
+            {
+                // The write is complete, reset the state.
+                int prevState = state.getAndSet(IDLE);
+
+                // If the write was pending we need
+                // to resume reading from the network.
+                if (prevState == PENDING)
+                    onFillable();
+            }
+
+            @Override
+            public void failed(Throwable x)
+            {
+                getEndPoint().close(x);
+            }
+        }
+        // end::echo-correct[]
+    }
+}

jetty-documentation/src/main/java/embedded/client/ClientConnectorDocSnippets.java

@@ -1,150 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-package embedded.client;
-
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.Executor;
-
-import org.eclipse.jetty.io.AbstractConnection;
-import org.eclipse.jetty.io.ClientConnectionFactory;
-import org.eclipse.jetty.io.ClientConnector;
-import org.eclipse.jetty.io.EndPoint;
-import org.eclipse.jetty.io.SelectorManager;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.eclipse.jetty.util.thread.QueuedThreadPool;
-import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
-import org.eclipse.jetty.util.thread.Scheduler;
-
-import static java.lang.System.Logger.Level.INFO;
-
-public class ClientConnectorDocSnippets
-{
-    public void simplest() throws Exception
-    {
-        // tag::simplest[]
-        ClientConnector clientConnector = new ClientConnector();
-        clientConnector.start();
-        // end::simplest[]
-    }
-
-    public void typical() throws Exception
-    {
-        // tag::typical[]
-        // Create and configure the SslContextFactory.
-        SslContextFactory.Client sslContextFactory = new SslContextFactory.Client();
-        sslContextFactory.addExcludeProtocols("TLSv1", "TLSv1.1");
-
-        // Create and configure the thread pool.
-        QueuedThreadPool threadPool = new QueuedThreadPool();
-        threadPool.setName("client");
-
-        // Create and configure the ClientConnector.
-        ClientConnector clientConnector = new ClientConnector();
-        clientConnector.setSslContextFactory(sslContextFactory);
-        clientConnector.setExecutor(threadPool);
-        clientConnector.start();
-        // end::typical[]
-    }
-
-    public void advanced() throws Exception
-    {
-        // tag::advanced[]
-        class CustomClientConnector extends ClientConnector
-        {
-            @Override
-            protected SelectorManager newSelectorManager()
-            {
-                return new ClientSelectorManager(getExecutor(), getScheduler(), getSelectors())
-                {
-                    @Override
-                    protected void endPointOpened(EndPoint endpoint)
-                    {
-                        System.getLogger("endpoint").log(INFO, "opened %s", endpoint);
-                    }
-
-                    @Override
-                    protected void endPointClosed(EndPoint endpoint)
-                    {
-                        System.getLogger("endpoint").log(INFO, "closed %s", endpoint);
-                    }
-                };
-            }
-        }
-
-        // Create and configure the thread pool.
-        QueuedThreadPool threadPool = new QueuedThreadPool();
-        threadPool.setName("client");
-
-        // Create and configure the scheduler.
-        Scheduler scheduler = new ScheduledExecutorScheduler("scheduler-client", false);
-
-        // Create and configure the custom ClientConnector.
-        CustomClientConnector clientConnector = new CustomClientConnector();
-        clientConnector.setExecutor(threadPool);
-        clientConnector.setScheduler(scheduler);
-        clientConnector.start();
-        // end::advanced[]
-    }
-
-    public void connect() throws Exception
-    {
-        class CustomHTTPConnection extends AbstractConnection
-        {
-            public CustomHTTPConnection(EndPoint endPoint, Executor executor)
-            {
-                super(endPoint, executor);
-            }
-
-            @Override
-            public void onOpen()
-            {
-                super.onOpen();
-            }
-
-            @Override
-            public void onFillable()
-            {
-            }
-        }
-
-        ClientConnector clientConnector = new ClientConnector();
-        clientConnector.start();
-
-        String host = "wikipedia.org";
-        int port = 80;
-        SocketAddress address = new InetSocketAddress(host, port);
-
-        ClientConnectionFactory connectionFactory = (endPoint, context) ->
-        {
-            System.getLogger("connection").log(INFO, "Creating connection for {0}", endPoint);
-            return new CustomHTTPConnection(endPoint, clientConnector.getExecutor());
-        };
-        Map<String, Object> context = new HashMap<>();
-        context.put(ClientConnector.CLIENT_CONNECTION_FACTORY_CONTEXT_KEY, connectionFactory);
-        clientConnector.connect(address, context);
-    }
-
-    public static void main(String[] args) throws Exception
-    {
-        new ClientConnectorDocSnippets().connect();
-    }
-}

jetty-documentation/src/main/java/embedded/client/ClientConnectorDocs.java

@@ -0,0 +1,429 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package embedded.client;
+
+import java.io.ByteArrayOutputStream;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.function.Consumer;
+
+import org.eclipse.jetty.io.AbstractConnection;
+import org.eclipse.jetty.io.ClientConnectionFactory;
+import org.eclipse.jetty.io.ClientConnector;
+import org.eclipse.jetty.io.EndPoint;
+import org.eclipse.jetty.io.SelectorManager;
+import org.eclipse.jetty.io.ssl.SslClientConnectionFactory;
+import org.eclipse.jetty.io.ssl.SslConnection;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.Promise;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
+import org.eclipse.jetty.util.thread.Scheduler;
+
+import static java.lang.System.Logger.Level.INFO;
+
+public class ClientConnectorDocs
+{
+    public void simplest() throws Exception
+    {
+        // tag::simplest[]
+        ClientConnector clientConnector = new ClientConnector();
+        clientConnector.start();
+        // end::simplest[]
+    }
+
+    public void typical() throws Exception
+    {
+        // tag::typical[]
+        // Create and configure the SslContextFactory.
+        SslContextFactory.Client sslContextFactory = new SslContextFactory.Client();
+        sslContextFactory.addExcludeProtocols("TLSv1", "TLSv1.1");
+
+        // Create and configure the thread pool.
+        QueuedThreadPool threadPool = new QueuedThreadPool();
+        threadPool.setName("client");
+
+        // Create and configure the ClientConnector.
+        ClientConnector clientConnector = new ClientConnector();
+        clientConnector.setSslContextFactory(sslContextFactory);
+        clientConnector.setExecutor(threadPool);
+        clientConnector.start();
+        // end::typical[]
+    }
+
+    public void advanced() throws Exception
+    {
+        // tag::advanced[]
+        class CustomClientConnector extends ClientConnector
+        {
+            @Override
+            protected SelectorManager newSelectorManager()
+            {
+                return new ClientSelectorManager(getExecutor(), getScheduler(), getSelectors())
+                {
+                    @Override
+                    protected void endPointOpened(EndPoint endpoint)
+                    {
+                        System.getLogger("endpoint").log(INFO, "opened %s", endpoint);
+                    }
+
+                    @Override
+                    protected void endPointClosed(EndPoint endpoint)
+                    {
+                        System.getLogger("endpoint").log(INFO, "closed %s", endpoint);
+                    }
+                };
+            }
+        }
+
+        // Create and configure the thread pool.
+        QueuedThreadPool threadPool = new QueuedThreadPool();
+        threadPool.setName("client");
+
+        // Create and configure the scheduler.
+        Scheduler scheduler = new ScheduledExecutorScheduler("scheduler-client", false);
+
+        // Create and configure the custom ClientConnector.
+        CustomClientConnector clientConnector = new CustomClientConnector();
+        clientConnector.setExecutor(threadPool);
+        clientConnector.setScheduler(scheduler);
+        clientConnector.start();
+        // end::advanced[]
+    }
+
+    public void connect() throws Exception
+    {
+        // tag::connect[]
+        class CustomConnection extends AbstractConnection
+        {
+            public CustomConnection(EndPoint endPoint, Executor executor)
+            {
+                super(endPoint, executor);
+            }
+
+            @Override
+            public void onOpen()
+            {
+                super.onOpen();
+                System.getLogger("connection").log(INFO, "Opened connection {0}", this);
+            }
+
+            @Override
+            public void onFillable()
+            {
+            }
+        }
+
+        ClientConnector clientConnector = new ClientConnector();
+        clientConnector.start();
+
+        String host = "serverHost";
+        int port = 8080;
+        SocketAddress address = new InetSocketAddress(host, port);
+
+        // The ClientConnectionFactory that creates CustomConnection instances.
+        ClientConnectionFactory connectionFactory = (endPoint, context) ->
+        {
+            System.getLogger("connection").log(INFO, "Creating connection for {0}", endPoint);
+            return new CustomConnection(endPoint, clientConnector.getExecutor());
+        };
+
+        // The Promise to notify of connection creation success or failure.
+        CompletableFuture<CustomConnection> connectionPromise = new Promise.Completable<>();
+
+        // Populate the context with the mandatory keys to create and obtain connections.
+        Map<String, Object> context = new HashMap<>();
+        context.put(ClientConnector.CLIENT_CONNECTION_FACTORY_CONTEXT_KEY, connectionFactory);
+        context.put(ClientConnector.CONNECTION_PROMISE_CONTEXT_KEY, connectionPromise);
+        clientConnector.connect(address, context);
+
+        // Use the Connection when it's available.
+
+        // Use it in a non-blocking way via CompletableFuture APIs.
+        connectionPromise.whenComplete((connection, failure) ->
+        {
+            System.getLogger("connection").log(INFO, "Created connection for {0}", connection);
+        });
+
+        // Alternatively, you can block waiting for the connection (or a failure).
+        // CustomConnection connection = connectionPromise.get();
+        // end::connect[]
+    }
+
+    public void telnet() throws Exception
+    {
+        // tag::telnet[]
+        class TelnetConnection extends AbstractConnection
+        {
+            private final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+            private Consumer<String> consumer;
+
+            public TelnetConnection(EndPoint endPoint, Executor executor)
+            {
+                super(endPoint, executor);
+            }
+
+            @Override
+            public void onOpen()
+            {
+                super.onOpen();
+
+                // Declare interest for fill events.
+                fillInterested();
+            }
+
+            @Override
+            public void onFillable()
+            {
+                try
+                {
+                    ByteBuffer buffer = BufferUtil.allocate(1024);
+                    while (true)
+                    {
+                        int filled = getEndPoint().fill(buffer);
+                        if (filled > 0)
+                        {
+                            while (buffer.hasRemaining())
+                            {
+                                // Search for newline.
+                                byte read = buffer.get();
+                                if (read == '\n')
+                                {
+                                    // Notify the consumer of the line.
+                                    consumer.accept(bytes.toString(StandardCharsets.UTF_8));
+                                    bytes.reset();
+                                }
+                                else
+                                {
+                                    bytes.write(read);
+                                }
+                            }
+                        }
+                        else if (filled == 0)
+                        {
+                            // No more bytes to fill, declare
+                            // again interest for fill events.
+                            fillInterested();
+                            return;
+                        }
+                        else
+                        {
+                            // The other peer closed the
+                            // connection, close it back.
+                            getEndPoint().close();
+                            return;
+                        }
+                    }
+                }
+                catch (Exception x)
+                {
+                    getEndPoint().close(x);
+                }
+            }
+
+            public void onLine(Consumer<String> consumer)
+            {
+                this.consumer = consumer;
+            }
+
+            public void writeLine(String line, Callback callback)
+            {
+                line = line + "\r\n";
+                getEndPoint().write(callback, ByteBuffer.wrap(line.getBytes(StandardCharsets.UTF_8)));
+            }
+        }
+
+        ClientConnector clientConnector = new ClientConnector();
+        clientConnector.start();
+
+        String host = "wikipedia.org";
+        int port = 80;
+        SocketAddress address = new InetSocketAddress(host, port);
+
+        ClientConnectionFactory connectionFactory = (endPoint, context) ->
+            new TelnetConnection(endPoint, clientConnector.getExecutor());
+
+        CompletableFuture<TelnetConnection> connectionPromise = new Promise.Completable<>();
+
+        Map<String, Object> context = new HashMap<>();
+        context.put(ClientConnector.CLIENT_CONNECTION_FACTORY_CONTEXT_KEY, connectionFactory);
+        context.put(ClientConnector.CONNECTION_PROMISE_CONTEXT_KEY, connectionPromise);
+        clientConnector.connect(address, context);
+
+        connectionPromise.whenComplete((connection, failure) ->
+        {
+            if (failure == null)
+            {
+                // Register a listener that receives string lines.
+                connection.onLine(line -> System.getLogger("app").log(INFO, "line: {0}", line));
+
+                // Write a line.
+                connection.writeLine("" +
+                    "GET / HTTP/1.0\r\n" +
+                    "", Callback.NOOP);
+            }
+            else
+            {
+                failure.printStackTrace();
+            }
+        });
+        // end::telnet[]
+    }
+
+    public void tlsTelnet() throws Exception
+    {
+        // tag::tlsTelnet[]
+        class TelnetConnection extends AbstractConnection
+        {
+            private final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+            private Consumer<String> consumer;
+
+            public TelnetConnection(EndPoint endPoint, Executor executor)
+            {
+                super(endPoint, executor);
+            }
+
+            @Override
+            public void onOpen()
+            {
+                super.onOpen();
+
+                // Declare interest for fill events.
+                fillInterested();
+            }
+
+            @Override
+            public void onFillable()
+            {
+                try
+                {
+                    ByteBuffer buffer = BufferUtil.allocate(1024);
+                    while (true)
+                    {
+                        int filled = getEndPoint().fill(buffer);
+                        if (filled > 0)
+                        {
+                            while (buffer.hasRemaining())
+                            {
+                                // Search for newline.
+                                byte read = buffer.get();
+                                if (read == '\n')
+                                {
+                                    // Notify the consumer of the line.
+                                    consumer.accept(bytes.toString(StandardCharsets.UTF_8));
+                                    bytes.reset();
+                                }
+                                else
+                                {
+                                    bytes.write(read);
+                                }
+                            }
+                        }
+                        else if (filled == 0)
+                        {
+                            // No more bytes to fill, declare
+                            // again interest for fill events.
+                            fillInterested();
+                            return;
+                        }
+                        else
+                        {
+                            // The other peer closed the
+                            // connection, close it back.
+                            getEndPoint().close();
+                            return;
+                        }
+                    }
+                }
+                catch (Exception x)
+                {
+                    getEndPoint().close(x);
+                }
+            }
+
+            public void onLine(Consumer<String> consumer)
+            {
+                this.consumer = consumer;
+            }
+
+            public void writeLine(String line, Callback callback)
+            {
+                line = line + "\r\n";
+                getEndPoint().write(callback, ByteBuffer.wrap(line.getBytes(StandardCharsets.UTF_8)));
+            }
+        }
+
+        ClientConnector clientConnector = new ClientConnector();
+        clientConnector.start();
+
+        // Use port 443 to contact the server using encrypted HTTP.
+        String host = "wikipedia.org";
+        int port = 443;
+        SocketAddress address = new InetSocketAddress(host, port);
+
+        ClientConnectionFactory connectionFactory = (endPoint, context) ->
+            new TelnetConnection(endPoint, clientConnector.getExecutor());
+
+        // Wrap the "telnet" ClientConnectionFactory with the SslClientConnectionFactory.
+        connectionFactory = new SslClientConnectionFactory(clientConnector.getSslContextFactory(),
+            clientConnector.getByteBufferPool(), clientConnector.getExecutor(), connectionFactory);
+
+        // We will obtain a SslConnection now.
+        CompletableFuture<SslConnection> connectionPromise = new Promise.Completable<>();
+
+        Map<String, Object> context = new HashMap<>();
+        context.put(ClientConnector.CLIENT_CONNECTION_FACTORY_CONTEXT_KEY, connectionFactory);
+        context.put(ClientConnector.CONNECTION_PROMISE_CONTEXT_KEY, connectionPromise);
+        clientConnector.connect(address, context);
+
+        connectionPromise.whenComplete((sslConnection, failure) ->
+        {
+            if (failure == null)
+            {
+                // Unwrap the SslConnection to access the "line" APIs in TelnetConnection.
+                TelnetConnection connection = (TelnetConnection)sslConnection.getDecryptedEndPoint().getConnection();
+                // Register a listener that receives string lines.
+                connection.onLine(line -> System.getLogger("app").log(INFO, "line: {0}", line));
+
+                // Write a line.
+                connection.writeLine("" +
+                    "GET / HTTP/1.0\r\n" +
+                    "", Callback.NOOP);
+            }
+            else
+            {
+                failure.printStackTrace();
+            }
+        });
+        // end::tlsTelnet[]
+    }
+
+    public static void main(String[] args) throws Exception
+    {
+        new ClientConnectorDocs().tlsTelnet();
+    }
+}

jetty-documentation/src/main/java/embedded/client/SelectorManagerDocSnippets.java

@@ -1,61 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-package embedded.client;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.channels.ServerSocketChannel;
-import java.nio.channels.SocketChannel;
-import java.util.Map;
-
-import org.eclipse.jetty.io.SelectorManager;
-
-public class SelectorManagerDocSnippets
-{
-    // tag::connect[]
-    public void connect(SelectorManager selectorManager, Map<String, Object> context) throws IOException
-    {
-        String host = "host";
-        int port = 8080;
-
-        // Create an unconnected SocketChannel.
-        SocketChannel socketChannel = SocketChannel.open();
-        socketChannel.configureBlocking(false);
-
-        // Connect and register to Jetty.
-        if (socketChannel.connect(new InetSocketAddress(host, port)))
-            selectorManager.accept(socketChannel, context);
-        else
-            selectorManager.connect(socketChannel, context);
-    }
-    // end::connect[]
-
-    // tag::accept[]
-    public void accept(ServerSocketChannel acceptor, SelectorManager selectorManager) throws IOException
-    {
-        // Wait until a client connects.
-        SocketChannel socketChannel = acceptor.accept();
-        socketChannel.configureBlocking(false);
-
-        // Accept and register to Jetty.
-        Object attachment = null;
-        selectorManager.accept(socketChannel, attachment);
-    }
-    // end::accept[]
-}

jetty-documentation/src/main/java/embedded/client/http/HTTPClientDocs.java

@@ -0,0 +1,850 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package embedded.client.http;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.CookieStore;
+import java.net.HttpCookie;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.LongConsumer;
+
+import org.eclipse.jetty.client.ConnectionPool;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.HttpClientTransport;
+import org.eclipse.jetty.client.HttpDestination;
+import org.eclipse.jetty.client.HttpProxy;
+import org.eclipse.jetty.client.ProxyConfiguration;
+import org.eclipse.jetty.client.RoundRobinConnectionPool;
+import org.eclipse.jetty.client.api.Authentication;
+import org.eclipse.jetty.client.api.AuthenticationStore;
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.api.Result;
+import org.eclipse.jetty.client.dynamic.HttpClientTransportDynamic;
+import org.eclipse.jetty.client.http.HttpClientConnectionFactory;
+import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
+import org.eclipse.jetty.client.util.AsyncRequestContent;
+import org.eclipse.jetty.client.util.BasicAuthentication;
+import org.eclipse.jetty.client.util.BufferingResponseListener;
+import org.eclipse.jetty.client.util.BytesRequestContent;
+import org.eclipse.jetty.client.util.DigestAuthentication;
+import org.eclipse.jetty.client.util.FutureResponseListener;
+import org.eclipse.jetty.client.util.InputStreamRequestContent;
+import org.eclipse.jetty.client.util.InputStreamResponseListener;
+import org.eclipse.jetty.client.util.OutputStreamRequestContent;
+import org.eclipse.jetty.client.util.PathRequestContent;
+import org.eclipse.jetty.client.util.StringRequestContent;
+import org.eclipse.jetty.fcgi.client.http.HttpClientTransportOverFCGI;
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpMethod;
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.http2.client.HTTP2Client;
+import org.eclipse.jetty.http2.client.http.ClientConnectionFactoryOverHTTP2;
+import org.eclipse.jetty.http2.client.http.HttpClientTransportOverHTTP2;
+import org.eclipse.jetty.io.ByteBufferPool;
+import org.eclipse.jetty.io.ClientConnectionFactory;
+import org.eclipse.jetty.io.ClientConnector;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.HttpCookieStore;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+
+import static java.lang.System.Logger.Level.INFO;
+
+@SuppressWarnings("unused")
+public class HTTPClientDocs
+{
+    public void start() throws Exception
+    {
+        // tag::start[]
+        // Instantiate HttpClient.
+        HttpClient httpClient = new HttpClient();
+
+        // Configure HttpClient, for example:
+        httpClient.setFollowRedirects(false);
+
+        // Start HttpClient.
+        httpClient.start();
+        // end::start[]
+    }
+
+    public void stop() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+        // tag::stop[]
+        // Stop HttpClient.
+        httpClient.stop();
+        // end::stop[]
+    }
+
+    public void tlsExplicit() throws Exception
+    {
+        // tag::tlsExplicit[]
+        SslContextFactory.Client sslContextFactory = new SslContextFactory.Client();
+
+        ClientConnector clientConnector = new ClientConnector();
+        clientConnector.setSslContextFactory(sslContextFactory);
+
+        HttpClient httpClient = new HttpClient(new HttpClientTransportDynamic(clientConnector));
+        httpClient.start();
+        // end::tlsExplicit[]
+    }
+
+    public void tlsNoValidation()
+    {
+        // tag::tlsNoValidation[]
+        SslContextFactory.Client sslContextFactory = new SslContextFactory.Client();
+        // Disable certificate validation at the TLS level.
+        sslContextFactory.setEndpointIdentificationAlgorithm(null);
+        // end::tlsNoValidation[]
+    }
+
+    public void tlsAppValidation()
+    {
+        // tag::tlsAppValidation[]
+        SslContextFactory.Client sslContextFactory = new SslContextFactory.Client();
+        // Only allow subdomains of domain.com.
+        sslContextFactory.setHostnameVerifier((hostName, session) -> hostName.endsWith(".domain.com"));
+        // end::tlsAppValidation[]
+    }
+
+    public void simpleBlockingGet() throws Exception
+    {
+        // tag::simpleBlockingGet[]
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // Perform a simple GET and wait for the response.
+        ContentResponse response = httpClient.GET("http://domain.com/path?query");
+        // end::simpleBlockingGet[]
+    }
+
+    public void headFluent() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::headFluent[]
+        ContentResponse response = httpClient.newRequest("http://domain.com/path?query")
+            .method(HttpMethod.HEAD)
+            .agent("Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0")
+            .send();
+        // end::headFluent[]
+    }
+
+    public void headNonFluent() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::headNonFluent[]
+        Request request = httpClient.newRequest("http://domain.com/path?query");
+        request.method(HttpMethod.HEAD);
+        request.agent("Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0");
+        ContentResponse response = request.send();
+        // end::headNonFluent[]
+    }
+
+    public void postFluent() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::postFluent[]
+        ContentResponse response = httpClient.POST("http://domain.com/entity/1")
+            .param("p", "value")
+            .send();
+        // end::postFluent[]
+    }
+
+    public void fileFluent() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::fileFluent[]
+        ContentResponse response = httpClient.POST("http://domain.com/upload")
+            .file(Paths.get("file_to_upload.txt"), "text/plain")
+            .send();
+        // end::fileFluent[]
+    }
+
+    public void totalTimeout() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::totalTimeout[]
+        ContentResponse response = httpClient.newRequest("http://domain.com/path?query")
+            .timeout(5, TimeUnit.SECONDS)
+            .send();
+        // end::totalTimeout[]
+    }
+
+    public void simpleNonBlocking() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::simpleNonBlocking[]
+        httpClient.newRequest("http://domain.com/path")
+            .send(result ->
+            {
+                // Your logic here
+            });
+        // end::simpleNonBlocking[]
+    }
+
+    public void nonBlockingTotalTimeout() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::nonBlockingTotalTimeout[]
+        httpClient.newRequest("http://domain.com/path")
+            .timeout(3, TimeUnit.SECONDS)
+            .send(result ->
+            {
+                /* Your logic here */
+            });
+        // end::nonBlockingTotalTimeout[]
+    }
+
+    // @checkstyle-disable-check : LeftCurly
+    public void listeners() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::listeners[]
+        httpClient.newRequest("http://domain.com/path")
+            // Add request hooks.
+            .onRequestQueued(request -> { /* ... */ })
+            .onRequestBegin(request -> { /* ... */ })
+            .onRequestHeaders(request -> { /* ... */ })
+            .onRequestCommit(request -> { /* ... */ })
+            .onRequestContent((request, content) -> { /* ... */ })
+            .onRequestFailure((request, failure) -> { /* ... */ })
+            .onRequestSuccess(request -> { /* ... */ })
+            // Add response hooks.
+            .onResponseBegin(response -> { /* ... */ })
+            .onResponseHeader((response, field) -> true)
+            .onResponseHeaders(response -> { /* ... */ })
+            .onResponseContentAsync((response, buffer, callback) -> callback.succeeded())
+            .onResponseFailure((response, failure) -> { /* ... */ })
+            .onResponseSuccess(response -> { /* ... */ })
+            // Result hook.
+            .send(result -> { /* ... */ });
+        // end::listeners[]
+    }
+
+    public void pathRequestContent() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::pathRequestContent[]
+        ContentResponse response = httpClient.POST("http://domain.com/upload")
+            .body(new PathRequestContent("text/plain", Paths.get("file_to_upload.txt")))
+            .send();
+        // end::pathRequestContent[]
+    }
+
+    public void inputStreamRequestContent() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::inputStreamRequestContent[]
+        ContentResponse response = httpClient.POST("http://domain.com/upload")
+            .body(new InputStreamRequestContent("text/plain", new FileInputStream("file_to_upload.txt")))
+            .send();
+        // end::inputStreamRequestContent[]
+    }
+
+    public void bytesStringRequestContent() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        byte[] bytes = new byte[1024];
+        String string = new String(bytes);
+        // tag::bytesStringRequestContent[]
+        ContentResponse bytesResponse = httpClient.POST("http://domain.com/upload")
+            .body(new BytesRequestContent("text/plain", bytes))
+            .send();
+
+        ContentResponse stringResponse = httpClient.POST("http://domain.com/upload")
+            .body(new StringRequestContent("text/plain", string))
+            .send();
+        // end::bytesStringRequestContent[]
+    }
+
+    public void asyncRequestContent() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::asyncRequestContent[]
+        AsyncRequestContent content = new AsyncRequestContent();
+        httpClient.POST("http://domain.com/upload")
+            .body(content)
+            .send(result ->
+            {
+                // Your logic here
+            });
+
+        // Content not available yet here.
+
+        // An event happens in some other class, in some other thread.
+        class ContentPublisher
+        {
+            void publish(ByteBufferPool bufferPool, byte[] bytes, boolean lastContent)
+            {
+                // Wrap the bytes into a new ByteBuffer.
+                ByteBuffer buffer = ByteBuffer.wrap(bytes);
+
+                // Offer the content, and release the ByteBuffer
+                // to the pool when the Callback is completed.
+                content.offer(buffer, Callback.from(() -> bufferPool.release(buffer)));
+
+                // Close AsyncRequestContent when all the content is arrived.
+                if (lastContent)
+                    content.close();
+            }
+        }
+        // end::asyncRequestContent[]
+    }
+
+    public void outputStreamRequestContent() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::outputStreamRequestContent[]
+        OutputStreamRequestContent content = new OutputStreamRequestContent();
+
+        // Use try-with-resources to close the OutputStream when all content is written.
+        try (OutputStream output = content.getOutputStream())
+        {
+            httpClient.POST("http://localhost:8080/")
+                .body(content)
+                .send(result ->
+                {
+                    // Your logic here
+                });
+
+            // Content not available yet here.
+
+            // Content is now available.
+            byte[] bytes = new byte[]{'h', 'e', 'l', 'l', 'o'};
+            output.write(bytes);
+        }
+        // End of try-with-resource, output.close() called automatically to signal end of content.
+        // end::outputStreamRequestContent[]
+    }
+
+    public void futureResponseListener() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::futureResponseListener[]
+        Request request = httpClient.newRequest("http://domain.com/path");
+
+        // Limit response content buffer to 512 KiB.
+        FutureResponseListener listener = new FutureResponseListener(request, 512 * 1024);
+
+        request.send(listener);
+
+        // Wait at most 5 seconds for request+response to complete.
+        ContentResponse response = listener.get(5, TimeUnit.SECONDS);
+        // end::futureResponseListener[]
+    }
+
+    public void bufferingResponseListener() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::bufferingResponseListener[]
+        httpClient.newRequest("http://domain.com/path")
+            // Buffer response content up to 8 MiB
+            .send(new BufferingResponseListener(8 * 1024 * 1024)
+            {
+                @Override
+                public void onComplete(Result result)
+                {
+                    if (!result.isFailed())
+                    {
+                        byte[] responseContent = getContent();
+                        // Your logic here
+                    }
+                }
+            });
+        // end::bufferingResponseListener[]
+    }
+
+    public void inputStreamResponseListener() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::inputStreamResponseListener[]
+        InputStreamResponseListener listener = new InputStreamResponseListener();
+        httpClient.newRequest("http://domain.com/path")
+            .send(listener);
+
+        // Wait for the response headers to arrive.
+        Response response = listener.get(5, TimeUnit.SECONDS);
+
+        // Look at the response before streaming the content.
+        if (response.getStatus() == HttpStatus.OK_200)
+        {
+            // Use try-with-resources to close input stream.
+            try (InputStream responseContent = listener.getInputStream())
+            {
+                // Your logic here
+            }
+        }
+        else
+        {
+            response.abort(new IOException("Unexpected HTTP response"));
+        }
+        // end::inputStreamResponseListener[]
+    }
+
+    public void demandedContentListener() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        String host1 = "localhost";
+        String host2 = "localhost";
+        int port1 = 8080;
+        int port2 = 8080;
+        // tag::demandedContentListener[]
+        // Prepare a request to server1, the source.
+        Request request1 = httpClient.newRequest(host1, port1)
+            .path("/source");
+
+        // Prepare a request to server2, the sink.
+        AsyncRequestContent content2 = new AsyncRequestContent();
+        Request request2 = httpClient.newRequest(host2, port2)
+            .path("/sink")
+            .body(content2);
+
+        request1.onResponseContentDemanded(new Response.DemandedContentListener()
+        {
+            @Override
+            public void onBeforeContent(Response response, LongConsumer demand)
+            {
+                request2.onRequestCommit(request ->
+                {
+                    // Only when the request to server2 has been sent,
+                    // then demand response content from server1.
+                    demand.accept(1);
+                });
+
+                // Send the request to server2.
+                request2.send(result -> System.getLogger("forwarder").log(INFO, "Forwarding to server2 complete"));
+            }
+
+            @Override
+            public void onContent(Response response, LongConsumer demand, ByteBuffer content, Callback callback)
+            {
+                // When response content is received from server1, forward it to server2.
+                content2.offer(content, Callback.from(() ->
+                {
+                    // When the request content to server2 is sent,
+                    // succeed the callback to recycle the buffer.
+                    callback.succeeded();
+                    // Then demand more response content from server1.
+                    demand.accept(1);
+                }, callback::failed));
+            }
+        });
+
+        // When the response content from server1 is complete,
+        // complete also the request content to server2.
+        request1.onResponseSuccess(response -> content2.close());
+
+        // Send the request to server1.
+        request1.send(result -> System.getLogger("forwarder").log(INFO, "Sourcing from server1 complete"));
+        // end::demandedContentListener[]
+    }
+
+    public void getCookies() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::getCookies[]
+        CookieStore cookieStore = httpClient.getCookieStore();
+        List<HttpCookie> cookies = cookieStore.get(URI.create("http://domain.com/path"));
+        // end::getCookies[]
+    }
+
+    public void setCookie() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::setCookie[]
+        CookieStore cookieStore = httpClient.getCookieStore();
+        HttpCookie cookie = new HttpCookie("foo", "bar");
+        cookie.setDomain("domain.com");
+        cookie.setPath("/");
+        cookie.setMaxAge(TimeUnit.DAYS.toSeconds(1));
+        cookieStore.add(URI.create("http://domain.com"), cookie);
+        // end::setCookie[]
+    }
+
+    public void requestCookie() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::requestCookie[]
+        ContentResponse response = httpClient.newRequest("http://domain.com/path")
+            .cookie(new HttpCookie("foo", "bar"))
+            .send();
+        // end::requestCookie[]
+    }
+
+    public void removeCookie() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::removeCookie[]
+        CookieStore cookieStore = httpClient.getCookieStore();
+        URI uri = URI.create("http://domain.com");
+        List<HttpCookie> cookies = cookieStore.get(uri);
+        for (HttpCookie cookie : cookies)
+        {
+            cookieStore.remove(uri, cookie);
+        }
+        // end::removeCookie[]
+    }
+
+    public void emptyCookieStore() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::emptyCookieStore[]
+        httpClient.setCookieStore(new HttpCookieStore.Empty());
+        // end::emptyCookieStore[]
+    }
+
+    public void filteringCookieStore() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::filteringCookieStore[]
+        class GoogleOnlyCookieStore extends HttpCookieStore
+        {
+            @Override
+            public void add(URI uri, HttpCookie cookie)
+            {
+                if (uri.getHost().endsWith("google.com"))
+                    super.add(uri, cookie);
+            }
+        }
+
+        httpClient.setCookieStore(new GoogleOnlyCookieStore());
+        // end::filteringCookieStore[]
+    }
+
+    public void addAuthentication() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::addAuthentication[]
+        // Add authentication credentials.
+        AuthenticationStore auth = httpClient.getAuthenticationStore();
+
+        URI uri1 = new URI("http://mydomain.com/secure");
+        auth.addAuthentication(new BasicAuthentication(uri1, "MyRealm", "userName1", "password1"));
+
+        URI uri2 = new URI("http://otherdomain.com/admin");
+        auth.addAuthentication(new BasicAuthentication(uri1, "AdminRealm", "admin", "password"));
+        // end::addAuthentication[]
+    }
+
+    public void clearResults() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::clearResults[]
+        httpClient.getAuthenticationStore().clearAuthenticationResults();
+        // end::clearResults[]
+    }
+
+    public void preemptedResult() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::preemptedResult[]
+        AuthenticationStore auth = httpClient.getAuthenticationStore();
+        URI uri = URI.create("http://domain.com/secure");
+        auth.addAuthenticationResult(new BasicAuthentication.BasicResult(uri, "username", "password"));
+        // end::preemptedResult[]
+    }
+
+    public void requestPreemptedResult() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::requestPreemptedResult[]
+        URI uri = URI.create("http://domain.com/secure");
+        Authentication.Result authn = new BasicAuthentication.BasicResult(uri, "username", "password");
+        Request request = httpClient.newRequest(uri);
+        authn.apply(request);
+        request.send();
+        // end::requestPreemptedResult[]
+    }
+
+    public void proxy() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::proxy[]
+        HttpProxy proxy = new HttpProxy("proxyHost", 8888);
+
+        // Do not proxy requests for localhost:8080.
+        proxy.getExcludedAddresses().add("localhost:8080");
+
+        // Add the new proxy to the list of proxies already registered.
+        ProxyConfiguration proxyConfig = httpClient.getProxyConfiguration();
+        proxyConfig.getProxies().add(proxy);
+
+        ContentResponse response = httpClient.GET("http://domain.com/path");
+        // end::proxy[]
+    }
+
+    public void proxyAuthentication() throws Exception
+    {
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // tag::proxyAuthentication[]
+        AuthenticationStore auth = httpClient.getAuthenticationStore();
+
+        // Proxy credentials.
+        URI proxyURI = new URI("http://proxy.net:8080");
+        auth.addAuthentication(new BasicAuthentication(proxyURI, "ProxyRealm", "proxyUser", "proxyPass"));
+
+        // Server credentials.
+        URI serverURI = new URI("http://domain.com/secure");
+        auth.addAuthentication(new DigestAuthentication(serverURI, "ServerRealm", "serverUser", "serverPass"));
+
+        // Proxy configuration.
+        ProxyConfiguration proxyConfig = httpClient.getProxyConfiguration();
+        HttpProxy proxy = new HttpProxy("proxy.net", 8080);
+        proxyConfig.getProxies().add(proxy);
+
+        ContentResponse response = httpClient.newRequest(serverURI).send();
+        // end::proxyAuthentication[]
+    }
+
+    public void defaultTransport() throws Exception
+    {
+        // tag::defaultTransport[]
+        // No transport specified, using default.
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+        // end::defaultTransport[]
+    }
+
+    public void http11Transport() throws Exception
+    {
+        // tag::http11Transport[]
+        // Configure HTTP/1.1 transport.
+        HttpClientTransportOverHTTP transport = new HttpClientTransportOverHTTP();
+        transport.setHeaderCacheSize(16384);
+
+        HttpClient client = new HttpClient(transport);
+        client.start();
+        // end::http11Transport[]
+    }
+
+    public void http2Transport() throws Exception
+    {
+        // tag::http2Transport[]
+        // The HTTP2Client powers the HTTP/2 transport.
+        HTTP2Client h2Client = new HTTP2Client();
+        h2Client.setInitialSessionRecvWindow(64 * 1024 * 1024);
+
+        // Create and configure the HTTP/2 transport.
+        HttpClientTransportOverHTTP2 transport = new HttpClientTransportOverHTTP2(h2Client);
+        transport.setUseALPN(true);
+
+        HttpClient client = new HttpClient(transport);
+        client.start();
+        // end::http2Transport[]
+    }
+
+    public void fcgiTransport() throws Exception
+    {
+        // tag::fcgiTransport[]
+        String scriptRoot = "/var/www/wordpress";
+        HttpClientTransportOverFCGI transport = new HttpClientTransportOverFCGI(scriptRoot);
+
+        HttpClient client = new HttpClient(transport);
+        client.start();
+        // end::fcgiTransport[]
+    }
+
+    public void dynamicDefault() throws Exception
+    {
+        // tag::dynamicDefault[]
+        // Dynamic transport speaks HTTP/1.1 by default.
+        HttpClientTransportDynamic transport = new HttpClientTransportDynamic();
+
+        HttpClient client = new HttpClient(transport);
+        client.start();
+        // end::dynamicDefault[]
+    }
+
+    public void dynamicOneProtocol() throws Exception
+    {
+        // tag::dynamicOneProtocol[]
+        ClientConnector connector = new ClientConnector();
+
+        // Equivalent to HttpClientTransportOverHTTP.
+        HttpClientTransportDynamic http11Transport = new HttpClientTransportDynamic(connector, HttpClientConnectionFactory.HTTP11);
+
+        // Equivalent to HttpClientTransportOverHTTP2.
+        HTTP2Client http2Client = new HTTP2Client(connector);
+        HttpClientTransportDynamic http2Transport = new HttpClientTransportDynamic(connector, new ClientConnectionFactoryOverHTTP2.HTTP2(http2Client));
+        // end::dynamicOneProtocol[]
+    }
+
+    public void dynamicH1H2() throws Exception
+    {
+        // tag::dynamicH1H2[]
+        ClientConnector connector = new ClientConnector();
+
+        ClientConnectionFactory.Info http1 = HttpClientConnectionFactory.HTTP11;
+
+        HTTP2Client http2Client = new HTTP2Client(connector);
+        ClientConnectionFactoryOverHTTP2.HTTP2 http2 = new ClientConnectionFactoryOverHTTP2.HTTP2(http2Client);
+
+        HttpClientTransportDynamic transport = new HttpClientTransportDynamic(connector, http1, http2);
+
+        HttpClient client = new HttpClient(transport);
+        client.start();
+        // end::dynamicH1H2[]
+    }
+
+    public void dynamicClearText() throws Exception
+    {
+        // tag::dynamicClearText[]
+        ClientConnector connector = new ClientConnector();
+        ClientConnectionFactory.Info http1 = HttpClientConnectionFactory.HTTP11;
+        HTTP2Client http2Client = new HTTP2Client(connector);
+        ClientConnectionFactoryOverHTTP2.HTTP2 http2 = new ClientConnectionFactoryOverHTTP2.HTTP2(http2Client);
+        HttpClientTransportDynamic transport = new HttpClientTransportDynamic(connector, http1, http2);
+        HttpClient client = new HttpClient(transport);
+        client.start();
+
+        // The server supports both HTTP/1.1 and HTTP/2 clear-text on port 8080.
+
+        // Make a clear-text request without explicit version.
+        // The first protocol specified to HttpClientTransportDynamic
+        // is picked, in this example will be HTTP/1.1.
+        ContentResponse http1Response = client.newRequest("host", 8080).send();
+
+        // Make a clear-text request with explicit version.
+        // Clear-text HTTP/2 is used for this request.
+        ContentResponse http2Response = client.newRequest("host", 8080)
+            // Specify the version explicitly.
+            .version(HttpVersion.HTTP_2)
+            .send();
+
+        // Make a clear-text upgrade request from HTTP/1.1 to HTTP/2.
+        // The request will start as HTTP/1.1, but the response will be HTTP/2.
+        ContentResponse upgradedResponse = client.newRequest("host", 8080)
+            .header(HttpHeader.UPGRADE, "h2c")
+            .header(HttpHeader.HTTP2_SETTINGS, "")
+            .header(HttpHeader.CONNECTION, "Upgrade, HTTP2-Settings")
+            .send();
+        // end::dynamicClearText[]
+    }
+
+    public void getConnectionPool() throws Exception
+    {
+        // tag::getConnectionPool[]
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        ConnectionPool connectionPool = httpClient.getDestinations().stream()
+            // Cast to HttpDestination.
+            .map(HttpDestination.class::cast)
+            // Find the destination by filtering on the Origin.
+            .filter(destination -> destination.getOrigin().getAddress().getHost().equals("domain.com"))
+            .findAny()
+            // Get the ConnectionPool.
+            .map(HttpDestination::getConnectionPool)
+            .orElse(null);
+        // end::getConnectionPool[]
+    }
+
+    public void setConnectionPool() throws Exception
+    {
+        // tag::setConnectionPool[]
+        HttpClient httpClient = new HttpClient();
+        httpClient.start();
+
+        // The max number of connections in the pool.
+        int maxConnectionsPerDestination = httpClient.getMaxConnectionsPerDestination();
+
+        // The max number of requests per connection (multiplexing).
+        // Start with 1, since this value is dynamically set to larger values if
+        // the transport supports multiplexing requests on the same connection.
+        int maxRequestsPerConnection = 1;
+
+        HttpClientTransport transport = httpClient.getTransport();
+
+        // Set the ConnectionPool.Factory using a lambda.
+        transport.setConnectionPoolFactory(destination ->
+            new RoundRobinConnectionPool(destination,
+                maxConnectionsPerDestination,
+                destination,
+                maxRequestsPerConnection));
+        // end::setConnectionPool[]
+    }
+}

jetty-documentation/src/main/java/embedded/client/http2/HTTP2ClientDocs.java

@@ -0,0 +1,438 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package embedded.client.http2;
+
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import org.eclipse.jetty.http.HttpFields;
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpURI;
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.http.MetaData;
+import org.eclipse.jetty.http2.ErrorCode;
+import org.eclipse.jetty.http2.api.Session;
+import org.eclipse.jetty.http2.api.Stream;
+import org.eclipse.jetty.http2.client.HTTP2Client;
+import org.eclipse.jetty.http2.frames.DataFrame;
+import org.eclipse.jetty.http2.frames.HeadersFrame;
+import org.eclipse.jetty.http2.frames.PushPromiseFrame;
+import org.eclipse.jetty.http2.frames.ResetFrame;
+import org.eclipse.jetty.http2.frames.SettingsFrame;
+import org.eclipse.jetty.io.ClientConnector;
+import org.eclipse.jetty.util.Callback;
+
+import static java.lang.System.Logger.Level.INFO;
+
+@SuppressWarnings("unused")
+public class HTTP2ClientDocs
+{
+    public void start() throws Exception
+    {
+        // tag::start[]
+        // Instantiate HTTP2Client.
+        HTTP2Client http2Client = new HTTP2Client();
+
+        // Configure HTTP2Client, for example:
+        http2Client.setStreamIdleTimeout(15000);
+
+        // Start HTTP2Client.
+        http2Client.start();
+        // end::start[]
+    }
+
+    public void stop() throws Exception
+    {
+        HTTP2Client http2Client = new HTTP2Client();
+        http2Client.start();
+        // tag::stop[]
+        // Stop HTTP2Client.
+        http2Client.stop();
+        // end::stop[]
+    }
+
+    public void clearTextConnect() throws Exception
+    {
+        HTTP2Client http2Client = new HTTP2Client();
+        http2Client.start();
+        // tag::clearTextConnect[]
+        // Address of the server's clear-text port.
+        SocketAddress serverAddress = new InetSocketAddress("localhost", 8080);
+
+        // Connect to the server, the CompletableFuture will be
+        // notified when the connection is succeeded (or failed).
+        CompletableFuture<Session> sessionCF = http2Client.connect(serverAddress, new Session.Listener.Adapter());
+
+        // Block to obtain the Session.
+        // Alternatively you can use the CompletableFuture APIs to avoid blocking.
+        Session session = sessionCF.get();
+        // end::clearTextConnect[]
+    }
+
+    public void encryptedConnect() throws Exception
+    {
+        // tag::encryptedConnect[]
+        HTTP2Client http2Client = new HTTP2Client();
+        http2Client.start();
+
+        ClientConnector connector = http2Client.getClientConnector();
+
+        // Address of the server's encrypted port.
+        SocketAddress serverAddress = new InetSocketAddress("localhost", 8443);
+
+        // Connect to the server, the CompletableFuture will be
+        // notified when the connection is succeeded (or failed).
+        CompletableFuture<Session> sessionCF = http2Client.connect(connector.getSslContextFactory(), serverAddress, new Session.Listener.Adapter());
+
+        // Block to obtain the Session.
+        // Alternatively you can use the CompletableFuture APIs to avoid blocking.
+        Session session = sessionCF.get();
+        // end::encryptedConnect[]
+    }
+
+    public void configure() throws Exception
+    {
+        HTTP2Client http2Client = new HTTP2Client();
+        http2Client.start();
+
+        // tag::configure[]
+        SocketAddress serverAddress = new InetSocketAddress("localhost", 8080);
+        http2Client.connect(serverAddress, new Session.Listener.Adapter()
+        {
+            @Override
+            public Map<Integer, Integer> onPreface(Session session)
+            {
+                Map<Integer, Integer> configuration = new HashMap<>();
+
+                // Disable push from the server.
+                configuration.put(SettingsFrame.ENABLE_PUSH, 0);
+
+                // Override HTTP2Client.initialStreamRecvWindow for this session.
+                configuration.put(SettingsFrame.INITIAL_WINDOW_SIZE, 1024 * 1024);
+
+                return configuration;
+            }
+        });
+        // end::configure[]
+    }
+
+    public void newStream() throws Exception
+    {
+        HTTP2Client http2Client = new HTTP2Client();
+        http2Client.start();
+        // tag::newStream[]
+        SocketAddress serverAddress = new InetSocketAddress("localhost", 8080);
+        CompletableFuture<Session> sessionCF = http2Client.connect(serverAddress, new Session.Listener.Adapter());
+        Session session = sessionCF.get();
+
+        // Configure the request headers.
+        HttpFields requestHeaders = new HttpFields();
+        requestHeaders.put(HttpHeader.USER_AGENT, "Jetty HTTP2Client {version}");
+
+        // The request metadata with method, URI and headers.
+        MetaData.Request request = new MetaData.Request("GET", new HttpURI("http://localhost:8080/path"), HttpVersion.HTTP_2, requestHeaders);
+
+        // The HTTP/2 HEADERS frame, with endStream=true
+        // to signal that this request has no content.
+        HeadersFrame headersFrame = new HeadersFrame(request, null, true);
+
+        // Open a Stream by sending the HEADERS frame.
+        session.newStream(headersFrame, new Stream.Listener.Adapter());
+        // end::newStream[]
+    }
+
+    public void newStreamWithData() throws Exception
+    {
+        HTTP2Client http2Client = new HTTP2Client();
+        http2Client.start();
+        // tag::newStreamWithData[]
+        SocketAddress serverAddress = new InetSocketAddress("localhost", 8080);
+        CompletableFuture<Session> sessionCF = http2Client.connect(serverAddress, new Session.Listener.Adapter());
+        Session session = sessionCF.get();
+
+        // Configure the request headers.
+        HttpFields requestHeaders = new HttpFields();
+        requestHeaders.put(HttpHeader.CONTENT_TYPE, "application/json");
+
+        // The request metadata with method, URI and headers.
+        MetaData.Request request = new MetaData.Request("POST", new HttpURI("http://localhost:8080/path"), HttpVersion.HTTP_2, requestHeaders);
+
+        // The HTTP/2 HEADERS frame, with endStream=false to
+        // signal that there will be more frames in this stream.
+        HeadersFrame headersFrame = new HeadersFrame(request, null, false);
+
+        // Open a Stream by sending the HEADERS frame.
+        CompletableFuture<Stream> streamCF = session.newStream(headersFrame, new Stream.Listener.Adapter());
+
+        // Block to obtain the Stream.
+        // Alternatively you can use the CompletableFuture APIs to avoid blocking.
+        Stream stream = streamCF.get();
+
+        // The request content, in two chunks.
+        String content1 = "{\"greet\": \"hello world\"}";
+        ByteBuffer buffer1 = StandardCharsets.UTF_8.encode(content1);
+        String content2 = "{\"user\": \"jetty\"}";
+        ByteBuffer buffer2 = StandardCharsets.UTF_8.encode(content2);
+
+        // Send the first DATA frame on the stream, with endStream=false
+        // to signal that there are more frames in this stream.
+        CompletableFuture<Void> dataCF1 = stream.data(new DataFrame(stream.getId(), buffer1, false));
+
+        // Only when the first chunk has been sent we can send the second,
+        // with endStream=true to signal that there are no more frames.
+        dataCF1.thenCompose(ignored -> stream.data(new DataFrame(stream.getId(), buffer2, true)));
+        // end::newStreamWithData[]
+    }
+
+    public void responseListener() throws Exception
+    {
+        HTTP2Client http2Client = new HTTP2Client();
+        http2Client.start();
+        SocketAddress serverAddress = new InetSocketAddress("localhost", 8080);
+        CompletableFuture<Session> sessionCF = http2Client.connect(serverAddress, new Session.Listener.Adapter());
+        Session session = sessionCF.get();
+
+        HttpFields requestHeaders = new HttpFields();
+        requestHeaders.put(HttpHeader.USER_AGENT, "Jetty HTTP2Client {version}");
+        MetaData.Request request = new MetaData.Request("GET", new HttpURI("http://localhost:8080/path"), HttpVersion.HTTP_2, requestHeaders);
+        HeadersFrame headersFrame = new HeadersFrame(request, null, true);
+
+        // tag::responseListener[]
+        // Open a Stream by sending the HEADERS frame.
+        session.newStream(headersFrame, new Stream.Listener.Adapter()
+        {
+            @Override
+            public void onHeaders(Stream stream, HeadersFrame frame)
+            {
+                MetaData metaData = frame.getMetaData();
+
+                // Is this HEADERS frame the response or the trailers?
+                if (metaData.isResponse())
+                {
+                    MetaData.Response response = (MetaData.Response)metaData;
+                    System.getLogger("http2").log(INFO, "Received response {0}", response);
+                }
+                else
+                {
+                    System.getLogger("http2").log(INFO, "Received trailers {0}", metaData.getFields());
+                }
+            }
+
+            @Override
+            public void onData(Stream stream, DataFrame frame, Callback callback)
+            {
+                // Get the content buffer.
+                ByteBuffer buffer = frame.getData();
+
+                // Consume the buffer, here - as an example - just log it.
+                System.getLogger("http2").log(INFO, "Consuming buffer {0}", buffer);
+
+                // Tell the implementation that the buffer has been consumed.
+                callback.succeeded();
+
+                // By returning from the method, implicitly tell the implementation
+                // to deliver to this method more DATA frames when they are available.
+            }
+        });
+        // end::responseListener[]
+    }
+
+    public void responseDataDemanded() throws Exception
+    {
+        HTTP2Client http2Client = new HTTP2Client();
+        http2Client.start();
+        SocketAddress serverAddress = new InetSocketAddress("localhost", 8080);
+        CompletableFuture<Session> sessionCF = http2Client.connect(serverAddress, new Session.Listener.Adapter());
+        Session session = sessionCF.get();
+
+        HttpFields requestHeaders = new HttpFields();
+        requestHeaders.put(HttpHeader.USER_AGENT, "Jetty HTTP2Client {version}");
+        MetaData.Request request = new MetaData.Request("GET", new HttpURI("http://localhost:8080/path"), HttpVersion.HTTP_2, requestHeaders);
+        HeadersFrame headersFrame = new HeadersFrame(request, null, true);
+
+        // tag::responseDataDemanded[]
+        class Chunk
+        {
+            private final ByteBuffer buffer;
+            private final Callback callback;
+
+            Chunk(ByteBuffer buffer, Callback callback)
+            {
+                this.buffer = buffer;
+                this.callback = callback;
+            }
+        }
+
+        // A queue that consumers poll to consume content asynchronously.
+        Queue<Chunk> dataQueue = new ConcurrentLinkedQueue<>();
+
+        // Open a Stream by sending the HEADERS frame.
+        session.newStream(headersFrame, new Stream.Listener.Adapter()
+        {
+            @Override
+            public void onDataDemanded(Stream stream, DataFrame frame, Callback callback)
+            {
+                // Get the content buffer.
+                ByteBuffer buffer = frame.getData();
+
+                // Store buffer to consume it asynchronously, and wrap the callback.
+                dataQueue.offer(new Chunk(buffer, Callback.from(() ->
+                {
+                    // When the buffer has been consumed, then:
+                    // A) succeed the nested callback.
+                    callback.succeeded();
+                    // B) demand more DATA frames.
+                    stream.demand(1);
+                }, callback::failed)));
+
+                // Do not demand more content here, to avoid to overflow the queue.
+            }
+        });
+        // end::responseDataDemanded[]
+    }
+
+    public void reset() throws Exception
+    {
+        HTTP2Client http2Client = new HTTP2Client();
+        http2Client.start();
+        SocketAddress serverAddress = new InetSocketAddress("localhost", 8080);
+        CompletableFuture<Session> sessionCF = http2Client.connect(serverAddress, new Session.Listener.Adapter());
+        Session session = sessionCF.get();
+
+        HttpFields requestHeaders = new HttpFields();
+        requestHeaders.put(HttpHeader.USER_AGENT, "Jetty HTTP2Client {version}");
+        MetaData.Request request = new MetaData.Request("GET", new HttpURI("http://localhost:8080/path"), HttpVersion.HTTP_2, requestHeaders);
+        HeadersFrame headersFrame = new HeadersFrame(request, null, true);
+
+        // tag::reset[]
+        // Open a Stream by sending the HEADERS frame.
+        CompletableFuture<Stream> streamCF = session.newStream(headersFrame, new Stream.Listener.Adapter()
+        {
+            @Override
+            public void onReset(Stream stream, ResetFrame frame)
+            {
+                // The server reset this stream.
+            }
+        });
+        Stream stream = streamCF.get();
+
+        // Reset this stream (for example, the user closed the application).
+        stream.reset(new ResetFrame(stream.getId(), ErrorCode.CANCEL_STREAM_ERROR.code), Callback.NOOP);
+        // end::reset[]
+    }
+
+    public void push() throws Exception
+    {
+        HTTP2Client http2Client = new HTTP2Client();
+        http2Client.start();
+        SocketAddress serverAddress = new InetSocketAddress("localhost", 8080);
+        CompletableFuture<Session> sessionCF = http2Client.connect(serverAddress, new Session.Listener.Adapter());
+        Session session = sessionCF.get();
+
+        HttpFields requestHeaders = new HttpFields();
+        requestHeaders.put(HttpHeader.USER_AGENT, "Jetty HTTP2Client {version}");
+        MetaData.Request request = new MetaData.Request("GET", new HttpURI("http://localhost:8080/path"), HttpVersion.HTTP_2, requestHeaders);
+        HeadersFrame headersFrame = new HeadersFrame(request, null, true);
+
+        // tag::push[]
+        // Open a Stream by sending the HEADERS frame.
+        CompletableFuture<Stream> streamCF = session.newStream(headersFrame, new Stream.Listener.Adapter()
+        {
+            @Override
+            public Stream.Listener onPush(Stream pushedStream, PushPromiseFrame frame)
+            {
+                // The "request" the client would make for the pushed resource.
+                MetaData.Request pushedRequest = frame.getMetaData();
+                // The pushed "request" URI.
+                HttpURI pushedURI = pushedRequest.getURI();
+                // The pushed "request" headers.
+                HttpFields pushedRequestHeaders = pushedRequest.getFields();
+
+                // If needed, retrieve the primary stream that triggered the push.
+                Stream primaryStream = pushedStream.getSession().getStream(frame.getStreamId());
+
+                // Return a Stream.Listener to listen for the pushed "response" events.
+                return new Stream.Listener.Adapter()
+                {
+                    @Override
+                    public void onHeaders(Stream stream, HeadersFrame frame)
+                    {
+                        // Handle the pushed stream "response".
+
+                        MetaData metaData = frame.getMetaData();
+                        if (metaData.isResponse())
+                        {
+                            // The pushed "response" headers.
+                            HttpFields pushedResponseHeaders = metaData.getFields();
+                        }
+                    }
+
+                    @Override
+                    public void onData(Stream stream, DataFrame frame, Callback callback)
+                    {
+                        // Handle the pushed stream "response" content.
+
+                        // The pushed stream "response" content bytes.
+                        ByteBuffer buffer = frame.getData();
+                        // Consume the buffer and complete the callback.
+                        callback.succeeded();
+                    }
+                };
+            }
+        });
+        // end::push[]
+    }
+
+    public void pushReset() throws Exception
+    {
+        HTTP2Client http2Client = new HTTP2Client();
+        http2Client.start();
+        SocketAddress serverAddress = new InetSocketAddress("localhost", 8080);
+        CompletableFuture<Session> sessionCF = http2Client.connect(serverAddress, new Session.Listener.Adapter());
+        Session session = sessionCF.get();
+
+        HttpFields requestHeaders = new HttpFields();
+        requestHeaders.put(HttpHeader.USER_AGENT, "Jetty HTTP2Client {version}");
+        MetaData.Request request = new MetaData.Request("GET", new HttpURI("http://localhost:8080/path"), HttpVersion.HTTP_2, requestHeaders);
+        HeadersFrame headersFrame = new HeadersFrame(request, null, true);
+
+        // tag::pushReset[]
+        // Open a Stream by sending the HEADERS frame.
+        CompletableFuture<Stream> streamCF = session.newStream(headersFrame, new Stream.Listener.Adapter()
+        {
+            @Override
+            public Stream.Listener onPush(Stream pushedStream, PushPromiseFrame frame)
+            {
+                // Reset the pushed stream to tell the server we are not interested.
+                pushedStream.reset(new ResetFrame(pushedStream.getId(), ErrorCode.CANCEL_STREAM_ERROR.code), Callback.NOOP);
+
+                // Not interested in listening to pushed response events.
+                return null;
+            }
+        });
+        // end::pushReset[]
+    }
+}