Merge branch 'jetty-10.0.x' of github.com:eclipse/jetty.project into jetty-10.0.x

Jenkinsfile

@@ -3,51 +3,46 @@
 pipeline {
   agent any
   // save some io during the build
-  options { durabilityHint('PERFORMANCE_OPTIMIZED') }
+  options { durabilityHint( 'PERFORMANCE_OPTIMIZED' ) }
   stages {
-    stage("Parallel Stage") {
+    stage( "Parallel Stage" ) {
       parallel {
-        stage("Build / Test - JDK11") {
+        stage( "Build / Test - JDK11" ) {
           agent {
             node { label 'linux' }
           }
-          options { timeout(time: 120, unit: 'MINUTES') }
           steps {
-            container('jetty-build') {
+            container( 'jetty-build' ) {
-              mavenBuild("jdk11", "-T3 -Pmongodb clean install", "maven3", true) // -Pautobahn
+              timeout( time: 120, unit: 'MINUTES' ) {
-              // Collect up the jacoco execution results (only on main build)
+                mavenBuild( "jdk11", "-T3 -Pmongodb clean install", "maven3", true ) // -Pautobahn
-              jacoco inclusionPattern: '**/org/eclipse/jetty/**/*.class',
+                // Collect up the jacoco execution results (only on main build)
-                exclusionPattern: '' +
+                jacoco inclusionPattern: '**/org/eclipse/jetty/**/*.class',
-                  // build tools
+                       exclusionPattern: '' +
-                  '**/org/eclipse/jetty/ant/**' +
+                               // build tools
-                  ',**/org/eclipse/jetty/maven/**' +
+                               '**/org/eclipse/jetty/ant/**' + ',**/org/eclipse/jetty/maven/**' +
-                  ',**/org/eclipse/jetty/jspc/**' +
+                               ',**/org/eclipse/jetty/jspc/**' +
-                  // example code / documentation
+                               // example code / documentation
-                  ',**/org/eclipse/jetty/embedded/**' +
+                               ',**/org/eclipse/jetty/embedded/**' + ',**/org/eclipse/jetty/asyncrest/**' +
-                  ',**/org/eclipse/jetty/asyncrest/**' +
+                               ',**/org/eclipse/jetty/demo/**' +
-                  ',**/org/eclipse/jetty/demo/**' +
+                               // special environments / late integrations
-                  // special environments / late integrations
+                               ',**/org/eclipse/jetty/gcloud/**' + ',**/org/eclipse/jetty/infinispan/**' +
-                  ',**/org/eclipse/jetty/gcloud/**' +
+                               ',**/org/eclipse/jetty/osgi/**' + ',**/org/eclipse/jetty/spring/**' +
-                  ',**/org/eclipse/jetty/infinispan/**' +
+                               ',**/org/eclipse/jetty/http/spi/**' +
-                  ',**/org/eclipse/jetty/osgi/**' +
+                               // test classes
-                  ',**/org/eclipse/jetty/spring/**' +
+                               ',**/org/eclipse/jetty/tests/**' + ',**/org/eclipse/jetty/test/**',
-                  ',**/org/eclipse/jetty/http/spi/**' +
+                       execPattern: '**/target/jacoco.exec',
-                  // test classes
+                       classPattern: '**/target/classes',
-                  ',**/org/eclipse/jetty/tests/**' +
+                       sourcePattern: '**/src/main/java'
-                  ',**/org/eclipse/jetty/test/**',
+                warnings consoleParsers: [[parserName: 'Maven'], [parserName: 'Java']]
-                execPattern: '**/target/jacoco.exec',
+                junit testResults: '**/target/surefire-reports/*.xml,**/target/invoker-reports/TEST*.xml,**/target/autobahntestsuite-reports/*.xml'
-                classPattern: '**/target/classes',
+              }
-                sourcePattern: '**/src/main/java'
-              warnings consoleParsers: [[parserName: 'Maven'], [parserName: 'Java']]
-              junit testResults: '**/target/surefire-reports/*.xml,**/target/invoker-reports/TEST*.xml,**/target/autobahntestsuite-reports/*.xml'
             }
           }
         }
-
+        stage( "Build / Test - JDK14" ) {
-        stage("Build / Test - JDK14") {
           agent { node { label 'linux' } }
           steps {
-            container('jetty-build') {
+            container( 'jetty-build' ) {
               timeout( time: 120, unit: 'MINUTES' ) {
                 mavenBuild( "jdk14", "-T3 -Pmongodb clean install", "maven3", true )
                 warnings consoleParsers: [[parserName: 'Maven'], [parserName: 'Java']]
@@ -57,10 +52,10 @@ pipeline {
           }
         }
 
-        stage("Build Javadoc") {
+        stage( "Build Javadoc" ) {
           agent { node { label 'linux' } }
           steps {
-            container('jetty-build') {
+            container( 'jetty-build' ) {
               timeout( time: 30, unit: 'MINUTES' ) {
                 mavenBuild( "jdk11",
                             "package source:jar javadoc:jar javadoc:aggregate-jar -Peclipse-release  -DskipTests -Dpmd.skip=true -Dcheckstyle.skip=true",
@@ -70,6 +65,17 @@ pipeline {
             }
           }
         }
+        stage( "Build Compact3" ) {
+          agent { node { label 'linux' } }
+          steps {
+            container( 'jetty-build' ) {
+              timeout( time: 30, unit: 'MINUTES' ) {
+                mavenBuild( "jdk11", "-T3 -Pcompact3 clean install -DskipTests", "maven3", true )
+                warnings consoleParsers: [[parserName: 'Maven'], [parserName: 'Java']]
+              }
+            }
+          }
+        }
       }
     }
   }
@@ -86,10 +92,11 @@ pipeline {
   }
 }
 
+
 def slackNotif() {
   script {
     try {
-      if (env.BRANCH_NAME == 'jetty-10.0.x' || env.BRANCH_NAME == 'jetty-9.4.x') {
+      if (env.BRANCH_NAME == 'jetty-10.0.x' || env.BRANCH_NAME == 'jetty-9.4.x' || env.BRANCH_NAME == 'jetty-11.0.x') {
         //BUILD_USER = currentBuild.rawBuild.getCause(Cause.UserIdCause).getUserId()
         // by ${BUILD_USER}
         COLOR_MAP = ['SUCCESS': 'good', 'FAILURE': 'danger', 'UNSTABLE': 'danger', 'ABORTED': 'danger']
@@ -126,7 +133,7 @@ def mavenBuild(jdk, cmdline, mvnName, junitPublishDisabled) {
     mavenOpts: mavenOpts,
     mavenLocalRepo: localRepo) {
     // Some common Maven command line + provided command line
-    sh "mvn -Pci -V -B -e -fae -Dmaven.test.failure.ignore=true -Djetty.testtracker.log=true $cmdline -Dunix.socket.tmp=" + env.JENKINS_HOME
+    sh "mvn -Premote-session-tests -Pci -V -B -e -fae -Dmaven.test.failure.ignore=true -Djetty.testtracker.log=true $cmdline -Dunix.socket.tmp=" + env.JENKINS_HOME
   }
 }
 

aggregates/jetty-all/pom.xml

@@ -178,7 +178,7 @@
     <!-- websocket support -->
     <dependency>
       <groupId>org.eclipse.jetty.websocket</groupId>
-      <artifactId>websocket-servlet</artifactId>
+      <artifactId>websocket-util-server</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>

aggregates/jetty-websocket-all/pom.xml

@@ -107,7 +107,7 @@
   <dependencies>
     <dependency>
       <groupId>org.eclipse.jetty.websocket</groupId>
-      <artifactId>websocket-servlet</artifactId>
+      <artifactId>websocket-util-server</artifactId>
       <version>${project.version}</version>
       <scope>provided</scope>
     </dependency>

examples/embedded/pom.xml

@@ -66,7 +66,7 @@
     </dependency>
     <dependency>
       <groupId>org.eclipse.jetty.websocket</groupId>
-      <artifactId>websocket-servlet</artifactId>
+      <artifactId>websocket-util-server</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>

examples/embedded/src/test/java/org/eclipse/jetty/embedded/ManyHandlersTest.java

@@ -23,6 +23,7 @@ import java.util.Map;
 
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpHeaderValue;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.server.Server;
@@ -59,7 +60,7 @@ public class ManyHandlersTest extends AbstractEmbeddedTest
 
         ContentResponse response = client.newRequest(uri)
             .method(HttpMethod.GET)
-            .header(HttpHeader.ACCEPT_ENCODING, "gzip")
+            .headers(headers -> headers.put(HttpHeader.ACCEPT_ENCODING, HttpHeaderValue.GZIP))
             .send();
         assertThat("HTTP Response Status", response.getStatus(), is(HttpStatus.OK_200));
 
@@ -84,7 +85,7 @@ public class ManyHandlersTest extends AbstractEmbeddedTest
         URI uri = server.getURI().resolve("/hello");
         ContentResponse response = client.newRequest(uri)
             .method(HttpMethod.GET)
-            .header(HttpHeader.ACCEPT_ENCODING, "gzip")
+            .headers(headers -> headers.put(HttpHeader.ACCEPT_ENCODING, HttpHeaderValue.GZIP))
             .send();
         assertThat("HTTP Response Status", response.getStatus(), is(HttpStatus.OK_200));
 

examples/embedded/src/test/java/org/eclipse/jetty/embedded/SecuredHelloHandlerTest.java

@@ -72,7 +72,7 @@ public class SecuredHelloHandlerTest extends AbstractEmbeddedTest
         String authEncoded = Base64.getEncoder().encodeToString("user:password".getBytes(UTF_8));
         ContentResponse response = client.newRequest(uri)
             .method(HttpMethod.GET)
-            .header(HttpHeader.AUTHORIZATION, "Basic " + authEncoded)
+            .headers(headers -> headers.put(HttpHeader.AUTHORIZATION, "Basic " + authEncoded))
             .send();
         assertThat("HTTP Response Status", response.getStatus(), is(HttpStatus.OK_200));
 

jetty-bom/pom.xml

@@ -376,7 +376,7 @@
       </dependency>
       <dependency>
         <groupId>org.eclipse.jetty.websocket</groupId>
-        <artifactId>websocket-servlet</artifactId>
+        <artifactId>websocket-util-server</artifactId>
         <version>10.0.0-SNAPSHOT</version>
       </dependency>
       <dependency>

jetty-client/src/main/java/org/eclipse/jetty/client/AuthenticationProtocolHandler.java

@@ -256,7 +256,7 @@ public abstract class AuthenticationProtocolHandler implements ProtocolHandler
         {
             HttpField field = oldRequest.getHeaders().getField(header);
             if (field != null && !newRequest.getHeaders().contains(header))
-                newRequest.put(field);
+                newRequest.headers(headers -> headers.put(field));
         }
 
         private void forwardSuccessComplete(HttpRequest request, Response response)

jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java

@@ -444,7 +444,7 @@ public class HttpClient extends ContainerLifeCycle
 
     protected Request copyRequest(HttpRequest oldRequest, URI newURI)
     {
-        Request newRequest = newHttpRequest(oldRequest.getConversation(), newURI);
+        HttpRequest newRequest = newHttpRequest(oldRequest.getConversation(), newURI);
         newRequest.method(oldRequest.getMethod())
             .version(oldRequest.getVersion())
             .body(oldRequest.getBody())
@@ -471,10 +471,8 @@ public class HttpClient extends ContainerLifeCycle
                 HttpHeader.PROXY_AUTHORIZATION == header)
                 continue;
 
-            String name = field.getName();
+            if (!newRequest.getHeaders().contains(field))
-            String value = field.getValue();
+                newRequest.addHeader(field);
-            if (!newRequest.getHeaders().contains(name, value))
-                newRequest.header(name, value);
         }
         return newRequest;
     }

jetty-client/src/main/java/org/eclipse/jetty/client/HttpConnection.java

@@ -122,9 +122,9 @@ public abstract class HttpConnection implements IConnection
         }
     }
 
-    protected void normalizeRequest(Request request)
+    protected void normalizeRequest(HttpRequest request)
     {
-        boolean normalized = ((HttpRequest)request).normalized();
+        boolean normalized = request.normalized();
         if (LOG.isDebugEnabled())
             LOG.debug("Normalizing {} {}", !normalized, request);
         if (normalized)
@@ -153,7 +153,7 @@ public abstract class HttpConnection implements IConnection
         if (version.getVersion() <= 11)
         {
             if (!headers.contains(HttpHeader.HOST))
-                request.put(getHttpDestination().getHostField());
+                request.addHeader(getHttpDestination().getHostField());
         }
 
         // Add content headers
@@ -167,22 +167,19 @@ public abstract class HttpConnection implements IConnection
             if (!headers.contains(HttpHeader.CONTENT_TYPE))
             {
                 String contentType = content.getContentType();
+                if (contentType == null)
+                    contentType = getHttpClient().getDefaultRequestContentType();
                 if (contentType != null)
                 {
-                    request.put(new HttpField(HttpHeader.CONTENT_TYPE, contentType));
+                    HttpField field = new HttpField(HttpHeader.CONTENT_TYPE, contentType);
-                }
+                    request.addHeader(field);
-                else
-                {
-                    contentType = getHttpClient().getDefaultRequestContentType();
-                    if (contentType != null)
-                        request.put(new HttpField(HttpHeader.CONTENT_TYPE, contentType));
                 }
             }
             long contentLength = content.getLength();
             if (contentLength >= 0)
             {
                 if (!headers.contains(HttpHeader.CONTENT_LENGTH))
-                    request.put(new HttpField.LongValueHttpField(HttpHeader.CONTENT_LENGTH, contentLength));
+                    request.addHeader(new HttpField.LongValueHttpField(HttpHeader.CONTENT_LENGTH, contentLength));
             }
         }
 
@@ -195,7 +192,10 @@ public abstract class HttpConnection implements IConnection
                 cookies = convertCookies(HttpCookieStore.matchPath(uri, cookieStore.get(uri)), null);
             cookies = convertCookies(request.getCookies(), cookies);
             if (cookies != null)
-                request.header(HttpHeader.COOKIE, cookies.toString());
+            {
+                HttpField cookieField = new HttpField(HttpHeader.COOKIE, cookies.toString());
+                request.addHeader(cookieField);
+            }
         }
 
         // Authentication

jetty-client/src/main/java/org/eclipse/jetty/client/HttpProxy.java

@@ -200,9 +200,9 @@ public class HttpProxy extends ProxyConfiguration.Proxy
             Origin.Address proxyAddress = destination.getConnectAddress();
             HttpClient httpClient = destination.getHttpClient();
             Request connect = new TunnelRequest(httpClient, proxyAddress)
-                    .method(HttpMethod.CONNECT)
+                .method(HttpMethod.CONNECT)
-                    .path(target)
+                .path(target)
-                    .header(HttpHeader.HOST, target);
+                .headers(headers -> headers.put(HttpHeader.HOST, target));
             ProxyConfiguration.Proxy proxy = destination.getProxy();
             if (proxy.isSecure())
                 connect.scheme(HttpScheme.HTTPS.asString());
@@ -262,8 +262,7 @@ public class HttpProxy extends ProxyConfiguration.Proxy
                 }
                 else
                 {
-                    HttpResponseException failure = new HttpResponseException("Unexpected " + response +
+                    HttpResponseException failure = new HttpResponseException("Unexpected " + response + " for " + response.getRequest(), response);
-                            " for " + response.getRequest(), response);
                     tunnelFailed(endPoint, failure);
                 }
             }

jetty-client/src/main/java/org/eclipse/jetty/client/HttpReceiver.java

@@ -72,7 +72,7 @@ import org.slf4j.LoggerFactory;
  */
 public abstract class HttpReceiver
 {
-    protected static final Logger LOG = LoggerFactory.getLogger(HttpReceiver.class);
+    private static final Logger LOG = LoggerFactory.getLogger(HttpReceiver.class);
 
     private final AtomicReference<ResponseState> responseState = new AtomicReference<>(ResponseState.IDLE);
     private final HttpChannel channel;
@@ -242,7 +242,7 @@ public abstract class HttpReceiver
         boolean process = notifier.notifyHeader(exchange.getConversation().getResponseListeners(), response, field);
         if (process)
         {
-            response.getHeaderFieldsMutable().add(field);
+            response.addHeader(field);
             HttpHeader fieldHeader = field.getHeader();
             if (fieldHeader != null)
             {

jetty-client/src/main/java/org/eclipse/jetty/client/HttpRequest.java

@@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiFunction;
+import java.util.function.Consumer;
 import java.util.function.LongConsumer;
 import java.util.function.Supplier;
 
@@ -306,34 +307,7 @@ public class HttpRequest implements Request
     }
 
     @Override
-    public Request set(HttpFields fields)
+    @Deprecated
-    {
-        headers.clear().add(fields);
-        return this;
-    }
-
-    @Override
-    public Request remove(HttpHeader header)
-    {
-        headers.remove(header);
-        return this;
-    }
-
-    @Override
-    public Request put(HttpField field)
-    {
-        headers.put(field);
-        return this;
-    }
-
-    @Override
-    public Request add(HttpField field)
-    {
-        headers.add(field);
-        return this;
-    }
-
-    @Override
     public Request header(String name, String value)
     {
         if (value == null)
@@ -344,6 +318,7 @@ public class HttpRequest implements Request
     }
 
     @Override
+    @Deprecated
     public Request header(HttpHeader header, String value)
     {
         if (value == null)
@@ -402,6 +377,19 @@ public class HttpRequest implements Request
         return headers;
     }
 
+    @Override
+    public Request headers(Consumer<HttpFields.Mutable> consumer)
+    {
+        consumer.accept(headers);
+        return this;
+    }
+
+    public HttpRequest addHeader(HttpField header)
+    {
+        headers.add(header);
+        return this;
+    }
+
     @Override
     @SuppressWarnings("unchecked")
     public <T extends RequestListener> List<T> getRequestListeners(Class<T> type)
@@ -707,7 +695,7 @@ public class HttpRequest implements Request
     public Request content(ContentProvider content, String contentType)
     {
         if (contentType != null)
-            header(HttpHeader.CONTENT_TYPE, contentType);
+            headers.put(HttpHeader.CONTENT_TYPE, contentType);
         return body(ContentProvider.toRequestContent(content));
     }
 
@@ -886,7 +874,7 @@ public class HttpRequest implements Request
      * headers, etc.</p>
      *
      * @return whether this request was already normalized
-     * @see HttpConnection#normalizeRequest(Request)
+     * @see HttpConnection#normalizeRequest(HttpRequest)
      */
     boolean normalized()
     {

jetty-client/src/main/java/org/eclipse/jetty/client/HttpResponse.java

@@ -20,6 +20,7 @@ package org.eclipse.jetty.client;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Consumer;
 
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
@@ -91,9 +92,16 @@ public class HttpResponse implements Response
         return headers.asImmutable();
     }
 
-    public HttpFields.Mutable getHeaderFieldsMutable()
+    public HttpResponse addHeader(HttpField header)
     {
-        return headers;
+        headers.add(header);
+        return this;
+    }
+
+    public HttpResponse headers(Consumer<HttpFields.Mutable> consumer)
+    {
+        consumer.accept(headers);
+        return this;
     }
 
     @Override
@@ -110,7 +118,7 @@ public class HttpResponse implements Response
 
     public HttpFields getTrailers()
     {
-        return trailers;
+        return trailers == null ? null : trailers.asImmutable();
     }
 
     public HttpResponse trailer(HttpField trailer)

jetty-client/src/main/java/org/eclipse/jetty/client/api/Request.java

@@ -31,10 +31,10 @@ import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.function.Consumer;
 
 import org.eclipse.jetty.client.HttpClient;
 import org.eclipse.jetty.client.util.InputStreamResponseListener;
-import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpMethod;
@@ -166,38 +166,22 @@ public interface Request
      */
     HttpFields getHeaders();
 
-    /** Set the headers, clearing any existing headers
-     * @param fields The fields to set
-     * @return this request object
-     */
-    Request set(HttpFields fields);
-
     /**
-     * @param header the header to remove
+     * Modifies the headers of this request.
+     *
+     * @param consumer the code that modifies the headers of this request
      * @return this request object
      */
-    Request remove(HttpHeader header);
+    Request headers(Consumer<HttpFields.Mutable> consumer);
-
-    /**
-     * @param field the field to add
-     * @return this request object
-     * @see #header(HttpHeader, String)
-     */
-    Request add(HttpField field);
-
-    /**
-     * @param field the field to put
-     * @return this request object
-     * @see #header(HttpHeader, String)
-     */
-    Request put(HttpField field);
 
     /**
      * @param name the name of the header
      * @param value the value of the header
      * @return this request object
      * @see #header(HttpHeader, String)
+     * @deprecated use {@link #headers(Consumer)} instead
      */
+    @Deprecated
     Request header(String name, String value);
 
     /**
@@ -208,7 +192,9 @@ public interface Request
      * @param header the header name
      * @param value the value of the header
      * @return this request object
+     * @deprecated use {@link #headers(Consumer)} instead
      */
+    @Deprecated
     Request header(HttpHeader header, String value);
 
     /**

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpConnectionOverHTTP.java

@@ -261,7 +261,7 @@ public class HttpConnectionOverHTTP extends AbstractConnection implements IConne
         @Override
         public SendFailure send(HttpExchange exchange)
         {
-            Request request = exchange.getRequest();
+            HttpRequest request = exchange.getRequest();
             normalizeRequest(request);
 
             // Save the old idle timeout to restore it.
@@ -276,7 +276,7 @@ public class HttpConnectionOverHTTP extends AbstractConnection implements IConne
         }
 
         @Override
-        protected void normalizeRequest(Request request)
+        protected void normalizeRequest(HttpRequest request)
         {
             super.normalizeRequest(request);
 
@@ -287,8 +287,7 @@ public class HttpConnectionOverHTTP extends AbstractConnection implements IConne
                         .idleTimeout(2 * connectTimeout, TimeUnit.MILLISECONDS);
             }
 
-            HttpRequest httpRequest = (HttpRequest)request;
+            HttpConversation conversation = request.getConversation();
-            HttpConversation conversation = httpRequest.getConversation();
             HttpUpgrader upgrader = (HttpUpgrader)conversation.getAttribute(HttpUpgrader.class.getName());
             if (upgrader == null)
             {
@@ -296,7 +295,7 @@ public class HttpConnectionOverHTTP extends AbstractConnection implements IConne
                 {
                     upgrader = ((HttpUpgrader.Factory)request).newHttpUpgrader(HttpVersion.HTTP_1_1);
                     conversation.setAttribute(HttpUpgrader.class.getName(), upgrader);
-                    upgrader.prepare(httpRequest);
+                    upgrader.prepare(request);
                 }
                 else
                 {
@@ -305,7 +304,7 @@ public class HttpConnectionOverHTTP extends AbstractConnection implements IConne
                     {
                         upgrader = new ProtocolHttpUpgrader(getHttpDestination(), protocol);
                         conversation.setAttribute(HttpUpgrader.class.getName(), upgrader);
-                        upgrader.prepare(httpRequest);
+                        upgrader.prepare(request);
                     }
                 }
             }

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpReceiverOverHTTP.java

@@ -38,9 +38,13 @@ import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.io.RetainableByteBuffer;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.Callback;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class HttpReceiverOverHTTP extends HttpReceiver implements HttpParser.ResponseHandler
 {
+    private static final Logger LOG = LoggerFactory.getLogger(HttpReceiverOverHTTP.class);
+
     private final HttpParser parser;
     private RetainableByteBuffer networkBuffer;
     private boolean shutdown;

jetty-client/src/main/java/org/eclipse/jetty/client/util/BasicAuthentication.java

@@ -105,7 +105,7 @@ public class BasicAuthentication extends AbstractAuthentication
         public void apply(Request request)
         {
             if (!request.getHeaders().contains(header, value))
-                request.header(header, value);
+                request.headers(headers -> headers.add(header, value));
         }
 
         @Override

jetty-client/src/main/java/org/eclipse/jetty/client/util/DigestAuthentication.java

@@ -204,7 +204,7 @@ public class DigestAuthentication extends AbstractAuthentication
             }
             value.append(", response=\"").append(hashA3).append("\"");
 
-            request.header(header, value.toString());
+            request.headers(headers -> headers.add(header, value.toString()));
         }
 
         private String nextNonceCount()

jetty-client/src/main/java/org/eclipse/jetty/client/util/FormContentProvider.java

@@ -18,11 +18,8 @@
 
 package org.eclipse.jetty.client.util;
 
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
-import java.nio.charset.UnsupportedCharsetException;
 
 import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.util.Fields;
@@ -53,29 +50,6 @@ public class FormContentProvider extends StringContentProvider
 
     public static String convert(Fields fields, Charset charset)
     {
-        // Assume 32 chars between name and value.
+        return FormRequestContent.convert(fields, charset);
-        StringBuilder builder = new StringBuilder(fields.getSize() * 32);
-        for (Fields.Field field : fields)
-        {
-            for (String value : field.getValues())
-            {
-                if (builder.length() > 0)
-                    builder.append("&");
-                builder.append(encode(field.getName(), charset)).append("=").append(encode(value, charset));
-            }
-        }
-        return builder.toString();
-    }
-
-    private static String encode(String value, Charset charset)
-    {
-        try
-        {
-            return URLEncoder.encode(value, charset.name());
-        }
-        catch (UnsupportedEncodingException x)
-        {
-            throw new UnsupportedCharsetException(charset.name());
-        }
     }
 }

jetty-client/src/main/java/org/eclipse/jetty/client/util/SPNEGOAuthentication.java

@@ -307,7 +307,7 @@ public class SPNEGOAuthentication extends AbstractAuthentication
         @Override
         public void apply(Request request)
         {
-            request.header(header, value);
+            request.headers(headers -> headers.add(header, value));
         }
     }
 

jetty-client/src/test/java/org/eclipse/jetty/client/ClientConnectionCloseTest.java

@@ -86,7 +86,7 @@ public class ClientConnectionCloseTest extends AbstractHttpClientServerTest
 
         var request = client.newRequest(host, port)
             .scheme(scenario.getScheme())
-            .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString())
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE))
             .body(new StringRequestContent("0"))
             .onRequestSuccess(r ->
             {
@@ -126,7 +126,7 @@ public class ClientConnectionCloseTest extends AbstractHttpClientServerTest
         long idleTimeout = 1000;
         var request = client.newRequest(host, port)
             .scheme(scenario.getScheme())
-            .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString())
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE))
             .idleTimeout(idleTimeout, TimeUnit.MILLISECONDS)
             .onRequestSuccess(r ->
             {
@@ -188,7 +188,7 @@ public class ClientConnectionCloseTest extends AbstractHttpClientServerTest
         CountDownLatch resultLatch = new CountDownLatch(1);
         var request = client.newRequest(host, port)
             .scheme(scenario.getScheme())
-            .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString())
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE))
             .body(content)
             .idleTimeout(idleTimeout, TimeUnit.MILLISECONDS)
             .onRequestSuccess(r ->
@@ -242,7 +242,7 @@ public class ClientConnectionCloseTest extends AbstractHttpClientServerTest
 
         var request = client.newRequest(host, port)
             .scheme(scenario.getScheme())
-            .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString())
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE))
             .onRequestSuccess(r ->
             {
                 HttpDestination destination = (HttpDestination)client.resolveDestination(r);
@@ -250,10 +250,7 @@ public class ClientConnectionCloseTest extends AbstractHttpClientServerTest
                 HttpConnectionOverHTTP connection = (HttpConnectionOverHTTP)connectionPool.getActiveConnections().iterator().next();
                 assertFalse(connection.getEndPoint().isOutputShutdown());
             })
-            .onResponseHeaders(r ->
+            .onResponseHeaders(r -> ((HttpResponse)r).headers(headers -> headers.remove(HttpHeader.CONNECTION)));
-            {
-                ((HttpResponse)r).getHeaderFieldsMutable().remove(HttpHeader.CONNECTION);
-            });
         ContentResponse response = request.send();
 
         assertEquals(HttpStatus.OK_200, response.getStatus());

jetty-client/src/test/java/org/eclipse/jetty/client/ConnectionPoolTest.java

@@ -19,7 +19,6 @@
 package org.eclipse.jetty.client;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
@@ -36,6 +35,7 @@ import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
 import org.eclipse.jetty.client.util.BytesRequestContent;
 import org.eclipse.jetty.client.util.FutureResponseListener;
 import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpHeaderValue;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.server.Handler;
@@ -45,7 +45,6 @@ import org.eclipse.jetty.util.IO;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -58,20 +57,10 @@ public class ConnectionPoolTest
     private ServerConnector connector;
     private HttpClient client;
 
-    public static Stream<Arguments> pools()
+    public static Stream<ConnectionPool.Factory> pools()
     {
-        List<Object[]> pools = new ArrayList<>();
+        return Stream.of(destination -> new DuplexConnectionPool(destination, 8, destination),
-        pools.add(new Object[]{
+            destination -> new RoundRobinConnectionPool(destination, 8, destination));
-            DuplexConnectionPool.class,
-            (ConnectionPool.Factory)
-                destination -> new DuplexConnectionPool(destination, 8, destination)
-        });
-        pools.add(new Object[]{
-            RoundRobinConnectionPool.class,
-            (ConnectionPool.Factory)
-                destination -> new RoundRobinConnectionPool(destination, 8, destination)
-        });
-        return pools.stream().map(Arguments::of);
     }
 
     private void start(final ConnectionPool.Factory factory, Handler handler) throws Exception
@@ -112,7 +101,7 @@ public class ConnectionPoolTest
 
     @ParameterizedTest(name = "[{index}] {0}")
     @MethodSource("pools")
-    public void test(Class<? extends ConnectionPool> connectionPoolClass, ConnectionPool.Factory factory) throws Exception
+    public void test(ConnectionPool.Factory factory) throws Exception
     {
         start(factory, new EmptyServerHandler()
         {
@@ -202,17 +191,17 @@ public class ConnectionPoolTest
             .method(method);
 
         if (clientClose)
-            request.header(HttpHeader.CONNECTION, "close");
+            request.headers(fields -> fields.put(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE));
         else if (serverClose)
-            request.header("X-Close", "true");
+            request.headers(fields -> fields.put("X-Close", "true"));
 
         switch (method)
         {
             case GET:
-                request.header("X-Download", String.valueOf(contentLength));
+                request.headers(fields -> fields.put("X-Download", String.valueOf(contentLength)));
                 break;
             case POST:
-                request.header(HttpHeader.CONTENT_LENGTH, String.valueOf(contentLength));
+                request.headers(fields -> fields.put(HttpHeader.CONTENT_LENGTH, String.valueOf(contentLength)));
                 request.body(new BytesRequestContent(new byte[contentLength]));
                 break;
             default:

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientCorrelationDataTest.java

@@ -54,7 +54,7 @@ public class HttpClientCorrelationDataTest extends AbstractHttpClientServerTest
             @Override
             public void onQueued(Request request)
             {
-                request.header(correlationName, correlation.get());
+                request.headers(headers -> headers.put(correlationName, correlation.get()));
             }
         });
 

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientProxyProtocolTest.java

@@ -29,6 +29,7 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Destination;
 import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpHeaderValue;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.http.MimeTypes;
 import org.eclipse.jetty.io.EndPoint;
@@ -248,7 +249,7 @@ public class HttpClientProxyProtocolTest
         // The proxy maps the client address, then sends the request.
         ContentResponse response = client.newRequest("localhost", serverPort)
             .tag(tag)
-            .header(HttpHeader.CONNECTION, "close")
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE))
             .send();
 
         assertEquals(HttpStatus.OK_200, response.getStatus());

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientProxyTest.java

@@ -379,7 +379,7 @@ public class HttpClientProxyTest extends AbstractHttpClientServerTest
         // Make a request, expect 407 + 204.
         ContentResponse response1 = client.newRequest(serverHost, serverPort)
             .scheme(scenario.getScheme())
-            .header(HttpHeader.AUTHORIZATION, "Basic foobar")
+            .headers(headers -> headers.put(HttpHeader.AUTHORIZATION, "Basic foobar"))
             .timeout(5, TimeUnit.SECONDS)
             .send();
 
@@ -390,7 +390,7 @@ public class HttpClientProxyTest extends AbstractHttpClientServerTest
         requests.set(0);
         ContentResponse response2 = client.newRequest(serverHost, serverPort)
             .scheme(scenario.getScheme())
-            .header(HttpHeader.AUTHORIZATION, "Basic foobar")
+            .headers(headers -> headers.put(HttpHeader.AUTHORIZATION, "Basic foobar"))
             .timeout(5, TimeUnit.SECONDS)
             .send();
 

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTLSTest.java

@@ -44,6 +44,7 @@ import javax.net.ssl.SSLSocket;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
 import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpHeaderValue;
 import org.eclipse.jetty.http.HttpScheme;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.io.ByteBufferPool;
@@ -380,7 +381,7 @@ public class HttpClientTLSTest
         // First request primes the TLS session.
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(HttpScheme.HTTPS.asString())
-            .header(HttpHeader.CONNECTION, "close")
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE))
             .timeout(5, TimeUnit.SECONDS)
             .send();
         assertEquals(HttpStatus.OK_200, response.getStatus());
@@ -416,7 +417,7 @@ public class HttpClientTLSTest
         // Second request should have the same session ID.
         response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(HttpScheme.HTTPS.asString())
-            .header(HttpHeader.CONNECTION, "close")
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE))
             .timeout(5, TimeUnit.SECONDS)
             .send();
         assertEquals(HttpStatus.OK_200, response.getStatus());

jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java

@@ -907,7 +907,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         // If no exceptions the test passes.
         client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
-            .header(HttpHeader.CONNECTION, "close")
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE))
             .send();
     }
 
@@ -938,8 +938,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
 
         response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
-            .header(HttpHeader.USER_AGENT, null)
+            .headers(headers -> headers.put(HttpHeader.USER_AGENT, userAgent))
-            .header(HttpHeader.USER_AGENT, userAgent)
             .timeout(5, TimeUnit.SECONDS)
             .send();
 
@@ -985,7 +984,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         // User agent explicitly removed.
         response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
-            .header(HttpHeader.USER_AGENT, null)
+            .headers(headers -> headers.remove(HttpHeader.USER_AGENT))
             .timeout(5, TimeUnit.SECONDS)
             .send();
 
@@ -1213,7 +1212,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
 
         ContentResponse response = client.newRequest("http://127.0.0.1:" + connector.getLocalPort() + "/path")
             .scheme(scenario.getScheme())
-            .header(HttpHeader.HOST, host)
+            .headers(headers -> headers.put(HttpHeader.HOST, host))
             .send();
 
         assertEquals(200, response.getStatus());
@@ -1239,7 +1238,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .version(HttpVersion.HTTP_1_0)
-            .header(HttpHeader.CONNECTION, HttpHeaderValue.KEEP_ALIVE.asString())
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.KEEP_ALIVE.asString()))
             .timeout(5, TimeUnit.SECONDS)
             .send();
 
@@ -1266,7 +1265,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         Request request = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .version(HttpVersion.HTTP_1_0)
-            .header(HttpHeader.CONNECTION, HttpHeaderValue.KEEP_ALIVE.asString())
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.KEEP_ALIVE.asString()))
             .timeout(timeout, TimeUnit.MILLISECONDS);
         FuturePromise<Connection> promise = new FuturePromise<>();
         Destination destination = client.resolveDestination(request);
@@ -1295,7 +1294,7 @@ public class HttpClientTest extends AbstractHttpClientServerTest
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .version(HttpVersion.HTTP_1_0)
-            .header(HttpHeader.CONNECTION, HttpHeaderValue.KEEP_ALIVE.asString())
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.KEEP_ALIVE.asString()))
             .timeout(5, TimeUnit.SECONDS)
             .send();
 
@@ -1647,8 +1646,8 @@ public class HttpClientTest extends AbstractHttpClientServerTest
             .timeout(321, TimeUnit.SECONDS)
             .idleTimeout(2221, TimeUnit.SECONDS)
             .followRedirects(true)
-            .header(HttpHeader.CONTENT_TYPE, "application/json")
+            .headers(headers -> headers.put(HttpHeader.CONTENT_TYPE, "application/json"))
-            .header("X-Some-Custom-Header", "some-value"));
+            .headers(headers -> headers.put("X-Some-Custom-Header", "some-value")));
 
         assertCopyRequest(client.newRequest("https://example.com")
             .method(HttpMethod.POST)
@@ -1657,30 +1656,30 @@ public class HttpClientTest extends AbstractHttpClientServerTest
             .timeout(123231, TimeUnit.SECONDS)
             .idleTimeout(232342, TimeUnit.SECONDS)
             .followRedirects(false)
-            .header(HttpHeader.ACCEPT, "application/json")
+            .headers(headers -> headers.put(HttpHeader.ACCEPT, "application/json"))
-            .header("X-Some-Other-Custom-Header", "some-other-value"));
+            .headers(headers -> headers.put("X-Some-Other-Custom-Header", "some-other-value")));
 
         assertCopyRequest(client.newRequest("https://example.com")
-            .header(HttpHeader.ACCEPT, "application/json")
+            .headers(headers -> headers.add(HttpHeader.ACCEPT, "application/json"))
-            .header(HttpHeader.ACCEPT, "application/xml")
+            .headers(headers -> headers.add(HttpHeader.ACCEPT, "application/xml"))
-            .header("x-same-name", "value1")
+            .headers(headers -> headers.add("x-same-name", "value1"))
-            .header("x-same-name", "value2"));
+            .headers(headers -> headers.add("x-same-name", "value2")));
 
         assertCopyRequest(client.newRequest("https://example.com")
-            .header(HttpHeader.ACCEPT, "application/json")
+            .headers(headers -> headers.put(HttpHeader.ACCEPT, "application/json"))
-            .header(HttpHeader.CONTENT_TYPE, "application/json"));
+            .headers(headers -> headers.put(HttpHeader.CONTENT_TYPE, "application/json")));
 
         assertCopyRequest(client.newRequest("https://example.com")
-            .header("Accept", "application/json")
+            .headers(headers -> headers.put("Accept", "application/json"))
-            .header("Content-Type", "application/json"));
+            .headers(headers -> headers.put("Content-Type", "application/json")));
 
         assertCopyRequest(client.newRequest("https://example.com")
-            .header("X-Custom-Header-1", "value1")
+            .headers(headers -> headers.put("X-Custom-Header-1", "value1"))
-            .header("X-Custom-Header-2", "value2"));
+            .headers(headers -> headers.put("X-Custom-Header-2", "value2")));
 
         assertCopyRequest(client.newRequest("https://example.com")
-            .header("X-Custom-Header-1", "value")
+            .headers(headers -> headers.put("X-Custom-Header-1", "value"))
-            .header("X-Custom-Header-2", "value"));
+            .headers(headers -> headers.put("X-Custom-Header-2", "value")));
     }
 
     @ParameterizedTest

jetty-client/src/test/java/org/eclipse/jetty/client/HttpConnectionLifecycleTest.java

@@ -189,7 +189,7 @@ public class HttpConnectionLifecycleTest extends AbstractHttpClientServerTest
             public void onBegin(Request request)
             {
                 // Remove the host header, this will make the request invalid
-                request.header(HttpHeader.HOST, null);
+                request.headers(headers -> headers.remove(HttpHeader.HOST));
             }
 
             @Override
@@ -251,7 +251,7 @@ public class HttpConnectionLifecycleTest extends AbstractHttpClientServerTest
             public void onBegin(Request request)
             {
                 // Remove the host header, this will make the request invalid
-                request.header(HttpHeader.HOST, null);
+                request.headers(headers -> headers.remove(HttpHeader.HOST));
             }
 
             @Override

jetty-client/src/test/java/org/eclipse/jetty/client/HttpCookieTest.java

@@ -198,7 +198,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
         ContentResponse response = send(client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .path("/foo")
-            .header(headerName, "0")
+            .headers(headers -> headers.put(headerName, "0"))
             .timeout(5, TimeUnit.SECONDS));
         assertEquals(HttpStatus.OK_200, response.getStatus());
 
@@ -207,7 +207,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
             ContentResponse r = send(client.newRequest("localhost", connector.getLocalPort())
                 .scheme(scenario.getScheme())
                 .path(path)
-                .header(headerName, "1")
+                .headers(headers -> headers.put(headerName, "1"))
                 .timeout(5, TimeUnit.SECONDS));
             assertEquals(HttpStatus.OK_200, r.getStatus());
         });
@@ -259,7 +259,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
         ContentResponse response = send(client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .path("/foo/bar")
-            .header(headerName, "0")
+            .headers(headers -> headers.put(headerName, "0"))
             .timeout(5, TimeUnit.SECONDS));
         assertEquals(HttpStatus.OK_200, response.getStatus());
 
@@ -268,7 +268,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
             ContentResponse r = send(client.newRequest("localhost", connector.getLocalPort())
                 .scheme(scenario.getScheme())
                 .path(path)
-                .header(headerName, "1")
+                .headers(headers -> headers.put(headerName, "1"))
                 .timeout(5, TimeUnit.SECONDS));
             assertEquals(HttpStatus.OK_200, r.getStatus());
         });
@@ -320,7 +320,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
         ContentResponse response = send(client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .path("/foo")
-            .header(headerName, "0")
+            .headers(headers -> headers.put(headerName, "0"))
             .timeout(5, TimeUnit.SECONDS));
         assertEquals(HttpStatus.OK_200, response.getStatus());
 
@@ -329,7 +329,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
             ContentResponse r = send(client.newRequest("localhost", connector.getLocalPort())
                 .scheme(scenario.getScheme())
                 .path(path)
-                .header(headerName, "1")
+                .headers(headers -> headers.put(headerName, "1"))
                 .timeout(5, TimeUnit.SECONDS));
             assertEquals(HttpStatus.OK_200, r.getStatus());
         });
@@ -381,7 +381,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
         ContentResponse response = send(client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .path("/foo/bar")
-            .header(headerName, "0")
+            .headers(headers -> headers.put(headerName, "0"))
             .timeout(5, TimeUnit.SECONDS));
         assertEquals(HttpStatus.OK_200, response.getStatus());
 
@@ -390,7 +390,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
             ContentResponse r = send(client.newRequest("localhost", connector.getLocalPort())
                 .scheme(scenario.getScheme())
                 .path(path)
-                .header(headerName, "1")
+                .headers(headers -> headers.put(headerName, "1"))
                 .timeout(5, TimeUnit.SECONDS));
             assertEquals(HttpStatus.OK_200, r.getStatus());
         });
@@ -444,7 +444,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
         ContentResponse response = send(client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .path("/foo")
-            .header(headerName, "0")
+            .headers(headers -> headers.put(headerName, "0"))
             .timeout(5, TimeUnit.SECONDS));
         assertEquals(HttpStatus.OK_200, response.getStatus());
 
@@ -453,7 +453,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
             ContentResponse r = send(client.newRequest("localhost", connector.getLocalPort())
                 .scheme(scenario.getScheme())
                 .path(path)
-                .header(headerName, "1")
+                .headers(headers -> headers.put(headerName, "1"))
                 .timeout(5, TimeUnit.SECONDS));
             assertEquals(HttpStatus.OK_200, r.getStatus());
         });
@@ -514,7 +514,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
         ContentResponse response = send(client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .path("/foo")
-            .header(headerName, "0")
+            .headers(headers -> headers.put(headerName, "0"))
             .timeout(5, TimeUnit.SECONDS));
         assertEquals(HttpStatus.OK_200, response.getStatus());
 
@@ -523,7 +523,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
             ContentResponse r = send(client.newRequest("localhost", connector.getLocalPort())
                 .scheme(scenario.getScheme())
                 .path(path)
-                .header(headerName, "1")
+                .headers(headers -> headers.put(headerName, "1"))
                 .timeout(5, TimeUnit.SECONDS));
             assertEquals(HttpStatus.OK_200, r.getStatus());
         });
@@ -587,7 +587,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
         ContentResponse response = send(client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .path("/foo")
-            .header(headerName, "0")
+            .headers(headers -> headers.put(headerName, "0"))
             .timeout(5, TimeUnit.SECONDS));
         assertEquals(HttpStatus.OK_200, response.getStatus());
 
@@ -596,7 +596,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
             ContentResponse r = send(client.newRequest("localhost", connector.getLocalPort())
                 .scheme(scenario.getScheme())
                 .path(path)
-                .header(headerName, "1")
+                .headers(headers -> headers.put(headerName, "1"))
                 .timeout(5, TimeUnit.SECONDS));
             assertEquals(HttpStatus.OK_200, r.getStatus());
         });
@@ -648,7 +648,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
         ContentResponse response = send(client.newRequest("localhost", connector.getLocalPort())
             .scheme(scenario.getScheme())
             .path("/foo/bar")
-            .header(headerName, "0")
+            .headers(headers -> headers.put(headerName, "0"))
             .timeout(5, TimeUnit.SECONDS));
         assertEquals(HttpStatus.OK_200, response.getStatus());
 
@@ -657,7 +657,7 @@ public class HttpCookieTest extends AbstractHttpClientServerTest
             ContentResponse r = send(client.newRequest("localhost", connector.getLocalPort())
                 .scheme(scenario.getScheme())
                 .path(path)
-                .header(headerName, "1")
+                .headers(headers -> headers.put(headerName, "1"))
                 .timeout(5, TimeUnit.SECONDS));
             assertEquals(HttpStatus.OK_200, r.getStatus());
         });

jetty-client/src/test/java/org/eclipse/jetty/client/NetworkTrafficListenerTest.java

@@ -22,8 +22,8 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.Socket;
 import java.nio.ByteBuffer;
-import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
 import java.nio.charset.StandardCharsets;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -44,7 +44,6 @@ import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.io.ManagedSelector;
 import org.eclipse.jetty.io.NetworkTrafficListener;
 import org.eclipse.jetty.io.NetworkTrafficSocketChannelEndPoint;
-import org.eclipse.jetty.io.SelectorManager;
 import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.NetworkTrafficServerConnector;
@@ -189,7 +188,7 @@ public class NetworkTrafficListenerTest
         });
 
         ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
-            .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString())
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE))
             .send();
         assertEquals(HttpStatus.OK_200, response.getStatus());
 
@@ -507,16 +506,9 @@ public class NetworkTrafficListenerTest
             super(new HttpClientTransportOverHTTP(new ClientConnector()
             {
                 @Override
-                protected SelectorManager newSelectorManager()
+                protected EndPoint newEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey selectionKey)
                 {
-                    return new ClientSelectorManager(getExecutor(), getScheduler(), getSelectors())
+                    return new NetworkTrafficSocketChannelEndPoint(channel, selector, selectionKey, getScheduler(), getIdleTimeout().toMillis(), listener.get());
-                    {
-                        @Override
-                        protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey selectionKey)
-                        {
-                            return new NetworkTrafficSocketChannelEndPoint(channel, selector, selectionKey, getScheduler(), getIdleTimeout().toMillis(), listener.get());
-                        }
-                    };
                 }
             }));
             this.listener = listener;

jetty-client/src/test/java/org/eclipse/jetty/client/api/Usage.java

@@ -81,7 +81,7 @@ public class Usage
             .path("/uri")
             .version(HttpVersion.HTTP_1_1)
             .param("a", "b")
-            .header("X-Header", "Y-value")
+            .headers(headers -> headers.put("X-Header", "Y-value"))
             .agent("Jetty HTTP Client")
             .idleTimeout(5000, TimeUnit.MILLISECONDS)
             .timeout(20, TimeUnit.SECONDS);

jetty-client/src/test/java/org/eclipse/jetty/client/http/HttpDestinationOverHTTPTest.java

@@ -262,7 +262,7 @@ public class HttpDestinationOverHTTPTest extends AbstractHttpClientServerTest
         int port = connector.getLocalPort();
         Request request = client.newRequest(host, port)
                 .scheme(scenario.getScheme())
-                .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString());
+                .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE));
         Destination destinationBefore = client.resolveDestination(request);
         ContentResponse response = request.send();
 
@@ -275,7 +275,7 @@ public class HttpDestinationOverHTTPTest extends AbstractHttpClientServerTest
 
         request = client.newRequest(host, port)
             .scheme(scenario.getScheme())
-            .header(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE.asString());
+            .headers(headers -> headers.put(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE));
         response = request.send();
 
         assertEquals(200, response.getStatus());

jetty-client/src/test/java/org/eclipse/jetty/client/ssl/SslBytesServerTest.java

@@ -50,10 +50,10 @@ import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jetty.client.ssl.SslBytesTest.TLSRecord.Type;
 import org.eclipse.jetty.http.HttpCompliance;
 import org.eclipse.jetty.http.HttpParser;
-import org.eclipse.jetty.io.ChannelEndPoint;
 import org.eclipse.jetty.io.Connection;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.io.ManagedSelector;
+import org.eclipse.jetty.io.SocketChannelEndPoint;
 import org.eclipse.jetty.io.ssl.SslConnection;
 import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.server.HttpConnection;
@@ -189,11 +189,11 @@ public class SslBytesServerTest extends SslBytesTest
         ServerConnector connector = new ServerConnector(server, null, null, null, 1, 1, sslFactory, httpFactory)
         {
             @Override
-            protected ChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key) throws IOException
+            protected SocketChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key) throws IOException
             {
-                ChannelEndPoint endp = super.newEndPoint(channel, selectSet, key);
+                SocketChannelEndPoint endPoint = super.newEndPoint(channel, selectSet, key);
-                serverEndPoint.set(endp);
+                serverEndPoint.set(endPoint);
-                return endp;
+                return endPoint;
             }
         };
         connector.setIdleTimeout(idleTimeout);

jetty-client/src/test/java/org/eclipse/jetty/client/util/TypedContentProviderTest.java

@@ -20,7 +20,6 @@ package org.eclipse.jetty.client.util;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
@@ -57,7 +56,7 @@ public class TypedContentProviderTest extends AbstractHttpClientServerTest
         start(scenario, new AbstractHandler()
         {
             @Override
-            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
             {
                 baseRequest.setHandled(true);
                 assertEquals("POST", request.getMethod());
@@ -90,13 +89,13 @@ public class TypedContentProviderTest extends AbstractHttpClientServerTest
         Fields fields = new Fields();
         fields.put(name1, value1);
         fields.add(name2, value2);
-        final String content = FormContentProvider.convert(fields);
+        final String content = FormRequestContent.convert(fields);
         final String contentType = "text/plain;charset=UTF-8";
 
         start(scenario, new AbstractHandler()
         {
             @Override
-            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException
             {
                 baseRequest.setHandled(true);
                 assertEquals("POST", request.getMethod());
@@ -109,7 +108,7 @@ public class TypedContentProviderTest extends AbstractHttpClientServerTest
             .scheme(scenario.getScheme())
             .method(HttpMethod.POST)
             .body(new FormRequestContent(fields))
-            .header(HttpHeader.CONTENT_TYPE, contentType)
+            .headers(headers -> headers.put(HttpHeader.CONTENT_TYPE, contentType))
             .send();
 
         assertEquals(200, response.getStatus());
@@ -124,7 +123,7 @@ public class TypedContentProviderTest extends AbstractHttpClientServerTest
         start(scenario, new AbstractHandler()
         {
             @Override
-            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException
             {
                 baseRequest.setHandled(true);
                 assertEquals("GET", request.getMethod());

jetty-client/src/test/resources/jetty-logging.properties

@@ -1,6 +1,6 @@
 # Jetty Logging using jetty-slf4j-impl
 #org.eclipse.jetty.LEVEL=DEBUG
 #org.eclipse.jetty.client.LEVEL=DEBUG
-#org.eclipse.jetty.io.ChannelEndPoint.LEVEL=DEBUG
+#org.eclipse.jetty.io.SocketChannelEndPoint.LEVEL=DEBUG
 #org.eclipse.jetty.io.ssl.LEVEL=DEBUG
 #org.eclipse.jetty.http.LEVEL=DEBUG

jetty-documentation/pom.xml

@@ -33,6 +33,26 @@
       <groupId>org.eclipse.jetty.toolchain</groupId>
       <artifactId>jetty-servlet-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-alpn-server</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-client</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-jmx</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-rewrite</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-server</artifactId>
@@ -48,36 +68,21 @@
       <artifactId>jetty-servlets</artifactId>
       <version>${project.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-rewrite</artifactId>
-      <version>${project.version}</version>
-    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-webapp</artifactId>
       <version>${project.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-alpn-server</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.eclipse.jetty.http2</groupId>
-      <artifactId>http2-server</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-client</artifactId>
-      <version>${project.version}</version>
-    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty.fcgi</groupId>
       <artifactId>fcgi-client</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty.http2</groupId>
+      <artifactId>http2-server</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty.http2</groupId>
       <artifactId>http2-http-client-transport</artifactId>

jetty-documentation/src/main/asciidoc/embedded-guide/arch-bean.adoc

@@ -31,8 +31,9 @@ The end result is that an application based on the Jetty libraries is a _tree_ o
 In server application the root of the component tree is a `Server` instance, while in client applications the root of the component tree is an `HttpClient` instance.
 
 Having all the Jetty components in a tree is beneficial in a number of use cases.
-It makes possible to register the components in the tree as JMX MBeans (TODO: xref the JMX section) so that a JMX console can look at the internal state of the components.
+It makes possible to register the components in the tree as xref:eg-arch-jmx[JMX MBeans] so that a JMX console can look at the internal state of the components.
-It also makes possible to dump the component tree (and therefore each component's internal state) to a log file or to the console for troubleshooting purposes (TODO: xref troubleshooting section).
+It also makes possible to xref:eg-troubleshooting-component-dump[dump the component tree] (and therefore each component's internal state) to a log file or to the console for xref:eg-troubleshooting[troubleshooting purposes].
+// TODO: add a section on Dumpable?
 
 [[eg-arch-bean-lifecycle]]
 ==== Jetty Component Lifecycle
@@ -40,7 +41,7 @@ It also makes possible to dump the component tree (and therefore each component'
 Jetty components typically have a life cycle: they can be started and stopped.
 The Jetty components that have a life cycle implement the `org.eclipse.jetty.util.component.LifeCycle` interface.
 
-Jetty components that contain other components extend the `org.eclipse.jetty.util.component.ContainerLifeCycle` class.
+Jetty components that contain other components implement the `org.eclipse.jetty.util.component.Container` interface and typically extend the `org.eclipse.jetty.util.component.ContainerLifeCycle` class.
 `ContainerLifeCycle` can contain these type of components, also called __bean__s:
 
 * _managed_ beans, `LifeCycle` instances whose life cycle is tied to the life cycle of their container
@@ -94,8 +95,72 @@ include::{doc_code}/embedded/ComponentDocs.java[tags=restart]
 `Service` can be stopped independently of `Root`, and re-started.
 Starting and stopping a non-root component does not alter the structure of the component tree, just the state of the subtree starting from the component that has been stopped and re-started.
 
+`Container` provides an API to find beans in the component tree:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/ComponentDocs.java[tags=getBeans]
+----
+
+You can add your own beans to the component tree at application startup time, and later find them from your application code to access their services.
+
+[TIP]
+====
+The component tree should be used for long-lived or medium-lived components such as thread pools, web application contexts, etc.
+
+It is not recommended adding to, and removing from, the component tree short-lived objects such as HTTP requests or TCP connections, for performance reasons.
+
+If you need component tree features such as automatic xref:eg-arch-jmx[export to JMX] or xref:eg-troubleshooting-component-dump[dump capabilities] for short-lived objects, consider having a long-lived container in the component tree instead.
+You can make the long-lived container efficient at adding/removing the short-lived components using a data structure that is not part of the component tree, and make the long-lived container handle the JMX and dump features for the short-lived components.
+====
+
 [[eg-arch-bean-listener]]
 ==== Jetty Component Listeners
 
-// TODO: LifeCycle.Listener
+A component that extends `AbstractLifeCycle` inherits the possibility to add/remove event _listeners_ for various events emitted by components.
-// TODO: Container.Listener + InheritedListener
+
+A component that implements `java.util.EventListener` that is added to a `ContainerLifeCycle` is also registered as an event listener.
+
+The following sections describe in details the various listeners available in the Jetty component architecture.
+
+[[eg-arch-bean-listener-lifecycle]]
+===== LifeCycle.Listener
+
+A `LifeCycle.Listener` emits events for life cycle events such as starting, stopping and failures:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/ComponentDocs.java[tags=lifecycleListener]
+----
+
+For example, a life cycle listener attached to a `Server` instance could be used to create (for the _started_ event) and delete (for the _stopped_ event) a file containing the process ID of the JVM that runs the `Server`.
+
+[[eg-arch-bean-listener-container]]
+===== Container.Listener
+
+A component that implements `Container` is a container for other components and `ContainerLifeCycle` is the typical implementation.
+
+A `Container` emits events when a component (also called _bean_) is added to or removed from the container:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/ComponentDocs.java[tags=containerListener]
+----
+
+A `Container.Listener` added as a bean will also be registered as a listener:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/ComponentDocs.java[tags=containerSiblings]
+----
+
+[[eg-arch-bean-listener-inherited]]
+===== Container.InheritedListener
+
+A `Container.InheritedListener` is a listener that will be added to all descendants that are also ``Container``s.
+
+Listeners of this type may be added to the component tree root only, but will be notified of every descendant component that is added to or removed from the component tree (not only first level children).
+
+The primary use of `Container.InheritedListener` within the Jetty Libraries is `MBeanContainer` from the xref:eg-arch-jmx[Jetty JMX support].
+
+`MBeanContainer` listens for every component added to the tree, converts it to an MBean and registers it to the MBeanServer; for every component removed from the tree, it unregisters the corresponding MBean from the MBeanServer.

jetty-documentation/src/main/asciidoc/embedded-guide/arch-jmx.adoc

@@ -0,0 +1,306 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+[[eg-arch-jmx]]
+== Jetty JMX Support
+
+The Java Management Extensions (JMX) APIs are standard API for managing and monitoring resources such as applications, devices, services, and the Java Virtual Machine itself.
+
+The JMX API includes remote access, so a remote management console such as link:https://openjdk.java.net/projects/jmc/[Java Mission Control] can interact with a running application for these purposes.
+
+Jetty architecture is based on xref:eg-arch-bean[components] organized in a tree. Every time a component is added to or removed from the component tree, an event is emitted, and xref:eg-arch-bean-listener-container[Container.Listener] implementations can listen to those events and perform additional actions.
+
+`org.eclipse.jetty.jmx.MBeanContainer` listens to those events and registers/unregisters the Jetty components as MBeans into the platform MBeanServer.
+
+The Jetty components are annotated with xref:eg-arch-jmx-annotation[Jetty JMX annotations] so that they can provide specific JMX metadata such as attributes and operations that should be exposed via JMX.
+
+Therefore, when a component is added to the component tree, `MBeanContainer` is notified, it creates the MBean from the component POJO and registers it to the `MBeanServer`.
+Similarly, when a component is removed from the tree, `MBeanContainer` is notified, and unregisters the MBean from the `MBeanServer`.
+
+The Maven coordinates for the Jetty JMX support are:
+
+[source,xml,subs=normal]
+----
+<dependency>
+  <groupId>org.eclipse.jetty</groupId>
+  <artifactId>jetty-jmx</artifactId>
+  <version>{version}</version>
+</dependency>
+----
+
+=== Enabling JMX Support
+
+Enabling JMX support is always recommended because it provides valuable information about the system, both for monitoring purposes and for troubleshooting purposes in case of problems.
+
+To enable JMX support on the server:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/JMXDocs.java[tags=server]
+----
+
+Similarly on the client:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/JMXDocs.java[tags=client]
+----
+
+[NOTE]
+====
+The MBeans exported to the platform MBeanServer can only be accessed locally (from the same machine), not from remote machines.
+
+This means that this configuration is enough for development, where you have easy access (with graphical user interface) to the machine where Jetty runs, but it is typically not enough when the machine where Jetty runs is remote, or only accessible via SSH or otherwise without graphical user interface support.
+In these cases, you have to enable xref:eg-arch-jmx-remote[JMX Remote Access].
+====
+
+// TODO: add a section about how to expose logging once #4830 is fixed.
+
+[[eg-arch-jmx-remote]]
+=== Enabling JMX Remote Access
+
+There are two ways of enabling remote connectivity so that JMC can connect to the remote JVM to visualize MBeans.
+
+* Use the `com.sun.management.jmxremote` system property on the command line.
+Unfortunately, this solution does not work well with firewalls and is not flexible.
+* Use Jetty's `ConnectorServer` class.
+
+`org.eclipse.jetty.jmx.ConnectorServer` will use by default RMI to allow connection from remote clients, and it is a wrapper around the standard JDK class `JMXConnectorServer`, which is the class that provides remote access to JMX clients.
+
+Connecting to the remote JVM is a two step process:
+
+* First, the client will connect to the RMI _registry_ to download the RMI stub for the `JMXConnectorServer`; this RMI stub contains the IP address and port to connect to the RMI server, i.e. the remote `JMXConnectorServer`.
+* Second, the client uses the RMI stub to connect to the RMI _server_ (i.e. the remote `JMXConnectorServer`) typically on an address and port that may be different from the RMI registry address and port.
+
+The host and port configuration for the RMI registry and the RMI server is specified by a `JMXServiceURL`.
+The string format of an RMI `JMXServiceURL` is:
+
+[source,screen]
+----
+service:jmx:rmi://<rmi_server_host>:<rmi_server_port>/jndi/rmi://<rmi_registry_host>:<rmi_registry_port>/jmxrmi
+----
+
+Default values are:
+
+[source,screen]
+----
+rmi_server_host = localhost
+rmi_server_port = 1099
+rmi_registry_host = localhost
+rmi_registry_port = 1099
+----
+
+With the default configuration, only clients that are local to the server machine can connect to the RMI registry and RMI server - this is done for security reasons.
+With this configuration it would still be possible to access the MBeans from remote using a xref:eg-arch-jmx-remote-ssh-tunnel[SSH tunnel].
+
+By specifying an appropriate `JMXServiceURL`, you can fine tune the network interfaces the RMI registry and the RMI server bind to, and the ports that the RMI registry and the RMI server listen to.
+The RMI server and RMI registry hosts and ports can be the same (as in the default configuration) because RMI is able to multiplex traffic arriving to a port to multiple RMI objects.
+
+If you need to allow JMX remote access through a firewall, you must open both the RMI registry and the RMI server ports.
+
+`JMXServiceURL` common examples:
+
+[source,screen]
+----
+service:jmx:rmi:///jndi/rmi:///jmxrmi
+  rmi_server_host = local host address
+  rmi_server_port = randomly chosen
+  rmi_registry_host = local host address
+  rmi_registry_port = 1099
+
+service:jmx:rmi://0.0.0.0:1099/jndi/rmi://0.0.0.0:1099/jmxrmi
+  rmi_server_host = any address
+  rmi_server_port = 1099
+  rmi_registry_host = any address
+  rmi_registry_port = 1099
+
+service:jmx:rmi://localhost:1100/jndi/rmi://localhost:1099/jmxrmi
+  rmi_server_host = loopback address
+  rmi_server_port = 1100
+  rmi_registry_host = loopback address
+  rmi_registry_port = 1099
+----
+
+[NOTE]
+====
+When `ConnectorServer` is started, its RMI stub is exported to the RMI registry.
+The RMI stub contains the IP address and port to connect to the RMI object, but the IP address is typically the machine host name, not the host specified in the `JMXServiceURL`.
+
+To control the IP address stored in the RMI stub you need to set the system property `java.rmi.server.hostname` with the desired value.
+This is especially important when binding the RMI server host to the loopback address for security reasons. See also xref:eg-arch-jmx-remote-ssh-tunnel[JMX Remote Access via SSH Tunnel.]
+====
+
+To allow JMX remote access, create and configure a `ConnectorServer`:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/JMXDocs.java[tags=remote]
+----
+
+[[eg-arch-jmx-remote-authorization]]
+==== JMX Remote Access Authorization
+
+The standard `JMXConnectorServer` provides several options to authorize access, for example via JAAS or via configuration files.
+For a complete guide to controlling authentication and authorization in JMX, see https://docs.oracle.com/en/java/javase/11/management/[the official JMX documentation].
+
+In the sections below we detail one way to setup JMX authentication and authorization, using configuration files for users, passwords and roles:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/JMXDocs.java[tags=remoteAuthorization]
+----
+
+The `users.access` file format is defined in the `$JAVA_HOME/conf/management/jmxremote.access` file.
+A simplified version is the following:
+
+.users.access
+[source,screen]
+----
+user1 readonly
+user2 readwrite
+----
+
+The `users.password` file format is defined in the `$JAVA_HOME/conf/management/jmxremote.password.template` file.
+A simplified version is the following:
+
+.users.password
+[source,screen]
+----
+user1 password1
+user2 password2
+----
+
+CAUTION: The `users.access` and `users.password` files are not standard `*.properties` files -- the user must be separated from the role or password by a space character.
+
+===== Securing JMX Remote Access with TLS
+
+The JMX communication via RMI happens by default in clear-text.
+
+It is possible to configure the `ConnectorServer` with a `SslContextFactory` so that the JMX communication via RMI is encrypted:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/JMXDocs.java[tags=tlsRemote]
+----
+
+It is possible to use the same `SslContextFactory.Server` used to configure the Jetty `ServerConnector` that supports TLS also for the JMX communication via RMI.
+
+The keystore must contain a valid certificate signed by a Certification Authority.
+
+The RMI mechanic is the usual one: the RMI client (typically a monitoring console) will connect first to the RMI registry (using TLS), download the RMI server stub that contains the address and port of the RMI server to connect to, then connect to the RMI server (using TLS).
+
+This also mean that if the RMI registry and the RMI server are on different hosts, the RMI client must have available the cryptographic material to validate both hosts.
+
+Having certificates signed by a Certification Authority simplifies by a lot the configuration needed to get the JMX communication over TLS working properly.
+
+If that is not the case (for example the certificate is self-signed), then you need to specify the required system properties that allow RMI (especially when acting as an RMI client) to retrieve the cryptographic material necessary to establish the TLS connection.
+
+For example, trying to connect using the JDK standard `JMXConnector` with both the RMI server and the RMI registry via TLS to `domain.com` with a self-signed certificate:
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/JMXDocs.java[tags=tlsJMXConnector]
+----
+
+Similarly, to launch JMC:
+
+[source,screen]
+----
+$ jmc -vmargs -Djavax.net.ssl.trustStore=/path/to/trustStore -Djavax.net.ssl.trustStorePassword=secret
+----
+
+IMPORTANT: These system properties are required when launching the `ConnectorServer` too, on the server, because it acts as an RMI client with respect to the RMI registry.
+
+[[eg-arch-jmx-remote-ssh-tunnel]]
+===== JMX Remote Access with Port Forwarding via SSH Tunnel
+
+You can access JMX MBeans on a remote machine when the RMI ports are not open, for example because of firewall policies, but you have SSH access to the machine using local port forwarding via an SSH tunnel.
+
+In this case you want to configure the `ConnectorServer` with a `JMXServiceURL` that binds the RMI server and the RMI registry to the loopback interface only: `service:jmx:rmi://localhost:1099/jndi/rmi://localhost:1099/jmxrmi`.
+
+Then you setup the local port forwarding with the SSH tunnel:
+
+[source,screen]
+----
+$ ssh -L 1099:localhost:1099 <user>@<machine_host>
+----
+
+Now you can use JConsole or JMC to connect to `localhost:1099` on your local computer.
+The traffic will be forwarded to `machine_host` and when there, SSH will forward the traffic to `localhost:1099`, which is exactly where the `ConnectorServer` listens.
+
+When you configure `ConnectorServer` in this way, you must set the system property `-Djava.rmi.server.hostname=localhost`, on the server.
+This is required because when the RMI server is exported, its address and port are stored in the RMI stub. You want the address in the RMI stub to be `localhost` so that when the RMI stub is downloaded to the remote client, the RMI communication will go through the SSH tunnel.
+
+[[eg-arch-jmx-annotation]]
+=== Jetty JMX Annotations
+
+The Jetty JMX support, and in particular `MBeanContainer`, is notified every time a bean is added to the component tree.
+
+The bean is scanned for Jetty JMX annotations to obtain JMX metadata: the JMX attributes and JMX operations.
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/JMXDocs.java[tags=jmxAnnotation]
+----
+
+The JMX metadata and the bean are wrapped by an instance of `org.eclipse.jetty.jmx.ObjectMBean` that exposes the JMX metadata and, upon request from JMX consoles, invokes methods on the bean to get/set attribute values and perform operations.
+
+You can provide a custom subclass of `ObjectMBean` to further customize how the bean is exposed to JMX.
+
+The custom `ObjectMBean` subclass must respect the following naming convention: `<package>.jmx.<class>MBean`.
+For example, class `com.acme.Foo` may have a custom `ObjectMBean` subclass named `com.acme.**jmx**.Foo**MBean**`.
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/JMXDocs.java[tags=jmxCustomMBean]
+----
+
+The custom `ObjectMBean` subclass is also scanned for Jetty JMX annotations and overrides the JMX metadata obtained by scanning the bean class.
+This allows to annotate only the custom `ObjectMBean` subclass and keep the bean class free of the Jetty JMX annotations.
+
+[source,java,indent=0]
+----
+include::{doc_code}/embedded/JMXDocs.java[tags=jmxCustomMBeanOverride]
+----
+
+The scan for Jetty JMX annotations is performed on the bean class and all the interfaces implemented by the bean class, then on the super-class and all the interfaces implemented by the super-class and so on until `java.lang.Object` is reached.
+For each type -- class or interface, the corresponding `+*.jmx.*MBean+` is looked up and scanned as well with the same algorithm.
+For each type, the scan looks for the class-level annotation `@ManagedObject`.
+If it is found, the scan looks for method-level `@ManagedAttribute` and `@ManagedOperation` annotations; otherwise it skips the current type and moves to the next type to scan.
+
+==== @ManagedObject
+
+The `@ManagedObject` annotation is used on a class at the top level to indicate that it should be exposed as an MBean.
+It has only one attribute to it which is used as the description of the MBean.
+
+==== @ManagedAttribute
+
+The `@ManagedAttribute` annotation is used to indicate that a given method is exposed as a JMX attribute.
+This annotation is placed always on the getter method of a given attribute.
+Unless the `readonly` attribute is set to `true` in the annotation, a corresponding setter is looked up following normal naming conventions.
+For example if this annotation is on a method called `String getFoo()` then a method called `void setFoo(String)` would be looked up, and if found wired as the setter for the JMX attribute.
+
+==== @ManagedOperation
+
+The `@ManagedOperation` annotation is used to indicate that a given method is exposed as a JMX operation.
+A JMX operation has an _impact_ that can be `INFO` if the operation returns a value without modifying the object, `ACTION` if the operation does not return a value but modifies the object, and "ACTION_INFO" if the operation both returns a value and modifies the object.
+If the _impact_ is not specified, it has the default value of `UNKNOWN`.
+
+==== @Name
+
+The `@Name` annotation is used to assign a name and description to parameters in method signatures so that when rendered by JMX consoles it is clearer what the parameter meaning is.

jetty-documentation/src/main/asciidoc/embedded-guide/arch.adoc

@@ -21,5 +21,6 @@
 == Jetty Architecture
 
 include::arch-bean.adoc[]
+include::arch-jmx.adoc[]
 include::arch-listener.adoc[]
 include::arch-io.adoc[]

jetty-documentation/src/main/asciidoc/embedded-guide/server/http/server-http-handler-use.adoc

@@ -245,8 +245,7 @@ Server
 * Number of responses grouped by HTTP code (i.e. how many `2xx` responses, how many `3xx` responses, etc.)
 * Total response content bytes
 
-Server applications can read these values and use them internally, or expose them via some service, or export them via JMX.
+Server applications can read these values and use them internally, or expose them via some service, or xref:eg-arch-jmx[export them to JMX].
-// TODO: xref to the JMX section.
 
 `StatisticsHandler` can be configured at the server level or at the context level.
 

jetty-documentation/src/main/asciidoc/embedded-guide/troubleshooting.adoc

@@ -20,6 +20,13 @@
 [[eg-troubleshooting]]
 == Troubleshooting Jetty
 
+TODO: introduction
+// TODO: explain the process to troubleshoot Jetty:
+// TODO: #1 enable JMX
+// TODO: #2 enable GC logs
+// TODO: #3 take jvm/component dumps
+// TODO: #4 enable debug logging if you can
+
 [[eg-troubleshooting-logging]]
 === Logging
 
@@ -66,6 +73,24 @@ If you want to enable DEBUG logging but only for the HTTP/2 classes:
 java -Dorg.eclipse.jetty.http2.LEVEL=DEBUG --class-path ...
 ----
 
+[[eg-troubleshooting-thread-dump]]
+=== JVM Thread Dump
+TODO
+
+[[eg-troubleshooting-component-dump]]
+=== Jetty Component Tree Dump
+
+Jetty components are organized in a xref:eg-arch-bean[component tree].
+
+At the root of the component tree there is typically a `ContainerLifeCycle` instance -- typically a `Server` instance on the server and an `HttpClient` instance on the client.
+
+`ContainerLifeCycle` has built-in _dump_ APIs that can be invoked either directly or xref:eg-arch-jmx[via JMX].
+
+// TODO: images from JMC?
+// TODO: Command line JMX will be in JMX section.
+
+TIP: You can get more details from a Jetty's `QueuedThreadPool` dump by enabling detailed dumps via `queuedThreadPool.setDetailedDump(true)`.
+
 [[eg-troubleshooting-debugging]]
 === Debugging
 

jetty-documentation/src/main/java/embedded/ComponentDocs.java

@@ -18,11 +18,19 @@
 
 package embedded;
 
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 
+import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.util.component.AbstractLifeCycle;
+import org.eclipse.jetty.util.component.Container;
 import org.eclipse.jetty.util.component.ContainerLifeCycle;
+import org.eclipse.jetty.util.component.LifeCycle;
+
+import static java.lang.System.Logger.Level.INFO;
 
 @SuppressWarnings("unused")
 public class ComponentDocs
@@ -140,4 +148,142 @@ public class ComponentDocs
         service.start();
         // end::restart[]
     }
+
+    public void getBeans() throws Exception
+    {
+        // tag::getBeans[]
+        class Root extends ContainerLifeCycle
+        {
+        }
+
+        class Service extends ContainerLifeCycle
+        {
+            private ScheduledExecutorService scheduler;
+
+            @Override
+            protected void doStart() throws Exception
+            {
+                scheduler = Executors.newSingleThreadScheduledExecutor();
+                addBean(scheduler);
+                super.doStart();
+            }
+
+            @Override
+            protected void doStop() throws Exception
+            {
+                super.doStop();
+                removeBean(scheduler);
+                scheduler.shutdown();
+            }
+        }
+
+        Root root = new Root();
+        Service service = new Service();
+        root.addBean(service);
+
+        // Start the Root component.
+        root.start();
+
+        // Find all the direct children of root.
+        Collection<Object> children = root.getBeans();
+        // children contains only service
+
+        // Find all descendants of root that are instance of a particular class.
+        Collection<ScheduledExecutorService> schedulers = root.getContainedBeans(ScheduledExecutorService.class);
+        // schedulers contains the service scheduler.
+        // end::getBeans[]
+    }
+
+    public void lifecycleListener()
+    {
+        // tag::lifecycleListener[]
+        Server server = new Server();
+
+        // Add an event listener of type LifeCycle.Listener.
+        server.addEventListener(new LifeCycle.Listener()
+        {
+            @Override
+            public void lifeCycleStarted(LifeCycle lifeCycle)
+            {
+                System.getLogger("server").log(INFO, "Server {0} has been started", lifeCycle);
+            }
+
+            @Override
+            public void lifeCycleFailure(LifeCycle lifeCycle, Throwable failure)
+            {
+                System.getLogger("server").log(INFO, "Server {0} failed to start", lifeCycle, failure);
+            }
+
+            @Override
+            public void lifeCycleStopped(LifeCycle lifeCycle)
+            {
+                System.getLogger("server").log(INFO, "Server {0} has been stopped", lifeCycle);
+            }
+        });
+        // end::lifecycleListener[]
+    }
+
+    public void containerListener()
+    {
+        // tag::containerListener[]
+        Server server = new Server();
+
+        // Add an event listener of type LifeCycle.Listener.
+        server.addEventListener(new Container.Listener()
+        {
+            @Override
+            public void beanAdded(Container parent, Object child)
+            {
+                System.getLogger("server").log(INFO, "Added bean {1} to {0}", parent, child);
+            }
+
+            @Override
+            public void beanRemoved(Container parent, Object child)
+            {
+                System.getLogger("server").log(INFO, "Removed bean {1} from {0}", parent, child);
+            }
+        });
+        // end::containerListener[]
+    }
+
+    public void containerSiblings()
+    {
+        // tag::containerSiblings[]
+        class Parent extends ContainerLifeCycle
+        {
+        }
+
+        class Child
+        {
+        }
+
+        // The older child takes care of its siblings.
+        class OlderChild extends Child implements Container.Listener
+        {
+            private Set<Object> siblings = new HashSet<>();
+
+            @Override
+            public void beanAdded(Container parent, Object child)
+            {
+                siblings.add(child);
+            }
+
+            @Override
+            public void beanRemoved(Container parent, Object child)
+            {
+                siblings.remove(child);
+            }
+        }
+
+        Parent parent = new Parent();
+
+        Child older = new OlderChild();
+        // The older child is a child bean _and_ a listener.
+        parent.addBean(older);
+
+        Child younger = new Child();
+        // Adding a younger child will notify the older child.
+        parent.addBean(younger);
+        // end::containerSiblings[]
+    }
 }

jetty-documentation/src/main/java/embedded/JMXDocs.java

@@ -0,0 +1,276 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package embedded;
+
+import java.lang.management.ManagementFactory;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.management.ObjectName;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+import javax.management.remote.JMXServiceURL;
+import javax.rmi.ssl.SslRMIClientSocketFactory;
+
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.jmx.ConnectorServer;
+import org.eclipse.jetty.jmx.MBeanContainer;
+import org.eclipse.jetty.jmx.ObjectMBean;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.util.annotation.ManagedAttribute;
+import org.eclipse.jetty.util.annotation.ManagedObject;
+import org.eclipse.jetty.util.annotation.ManagedOperation;
+import org.eclipse.jetty.util.annotation.Name;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+
+@SuppressWarnings("unused")
+public class JMXDocs
+{
+    public void server()
+    {
+        // tag::server[]
+        Server server = new Server();
+
+        // Create an MBeanContainer with the platform MBeanServer.
+        MBeanContainer mbeanContainer = new MBeanContainer(ManagementFactory.getPlatformMBeanServer());
+
+        // Add MBeanContainer to the root component.
+        server.addBean(mbeanContainer);
+        // end::server[]
+    }
+
+    public void client()
+    {
+        // tag::client[]
+        HttpClient httpClient = new HttpClient();
+
+        // Create an MBeanContainer with the platform MBeanServer.
+        MBeanContainer mbeanContainer = new MBeanContainer(ManagementFactory.getPlatformMBeanServer());
+
+        // Add MBeanContainer to the root component.
+        httpClient.addBean(mbeanContainer);
+        // end::client[]
+    }
+
+    public void remote() throws Exception
+    {
+        // tag::remote[]
+        Server server = new Server();
+
+        // Setup Jetty JMX.
+        MBeanContainer mbeanContainer = new MBeanContainer(ManagementFactory.getPlatformMBeanServer());
+        server.addBean(mbeanContainer);
+
+        // Setup ConnectorServer.
+
+        // Bind the RMI server to the wildcard address and port 1999.
+        // Bind the RMI registry to the wildcard address and port 1099.
+        JMXServiceURL jmxURL = new JMXServiceURL("rmi", null, 1999, "/jndi/rmi:///jmxrmi");
+        ConnectorServer jmxServer = new ConnectorServer(jmxURL, "org.eclipse.jetty.jmx:name=rmiconnectorserver");
+
+        // Add ConnectorServer as a bean, so it is started
+        // with the Server and also exported as MBean.
+        server.addBean(jmxServer);
+
+        server.start();
+        // end::remote[]
+    }
+
+    public static void main(String[] args) throws Exception
+    {
+        new JMXDocs().remote();
+    }
+
+    public void remoteAuthorization() throws Exception
+    {
+        // tag::remoteAuthorization[]
+        Server server = new Server();
+
+        // Setup Jetty JMX.
+        MBeanContainer mbeanContainer = new MBeanContainer(ManagementFactory.getPlatformMBeanServer());
+        server.addBean(mbeanContainer);
+
+        // Setup ConnectorServer.
+        JMXServiceURL jmxURL = new JMXServiceURL("rmi", null, 1099, "/jndi/rmi:///jmxrmi");
+        Map<String, Object> env = new HashMap<>();
+        env.put("com.sun.management.jmxremote.access.file", "/path/to/users.access");
+        env.put("com.sun.management.jmxremote.password.file", "/path/to/users.password");
+        ConnectorServer jmxServer = new ConnectorServer(jmxURL, env, "org.eclipse.jetty.jmx:name=rmiconnectorserver");
+        server.addBean(jmxServer);
+
+        server.start();
+        // end::remoteAuthorization[]
+    }
+
+    public void tlsRemote() throws Exception
+    {
+        // tag::tlsRemote[]
+        Server server = new Server();
+
+        // Setup Jetty JMX.
+        MBeanContainer mbeanContainer = new MBeanContainer(ManagementFactory.getPlatformMBeanServer());
+        server.addBean(mbeanContainer);
+
+        // Setup SslContextFactory.
+        SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
+        sslContextFactory.setKeyStorePath("/path/to/keystore");
+        sslContextFactory.setKeyStorePassword("secret");
+
+        // Setup ConnectorServer with SslContextFactory.
+        JMXServiceURL jmxURL = new JMXServiceURL("rmi", null, 1099, "/jndi/rmi:///jmxrmi");
+        ConnectorServer jmxServer = new ConnectorServer(jmxURL, null, "org.eclipse.jetty.jmx:name=rmiconnectorserver", sslContextFactory);
+        server.addBean(jmxServer);
+
+        server.start();
+        // end::tlsRemote[]
+    }
+
+    public void tlsJMXConnector() throws Exception
+    {
+        // tag::tlsJMXConnector[]
+        // System properties necessary for an RMI client to trust a self-signed certificate.
+        System.setProperty("javax.net.ssl.trustStore", "/path/to/trustStore");
+        System.setProperty("javax.net.ssl.trustStorePassword", "secret");
+
+        JMXServiceURL jmxURL = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://domain.com:1100/jmxrmi");
+
+        Map<String, Object> clientEnv = new HashMap<>();
+        // Required to connect to the RMI registry via TLS.
+        clientEnv.put(ConnectorServer.RMI_REGISTRY_CLIENT_SOCKET_FACTORY_ATTRIBUTE, new SslRMIClientSocketFactory());
+
+        try (JMXConnector client = JMXConnectorFactory.connect(jmxURL, clientEnv))
+        {
+            Set<ObjectName> names = client.getMBeanServerConnection().queryNames(null, null);
+        }
+        // end::tlsJMXConnector[]
+    }
+
+    public void jmxAnnotation() throws Exception
+    {
+        // tag::jmxAnnotation[]
+        // Annotate the class with @ManagedObject and provide a description.
+        @ManagedObject("Services that provide useful features")
+        class Services
+        {
+            private final Map<String, Object> services = new ConcurrentHashMap<>();
+            private boolean enabled = true;
+
+            // A read-only attribute with description.
+            @ManagedAttribute(value = "The number of services", readonly = true)
+            public int getServiceCount()
+            {
+                return services.size();
+            }
+
+            // A read-write attribute with description.
+            // Only the getter is annotated.
+            @ManagedAttribute(value = "Whether the services are enabled")
+            public boolean isEnabled()
+            {
+                return enabled;
+            }
+
+            // There is no need to annotate the setter.
+            public void setEnabled(boolean enabled)
+            {
+                this.enabled = enabled;
+            }
+
+            // An operation with description and impact.
+            // The @Name annotation is used to annotate parameters
+            // for example to display meaningful parameter names.
+            @ManagedOperation(value = "Retrieves the service with the given name", impact = "INFO")
+            public Object getService(@Name(value = "serviceName") String n)
+            {
+                return services.get(n);
+            }
+        }
+        // end::jmxAnnotation[]
+    }
+
+    public void jmxCustomMBean()
+    {
+        // tag::jmxCustomMBean[]
+        //package com.acme;
+        @ManagedObject
+        class Service
+        {
+        }
+
+        //package com.acme.jmx;
+        class ServiceMBean extends ObjectMBean
+        {
+            ServiceMBean(Object service)
+            {
+                super(service);
+            }
+        }
+        // end::jmxCustomMBean[]
+    }
+
+    public void jmxCustomMBeanOverride()
+    {
+        // tag::jmxCustomMBeanOverride[]
+        //package com.acme;
+        // No Jetty JMX annotations.
+        class CountService
+        {
+            private int count;
+
+            public int getCount()
+            {
+                return count;
+            }
+
+            public void addCount(int value)
+            {
+                count += value;
+            }
+        }
+
+        //package com.acme.jmx;
+        @ManagedObject("the count service")
+        class CountServiceMBean extends ObjectMBean
+        {
+            public CountServiceMBean(Object service)
+            {
+                super(service);
+            }
+
+            private CountService getCountService()
+            {
+                return (CountService)super.getManagedObject();
+            }
+
+            @ManagedAttribute("the current service count")
+            public int getCount()
+            {
+                return getCountService().getCount();
+            }
+
+            @ManagedOperation(value = "adds the given value to the service count", impact = "ACTION")
+            public void addCount(@Name("count delta") int value)
+            {
+                getCountService().addCount(value);
+            }
+        }
+        // end::jmxCustomMBeanOverride[]
+    }
+}

jetty-documentation/src/main/java/embedded/SelectorManagerDocs.java

@@ -34,6 +34,7 @@ import org.eclipse.jetty.io.SelectorManager;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.Callback;
 
+@SuppressWarnings("unused")
 public class SelectorManagerDocs
 {
     // tag::connect[]

jetty-documentation/src/main/java/embedded/client/http/HTTPClientDocs.java

@@ -739,7 +739,7 @@ public class HTTPClientDocs
         // end::dynamicDefault[]
     }
 
-    public void dynamicOneProtocol() throws Exception
+    public void dynamicOneProtocol()
     {
         // tag::dynamicOneProtocol[]
         ClientConnector connector = new ClientConnector();
@@ -798,9 +798,10 @@ public class HTTPClientDocs
         // Make a clear-text upgrade request from HTTP/1.1 to HTTP/2.
         // The request will start as HTTP/1.1, but the response will be HTTP/2.
         ContentResponse upgradedResponse = client.newRequest("host", 8080)
-            .header(HttpHeader.UPGRADE, "h2c")
+            .headers(headers -> headers
-            .header(HttpHeader.HTTP2_SETTINGS, "")
+                .put(HttpHeader.UPGRADE, "h2c")
-            .header(HttpHeader.CONNECTION, "Upgrade, HTTP2-Settings")
+                .put(HttpHeader.HTTP2_SETTINGS, "")
+                .put(HttpHeader.CONNECTION, "Upgrade, HTTP2-Settings"))
             .send();
         // end::dynamicClearText[]
     }

jetty-fcgi/fcgi-client/src/main/java/org/eclipse/jetty/fcgi/client/http/HttpConnectionOverFCGI.java

@@ -34,6 +34,7 @@ import org.eclipse.jetty.client.HttpClient;
 import org.eclipse.jetty.client.HttpConnection;
 import org.eclipse.jetty.client.HttpDestination;
 import org.eclipse.jetty.client.HttpExchange;
+import org.eclipse.jetty.client.HttpRequest;
 import org.eclipse.jetty.client.IConnection;
 import org.eclipse.jetty.client.SendFailure;
 import org.eclipse.jetty.client.api.Connection;
@@ -357,7 +358,7 @@ public class HttpConnectionOverFCGI extends AbstractConnection implements IConne
         @Override
         public SendFailure send(HttpExchange exchange)
         {
-            Request request = exchange.getRequest();
+            HttpRequest request = exchange.getRequest();
             normalizeRequest(request);
 
             // FCGI may be multiplexed, so one channel for each exchange.

jetty-fcgi/fcgi-server/src/main/java/org/eclipse/jetty/fcgi/server/proxy/FastCGIProxyServlet.java

@@ -182,12 +182,14 @@ public class FastCGIProxyServlet extends AsyncProxyServlet.Transparent
         // If the Host header is missing, add it.
         if (!proxyRequest.getHeaders().contains(HttpHeader.HOST))
         {
-            String host = request.getServerName();
+            String server = request.getServerName();
             int port = request.getServerPort();
             if (!getHttpClient().isDefaultPort(request.getScheme(), port))
-                host += ":" + port;
+                server += ":" + port;
-            proxyRequest.header(HttpHeader.HOST, host);
+            String host = server;
-            proxyRequest.header(HttpHeader.X_FORWARDED_HOST, host);
+            proxyRequest.headers(headers -> headers
+                .put(HttpHeader.HOST, host)
+                .put(HttpHeader.X_FORWARDED_HOST, host));
         }
 
         // PHP does not like multiple Cookie headers, coalesce into one.
@@ -202,8 +204,7 @@ public class FastCGIProxyServlet extends AsyncProxyServlet.Transparent
                 String cookie = cookies.get(i);
                 builder.append(cookie);
             }
-            proxyRequest.header(HttpHeader.COOKIE, null);
+            proxyRequest.headers(headers -> headers.put(HttpHeader.COOKIE, builder.toString()));
-            proxyRequest.header(HttpHeader.COOKIE, builder.toString());
         }
 
         super.sendProxyRequest(request, proxyResponse, proxyRequest);

jetty-gcloud/jetty-gcloud-session-manager/pom.xml

@@ -69,7 +69,7 @@
     </dependency>
     <dependency>
       <groupId>org.eclipse.jetty.websocket</groupId>
-      <artifactId>websocket-servlet</artifactId>
+      <artifactId>websocket-util-server</artifactId>
       <version>${project.version}</version>
       <scope>test</scope>
     </dependency>

jetty-hazelcast/pom.xml

@@ -59,18 +59,6 @@
       <version>${project.version}</version>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.eclipse.jetty.websocket</groupId>
-      <artifactId>websocket-servlet</artifactId>
-      <version>${project.version}</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.eclipse.jetty.websocket</groupId>
-      <artifactId>websocket-jetty-server</artifactId>
-      <version>${project.version}</version>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty.toolchain</groupId>
       <artifactId>jetty-test-helper</artifactId>

jetty-hazelcast/src/test/java/org/eclipse/jetty/hazelcast/session/TestHazelcastSessions.java

@@ -20,7 +20,6 @@ package org.eclipse.jetty.hazelcast.session;
 
 import java.io.IOException;
 import java.io.PrintWriter;
-import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -29,6 +28,7 @@ import javax.servlet.http.HttpSession;
 import org.eclipse.jetty.client.HttpClient;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
@@ -42,17 +42,13 @@ import org.junit.jupiter.api.Test;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestHazelcastSessions
 {
-    public static class TestServlet
+    public static class TestServlet extends HttpServlet
-        extends HttpServlet
     {
-
         @Override
-        protected void doGet(HttpServletRequest req, HttpServletResponse resp)
+        protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException
-            throws ServletException, IOException
         {
             String arg = req.getParameter("action");
             if (arg == null)
@@ -154,16 +150,17 @@ public class TestHazelcastSessions
                 client.GET("http://localhost:" + port + contextPath + "?action=set&value=" + value);
             assertEquals(HttpServletResponse.SC_OK, response.getStatus());
             String sessionCookie = response.getHeaders().get("Set-Cookie");
-            assertTrue(sessionCookie != null);
+            assertNotNull(sessionCookie);
             // Mangle the cookie, replacing Path with $Path, etc.
-            sessionCookie = sessionCookie.replaceFirst("(\\W)(P|p)ath=", "$1\\$Path=");
+            sessionCookie = sessionCookie.replaceFirst("(\\W)([Pp])ath=", "$1\\$Path=");
 
             String resp = response.getContentAsString();
             assertEquals(resp.trim(), String.valueOf(value));
 
             // Be sure the session value is still there
+            HttpField cookie = new HttpField("Cookie", sessionCookie);
             Request request = client.newRequest("http://localhost:" + port + contextPath + "?action=get");
-            request.header("Cookie", sessionCookie);
+            request.headers(headers -> headers.put(cookie));
             response = request.send();
             assertEquals(HttpServletResponse.SC_OK, response.getStatus());
 
@@ -172,13 +169,13 @@ public class TestHazelcastSessions
 
             //Delete the session
             request = client.newRequest("http://localhost:" + port + contextPath + "?action=del");
-            request.header("Cookie", sessionCookie);
+            request.headers(headers -> headers.put(cookie));
             response = request.send();
             assertEquals(HttpServletResponse.SC_OK, response.getStatus());
 
             //Check that the session is gone
             request = client.newRequest("http://localhost:" + port + contextPath + "?action=get");
-            request.header("Cookie", sessionCookie);
+            request.headers(headers -> headers.put(cookie));
             response = request.send();
             assertEquals(HttpServletResponse.SC_OK, response.getStatus());
             resp = response.getContentAsString();

jetty-home/pom.xml

@@ -663,7 +663,7 @@
     </dependency>
     <dependency>
       <groupId>org.eclipse.jetty.websocket</groupId>
-      <artifactId>websocket-servlet</artifactId>
+      <artifactId>websocket-util-server</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>

jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/AbstractTest.java

@@ -20,6 +20,8 @@ package org.eclipse.jetty.http2.client;
 
 import java.net.InetSocketAddress;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
 import javax.servlet.http.HttpServlet;
 
 import org.eclipse.jetty.http.HostPortHttpField;
@@ -30,6 +32,7 @@ import org.eclipse.jetty.http.MetaData;
 import org.eclipse.jetty.http2.FlowControlStrategy;
 import org.eclipse.jetty.http2.api.Session;
 import org.eclipse.jetty.http2.api.server.ServerSessionListener;
+import org.eclipse.jetty.http2.server.AbstractHTTP2ServerConnectionFactory;
 import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory;
 import org.eclipse.jetty.http2.server.RawHTTP2ServerConnectionFactory;
 import org.eclipse.jetty.server.ConnectionFactory;
@@ -69,10 +72,16 @@ public class AbstractTest
     }
 
     protected void start(ServerSessionListener listener) throws Exception
+    {
+        start(listener, x -> {});
+    }
+
+    protected void start(ServerSessionListener listener, Consumer<AbstractHTTP2ServerConnectionFactory> configurator) throws Exception
     {
         RawHTTP2ServerConnectionFactory connectionFactory = new RawHTTP2ServerConnectionFactory(new HttpConfiguration(), listener);
         connectionFactory.setInitialSessionRecvWindow(FlowControlStrategy.DEFAULT_WINDOW_SIZE);
         connectionFactory.setInitialStreamRecvWindow(FlowControlStrategy.DEFAULT_WINDOW_SIZE);
+        configurator.accept(connectionFactory);
         prepareServer(connectionFactory);
         server.start();
 

jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/ConcurrentStreamCreationTest.java

@@ -0,0 +1,107 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.http2.client;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.IntStream;
+
+import org.eclipse.jetty.http.HttpFields;
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.http.MetaData;
+import org.eclipse.jetty.http2.api.Session;
+import org.eclipse.jetty.http2.api.Stream;
+import org.eclipse.jetty.http2.api.server.ServerSessionListener;
+import org.eclipse.jetty.http2.frames.HeadersFrame;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.Promise;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class ConcurrentStreamCreationTest extends AbstractTest
+{
+    @Test
+    public void testConcurrentStreamCreation() throws Exception
+    {
+        int threads = 64;
+        int runs = 1;
+        int iterations = 1024;
+        int total = threads * runs * iterations;
+        CountDownLatch serverLatch = new CountDownLatch(total);
+        start(new ServerSessionListener.Adapter()
+        {
+            @Override
+            public Stream.Listener onNewStream(Stream stream, HeadersFrame frame)
+            {
+                MetaData.Response response = new MetaData.Response(HttpVersion.HTTP_2, HttpStatus.OK_200, HttpFields.EMPTY);
+                HeadersFrame responseFrame = new HeadersFrame(stream.getId(), response, null, true);
+                stream.headers(responseFrame, Callback.NOOP);
+                serverLatch.countDown();
+                return null;
+            }
+        }, h2 -> h2.setMaxConcurrentStreams(total));
+
+        Session session = newClient(new Session.Listener.Adapter());
+
+        CyclicBarrier barrier = new CyclicBarrier(threads);
+        CountDownLatch clientLatch = new CountDownLatch(total);
+        CountDownLatch responseLatch = new CountDownLatch(runs);
+        Promise<Stream> promise = new Promise.Adapter<Stream>()
+        {
+            @Override
+            public void succeeded(Stream stream)
+            {
+                clientLatch.countDown();
+            }
+        };
+        IntStream.range(0, threads).forEach(i -> new Thread(() ->
+        {
+            try
+            {
+                barrier.await();
+                IntStream.range(0, runs).forEach(j ->
+                        IntStream.range(0, iterations).forEach(k ->
+                        {
+                            MetaData.Request request = newRequest("GET", HttpFields.EMPTY);
+                            HeadersFrame requestFrame = new HeadersFrame(request, null, true);
+                            session.newStream(requestFrame, promise, new Stream.Listener.Adapter()
+                            {
+                                @Override
+                                public void onHeaders(Stream stream, HeadersFrame frame)
+                                {
+                                    int status = ((MetaData.Response)frame.getMetaData()).getStatus();
+                                    if (status == HttpStatus.OK_200 && frame.isEndStream())
+                                        responseLatch.countDown();
+                                }
+                            });
+                        }));
+            }
+            catch (Throwable x)
+            {
+                x.printStackTrace();
+            }
+        }).start());
+        assertTrue(clientLatch.await(total, TimeUnit.MILLISECONDS), String.format("Missing streams on client: %d/%d", clientLatch.getCount(), total));
+        assertTrue(serverLatch.await(total, TimeUnit.MILLISECONDS), String.format("Missing streams on server: %d/%d", serverLatch.getCount(), total));
+        assertTrue(responseLatch.await(total, TimeUnit.MILLISECONDS), String.format("Missing response on client: %d/%d", clientLatch.getCount(), total));
+    }
+}

jetty-http2/http2-client/src/test/java/org/eclipse/jetty/http2/client/StreamCloseTest.java

@@ -196,7 +196,7 @@ public class StreamCloseTest extends AbstractTest
             @Override
             public Stream.Listener onNewStream(Stream stream, HeadersFrame frame)
             {
-                PushPromiseFrame pushFrame = new PushPromiseFrame(stream.getId(), 0, newRequest("GET", HttpFields.EMPTY));
+                PushPromiseFrame pushFrame = new PushPromiseFrame(stream.getId(), newRequest("GET", HttpFields.EMPTY));
                 stream.push(pushFrame, new Promise.Adapter<Stream>()
                 {
                     @Override

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Session.java

@@ -22,10 +22,12 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.channels.ClosedChannelException;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Queue;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -78,6 +80,7 @@ public abstract class HTTP2Session extends ContainerLifeCycle implements ISessio
     private static final Logger LOG = LoggerFactory.getLogger(HTTP2Session.class);
 
     private final ConcurrentMap<Integer, IStream> streams = new ConcurrentHashMap<>();
+    private final StreamCreator streamCreator = new StreamCreator();
     private final AtomicBiInteger streamCount = new AtomicBiInteger(); // Hi = closed, Lo = stream count
     private final AtomicInteger localStreamIds = new AtomicInteger();
     private final AtomicInteger lastRemoteStreamId = new AtomicInteger();
@@ -532,6 +535,8 @@ public abstract class HTTP2Session extends ContainerLifeCycle implements ISessio
     @Override
     public void newStream(HeadersFrame frame, Promise<Stream> promise, Stream.Listener listener)
     {
+        streamCreator.newStream(frame, promise, listener);
+/*
         try
         {
             // Synchronization is necessary to atomically create
@@ -555,6 +560,7 @@ public abstract class HTTP2Session extends ContainerLifeCycle implements ISessio
         {
             promise.failed(x);
         }
+*/
     }
 
     /**
@@ -569,18 +575,14 @@ public abstract class HTTP2Session extends ContainerLifeCycle implements ISessio
      */
     public IStream newLocalStream(HeadersFrame frameIn, HeadersFrame[] frameOut)
     {
-        HeadersFrame frame = frameIn;
         int streamId = frameIn.getStreamId();
         if (streamId <= 0)
-        {
             streamId = localStreamIds.getAndAdd(2);
-            PriorityFrame priority = frameIn.getPriority();
+
-            priority = priority == null ? null : new PriorityFrame(streamId, priority.getParentStreamId(),
+        HeadersFrame frame = streamCreator.prepareHeadersFrame(streamId, frameIn);
-                priority.getWeight(), priority.isExclusive());
-            frame = new HeadersFrame(streamId, frameIn.getMetaData(), priority, frameIn.isEndStream());
-        }
         if (frameOut != null)
             frameOut[0] = frame;
+
         return createLocalStream(streamId, (MetaData.Request)frame.getMetaData());
     }
 
@@ -592,45 +594,13 @@ public abstract class HTTP2Session extends ContainerLifeCycle implements ISessio
     @Override
     public int priority(PriorityFrame frame, Callback callback)
     {
-        int streamId = frame.getStreamId();
+        return streamCreator.priority(frame, callback);
-        IStream stream = streams.get(streamId);
-        if (stream == null)
-        {
-            streamId = localStreamIds.getAndAdd(2);
-            frame = new PriorityFrame(streamId, frame.getParentStreamId(),
-                frame.getWeight(), frame.isExclusive());
-        }
-        control(stream, callback, frame);
-        return streamId;
     }
 
     @Override
     public void push(IStream stream, Promise<Stream> promise, PushPromiseFrame frame, Stream.Listener listener)
     {
-        try
+        streamCreator.push(frame, promise, listener);
-        {
-            // Synchronization is necessary to atomically create
-            // the stream id and enqueue the frame to be sent.
-            boolean queued;
-            synchronized (this)
-            {
-                int streamId = localStreamIds.getAndAdd(2);
-                frame = new PushPromiseFrame(frame.getStreamId(), streamId, frame.getMetaData());
-
-                IStream pushStream = createLocalStream(streamId, frame.getMetaData());
-                pushStream.setListener(listener);
-
-                ControlEntry entry = new ControlEntry(frame, pushStream, new StreamPromiseCallback(promise, pushStream));
-                queued = flusher.append(entry);
-            }
-            // Iterate outside the synchronized block.
-            if (queued)
-                flusher.iterate();
-        }
-        catch (Throwable x)
-        {
-            promise.failed(x);
-        }
     }
 
     @Override
@@ -1731,4 +1701,167 @@ public abstract class HTTP2Session extends ContainerLifeCycle implements ISessio
             terminate(failure);
         }
     }
+
+    /**
+     * SPEC: It is required that stream ids are monotonically increasing.
+     * Here we use a queue to atomically create the stream id and
+     * claim the slot in the queue. Concurrent threads will only
+     * flush up to the slot with a non-null entry to make sure
+     * frames are sent strictly in their stream id order.
+     * See https://tools.ietf.org/html/rfc7540#section-5.1.1.
+     */
+    private class StreamCreator
+    {
+        private final Queue<Slot> slots = new ArrayDeque<>();
+        private Thread flushing;
+
+        private int priority(PriorityFrame frame, Callback callback)
+        {
+            Slot slot = new Slot();
+            int currentStreamId = frame.getStreamId();
+            int streamId = reserveSlot(slot, currentStreamId);
+
+            if (currentStreamId <= 0)
+                frame = new PriorityFrame(streamId, frame.getParentStreamId(), frame.getWeight(), frame.isExclusive());
+
+            assignSlotAndFlush(slot, new ControlEntry(frame, null, callback));
+            return streamId;
+        }
+
+        private void newStream(HeadersFrame frame, Promise<Stream> promise, Stream.Listener listener)
+        {
+            Slot slot = new Slot();
+            int currentStreamId = frame.getStreamId();
+            int streamId = reserveSlot(slot, currentStreamId);
+
+            frame = prepareHeadersFrame(streamId, frame);
+
+            try
+            {
+                IStream stream = HTTP2Session.this.createLocalStream(streamId, (MetaData.Request)frame.getMetaData());
+                stream.setListener(listener);
+                stream.process(new PrefaceFrame(), Callback.NOOP);
+                assignSlotAndFlush(slot, new ControlEntry(frame, stream, new StreamPromiseCallback(promise, stream)));
+            }
+            catch (Throwable x)
+            {
+                releaseSlotFlushAndFail(slot, promise, x);
+            }
+        }
+
+        private HeadersFrame prepareHeadersFrame(int streamId, HeadersFrame frame)
+        {
+            if (frame.getStreamId() <= 0)
+            {
+                PriorityFrame priority = frame.getPriority();
+                priority = priority == null ? null : new PriorityFrame(streamId, priority.getParentStreamId(), priority.getWeight(), priority.isExclusive());
+                frame = new HeadersFrame(streamId, frame.getMetaData(), priority, frame.isEndStream());
+            }
+            return frame;
+        }
+
+        private void push(PushPromiseFrame frame, Promise<Stream> promise, Stream.Listener listener)
+        {
+            Slot slot = new Slot();
+            int streamId = reserveSlot(slot, 0);
+            frame = new PushPromiseFrame(frame.getStreamId(), streamId, frame.getMetaData());
+
+            try
+            {
+                IStream stream = HTTP2Session.this.createLocalStream(streamId, frame.getMetaData());
+                stream.setListener(listener);
+                assignSlotAndFlush(slot, new ControlEntry(frame, stream, new StreamPromiseCallback(promise, stream)));
+            }
+            catch (Throwable x)
+            {
+                releaseSlotFlushAndFail(slot, promise, x);
+            }
+        }
+
+        private void assignSlotAndFlush(Slot slot, ControlEntry entry)
+        {
+            // Every time a slot entry is assigned, we must flush.
+            slot.entry = entry;
+            flush();
+        }
+
+        private int reserveSlot(Slot slot, int streamId)
+        {
+            if (streamId <= 0)
+            {
+                synchronized (this)
+                {
+                    streamId = localStreamIds.getAndAdd(2);
+                    slots.offer(slot);
+                }
+            }
+            else
+            {
+                synchronized (this)
+                {
+                    slots.offer(slot);
+                }
+            }
+            return streamId;
+        }
+
+        private void releaseSlotFlushAndFail(Slot slot, Promise<Stream> promise, Throwable x)
+        {
+            synchronized (this)
+            {
+                slots.remove(slot);
+            }
+            flush();
+            promise.failed(x);
+        }
+
+        /**
+         * Flush goes over the entries of the slots queue to flush the entries,
+         * until either one of the following two conditions is true:
+         *  - The queue is empty.
+         *  - It reaches a slot with a null entry.
+         *  When a slot with a null entry is encountered, this means a concurrent thread reserved a slot
+         *  but hasn't set its entry yet. Since entries must be flushed in order, the thread encountering
+         *  the null entry must bail out and it is up to the concurrent thread to finish up flushing.
+         *  Note that only one thread can flush at any one time, if two threads happen to call flush
+         *  concurrently, one will do the work while the other will bail out, so it is safe that all
+         *  threads call flush after they're done reserving a slot and setting the entry.
+         */
+        private void flush()
+        {
+            Thread thread = Thread.currentThread();
+            boolean queued = false;
+            while (true)
+            {
+                ControlEntry entry;
+                synchronized (this)
+                {
+                    if (flushing == null)
+                        flushing = thread;
+                    else if (flushing != thread)
+                        return; // Another thread is flushing.
+
+                    Slot slot = slots.peek();
+                    entry = slot == null ? null : slot.entry;
+
+                    if (entry == null)
+                    {
+                        flushing = null;
+                        // No more slots or null entry, so we may iterate on the flusher.
+                        break;
+                    }
+
+                    slots.poll();
+                }
+                queued |= flusher.append(entry);
+            }
+            if (queued)
+                flusher.iterate();
+        }
+
+        private class Slot
+        {
+            private volatile ControlEntry entry;
+        }
+    }
 }

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/frames/PushPromiseFrame.java

@@ -26,6 +26,11 @@ public class PushPromiseFrame extends Frame
     private final int promisedStreamId;
     private final MetaData.Request metaData;
 
+    public PushPromiseFrame(int streamId, MetaData.Request metaData)
+    {
+        this(streamId, 0, metaData);
+    }
+
     public PushPromiseFrame(int streamId, int promisedStreamId, MetaData.Request metaData)
     {
         super(FrameType.PUSH_PROMISE);

jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpConnectionOverHTTP2.java

@@ -37,7 +37,6 @@ import org.eclipse.jetty.client.HttpRequest;
 import org.eclipse.jetty.client.HttpResponse;
 import org.eclipse.jetty.client.HttpUpgrader;
 import org.eclipse.jetty.client.SendFailure;
-import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.http.HttpURI;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.http.MetaData;
@@ -61,7 +60,7 @@ public class HttpConnectionOverHTTP2 extends HttpConnection implements Sweeper.S
     private final AtomicBoolean closed = new AtomicBoolean();
     private final AtomicInteger sweeps = new AtomicInteger();
     private final Session session;
-    private boolean recycleHttpChannels;
+    private boolean recycleHttpChannels = true;
 
     public HttpConnectionOverHTTP2(HttpDestination destination, Session session)
     {
@@ -126,14 +125,14 @@ public class HttpConnectionOverHTTP2 extends HttpConnection implements Sweeper.S
     }
 
     @Override
-    protected void normalizeRequest(Request request)
+    protected void normalizeRequest(HttpRequest request)
     {
         super.normalizeRequest(request);
         if (request instanceof HttpUpgrader.Factory)
         {
             HttpUpgrader upgrader = ((HttpUpgrader.Factory)request).newHttpUpgrader(HttpVersion.HTTP_2);
-            ((HttpRequest)request).getConversation().setAttribute(HttpUpgrader.class.getName(), upgrader);
+            request.getConversation().setAttribute(HttpUpgrader.class.getName(), upgrader);
-            upgrader.prepare((HttpRequest)request);
+            upgrader.prepare(request);
         }
     }
 

jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpReceiverOverHTTP2.java

@@ -50,9 +50,13 @@ import org.eclipse.jetty.http2.frames.ResetFrame;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.Callback;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class HttpReceiverOverHTTP2 extends HttpReceiver implements HTTP2Channel.Client
 {
+    private static final Logger LOG = LoggerFactory.getLogger(HttpReceiverOverHTTP2.class);
+
     private final ContentNotifier contentNotifier = new ContentNotifier(this);
 
     public HttpReceiverOverHTTP2(HttpChannel channel)

jetty-http2/http2-http-client-transport/src/test/java/org/eclipse/jetty/http2/client/http/PushedResourcesTest.java

@@ -64,7 +64,7 @@ public class PushedResourcesTest extends AbstractTest
             {
                 HttpURI pushURI = HttpURI.from("http://localhost:" + connector.getLocalPort() + pushPath);
                 MetaData.Request pushRequest = new MetaData.Request(HttpMethod.GET.asString(), pushURI, HttpVersion.HTTP_2, HttpFields.EMPTY);
-                stream.push(new PushPromiseFrame(stream.getId(), 0, pushRequest), new Promise.Adapter<>()
+                stream.push(new PushPromiseFrame(stream.getId(), pushRequest), new Promise.Adapter<>()
                 {
                     @Override
                     public void succeeded(Stream pushStream)

jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HTTP2ServerConnection.java

@@ -94,7 +94,7 @@ public class HTTP2ServerConnection extends HTTP2Connection
     private final AtomicLong totalResponses = new AtomicLong();
     private final ServerSessionListener listener;
     private final HttpConfiguration httpConfig;
-    private boolean recycleHttpChannels;
+    private boolean recycleHttpChannels = true;
 
     public HTTP2ServerConnection(ByteBufferPool byteBufferPool, Executor executor, EndPoint endPoint, HttpConfiguration httpConfig, ServerParser parser, ISession session, int inputBufferSize, ServerSessionListener listener)
     {

jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java

@@ -272,7 +272,7 @@ public class HttpTransportOverHTTP2 implements HttpTransport
         if (LOG.isDebugEnabled())
             LOG.debug("HTTP/2 Push {}", request);
 
-        stream.push(new PushPromiseFrame(stream.getId(), 0, request), new Promise<>()
+        stream.push(new PushPromiseFrame(stream.getId(), request), new Promise<>()
         {
             @Override
             public void succeeded(Stream pushStream)

jetty-http2/http2-server/src/test/java/org/eclipse/jetty/http2/server/HTTP2ServerTest.java

@@ -34,7 +34,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.UnaryOperator;
-import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -57,7 +56,6 @@ import org.eclipse.jetty.http2.frames.SettingsFrame;
 import org.eclipse.jetty.http2.generator.Generator;
 import org.eclipse.jetty.http2.parser.Parser;
 import org.eclipse.jetty.io.ByteBufferPool;
-import org.eclipse.jetty.io.ChannelEndPoint;
 import org.eclipse.jetty.io.ManagedSelector;
 import org.eclipse.jetty.io.SocketChannelEndPoint;
 import org.eclipse.jetty.logging.StacklessLogging;
@@ -118,7 +116,7 @@ public class HTTP2ServerTest extends AbstractServerTest
         startServer(new HttpServlet()
         {
             @Override
-            protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException
+            protected void service(HttpServletRequest req, HttpServletResponse resp)
             {
                 latch.countDown();
             }
@@ -175,7 +173,7 @@ public class HTTP2ServerTest extends AbstractServerTest
         startServer(new HttpServlet()
         {
             @Override
-            protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException
+            protected void service(HttpServletRequest req, HttpServletResponse resp) throws IOException
             {
                 latch.countDown();
                 resp.getOutputStream().write(content);
@@ -321,7 +319,7 @@ public class HTTP2ServerTest extends AbstractServerTest
         startServer(new HttpServlet()
         {
             @Override
-            protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
+            protected void service(HttpServletRequest request, HttpServletResponse response) throws IOException
             {
                 try
                 {
@@ -340,7 +338,7 @@ public class HTTP2ServerTest extends AbstractServerTest
         ServerConnector connector2 = new ServerConnector(server, new HTTP2ServerConnectionFactory(new HttpConfiguration()))
         {
             @Override
-            protected ChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key) throws IOException
+            protected SocketChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key)
             {
                 return new SocketChannelEndPoint(channel, selectSet, key, getScheduler())
                 {

jetty-io/src/main/java/org/eclipse/jetty/io/AbstractEndPoint.java

@@ -469,11 +469,11 @@ public abstract class AbstractEndPoint extends IdleTimeout implements EndPoint
             name = c.getSimpleName();
         }
 
-        return String.format("%s@%h{%s<->%s,%s,fill=%s,flush=%s,to=%d/%d}",
+        return String.format("%s@%h{l=%s,r=%s,%s,fill=%s,flush=%s,to=%d/%d}",
             name,
             this,
-            getRemoteAddress(),
             getLocalAddress(),
+            getRemoteAddress(),
             _state.get(),
             _fillInterest.toStateString(),
             _writeFlusher.toStateString(),

jetty-io/src/main/java/org/eclipse/jetty/io/ChannelEndPoint.java

@@ -1,429 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
-//
-// This program and the accompanying materials are made available under
-// the terms of the Eclipse Public License 2.0 which is available at
-// https://www.eclipse.org/legal/epl-2.0
-//
-// This Source Code may also be made available under the following
-// Secondary Licenses when the conditions for such availability set
-// forth in the Eclipse Public License, v. 2.0 are satisfied:
-// the Apache License v2.0 which is available at
-// https://www.apache.org/licenses/LICENSE-2.0
-//
-// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
-// ========================================================================
-//
-
-package org.eclipse.jetty.io;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.ByteChannel;
-import java.nio.channels.CancelledKeyException;
-import java.nio.channels.GatheringByteChannel;
-import java.nio.channels.SelectionKey;
-import java.nio.channels.Selector;
-
-import org.eclipse.jetty.util.BufferUtil;
-import org.eclipse.jetty.util.thread.Invocable;
-import org.eclipse.jetty.util.thread.Scheduler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Channel End Point.
- * <p>Holds the channel and socket for an NIO endpoint.
- */
-public abstract class ChannelEndPoint extends AbstractEndPoint implements ManagedSelector.Selectable
-{
-    private static final Logger LOG = LoggerFactory.getLogger(ChannelEndPoint.class);
-
-    private final ByteChannel _channel;
-    private final GatheringByteChannel _gather;
-    protected final ManagedSelector _selector;
-    protected final SelectionKey _key;
-    private boolean _updatePending;
-
-    /**
-     * The current value for {@link SelectionKey#interestOps()}.
-     */
-    protected int _currentInterestOps;
-
-    /**
-     * The desired value for {@link SelectionKey#interestOps()}.
-     */
-    protected int _desiredInterestOps;
-
-    private abstract class RunnableTask implements Runnable, Invocable
-    {
-        final String _operation;
-
-        protected RunnableTask(String op)
-        {
-            _operation = op;
-        }
-
-        @Override
-        public String toString()
-        {
-            return String.format("CEP:%s:%s:%s", ChannelEndPoint.this, _operation, getInvocationType());
-        }
-    }
-
-    private abstract class RunnableCloseable extends RunnableTask implements Closeable
-    {
-        protected RunnableCloseable(String op)
-        {
-            super(op);
-        }
-
-        @Override
-        public void close()
-        {
-            try
-            {
-                ChannelEndPoint.this.close();
-            }
-            catch (Throwable x)
-            {
-                LOG.warn("Unable to close ChannelEndPoint", x);
-            }
-        }
-    }
-
-    private final ManagedSelector.SelectorUpdate _updateKeyAction = new ManagedSelector.SelectorUpdate()
-    {
-        @Override
-        public void update(Selector selector)
-        {
-            updateKey();
-        }
-    };
-
-    private final Runnable _runFillable = new RunnableCloseable("runFillable")
-    {
-        @Override
-        public InvocationType getInvocationType()
-        {
-            return getFillInterest().getCallbackInvocationType();
-        }
-
-        @Override
-        public void run()
-        {
-            getFillInterest().fillable();
-        }
-    };
-
-    private final Runnable _runCompleteWrite = new RunnableCloseable("runCompleteWrite")
-    {
-        @Override
-        public InvocationType getInvocationType()
-        {
-            return getWriteFlusher().getCallbackInvocationType();
-        }
-
-        @Override
-        public void run()
-        {
-            getWriteFlusher().completeWrite();
-        }
-
-        @Override
-        public String toString()
-        {
-            return String.format("CEP:%s:%s:%s->%s", ChannelEndPoint.this, _operation, getInvocationType(), getWriteFlusher());
-        }
-    };
-
-    private final Runnable _runCompleteWriteFillable = new RunnableCloseable("runCompleteWriteFillable")
-    {
-        @Override
-        public InvocationType getInvocationType()
-        {
-            InvocationType fillT = getFillInterest().getCallbackInvocationType();
-            InvocationType flushT = getWriteFlusher().getCallbackInvocationType();
-            if (fillT == flushT)
-                return fillT;
-
-            if (fillT == InvocationType.EITHER && flushT == InvocationType.NON_BLOCKING)
-                return InvocationType.EITHER;
-
-            if (fillT == InvocationType.NON_BLOCKING && flushT == InvocationType.EITHER)
-                return InvocationType.EITHER;
-
-            return InvocationType.BLOCKING;
-        }
-
-        @Override
-        public void run()
-        {
-            getWriteFlusher().completeWrite();
-            getFillInterest().fillable();
-        }
-    };
-
-    public ChannelEndPoint(ByteChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler)
-    {
-        super(scheduler);
-        _channel = channel;
-        _selector = selector;
-        _key = key;
-        _gather = (channel instanceof GatheringByteChannel) ? (GatheringByteChannel)channel : null;
-    }
-
-    @Override
-    public boolean isOpen()
-    {
-        return _channel.isOpen();
-    }
-
-    @Override
-    public void doClose()
-    {
-        if (LOG.isDebugEnabled())
-            LOG.debug("doClose {}", this);
-        try
-        {
-            _channel.close();
-        }
-        catch (IOException e)
-        {
-            LOG.debug("Unable to close channel", e);
-        }
-        finally
-        {
-            super.doClose();
-        }
-    }
-
-    @Override
-    public void onClose(Throwable cause)
-    {
-        try
-        {
-            super.onClose(cause);
-        }
-        finally
-        {
-            if (_selector != null)
-                _selector.destroyEndPoint(this, cause);
-        }
-    }
-
-    @Override
-    public int fill(ByteBuffer buffer) throws IOException
-    {
-        if (isInputShutdown())
-            return -1;
-
-        int pos = BufferUtil.flipToFill(buffer);
-        int filled;
-        try
-        {
-            filled = _channel.read(buffer);
-            if (filled > 0)
-                notIdle();
-            else if (filled == -1)
-                shutdownInput();
-        }
-        catch (IOException e)
-        {
-            LOG.debug("Unable to shutdown output", e);
-            shutdownInput();
-            filled = -1;
-        }
-        finally
-        {
-            BufferUtil.flipToFlush(buffer, pos);
-        }
-        if (LOG.isDebugEnabled())
-            LOG.debug("filled {} {}", filled, BufferUtil.toDetailString(buffer));
-        return filled;
-    }
-
-    @Override
-    public boolean flush(ByteBuffer... buffers) throws IOException
-    {
-        long flushed = 0;
-        try
-        {
-            if (buffers.length == 1)
-                flushed = _channel.write(buffers[0]);
-            else if (_gather != null && buffers.length > 1)
-                flushed = _gather.write(buffers, 0, buffers.length);
-            else
-            {
-                for (ByteBuffer b : buffers)
-                {
-                    if (b.hasRemaining())
-                    {
-                        int l = _channel.write(b);
-                        if (l > 0)
-                            flushed += l;
-                        if (b.hasRemaining())
-                            break;
-                    }
-                }
-            }
-            if (LOG.isDebugEnabled())
-                LOG.debug("flushed {} {}", flushed, this);
-        }
-        catch (IOException e)
-        {
-            throw new EofException(e);
-        }
-
-        if (flushed > 0)
-            notIdle();
-
-        for (ByteBuffer b : buffers)
-        {
-            if (!BufferUtil.isEmpty(b))
-                return false;
-        }
-
-        return true;
-    }
-
-    public ByteChannel getChannel()
-    {
-        return _channel;
-    }
-
-    @Override
-    public Object getTransport()
-    {
-        return _channel;
-    }
-
-    @Override
-    protected void needsFillInterest()
-    {
-        changeInterests(SelectionKey.OP_READ);
-    }
-
-    @Override
-    protected void onIncompleteFlush()
-    {
-        changeInterests(SelectionKey.OP_WRITE);
-    }
-
-    @Override
-    public Runnable onSelected()
-    {
-        /**
-         * This method may run concurrently with {@link #changeInterests(int)}.
-         */
-
-        int readyOps = _key.readyOps();
-        int oldInterestOps;
-        int newInterestOps;
-        synchronized (this)
-        {
-            _updatePending = true;
-            // Remove the readyOps, that here can only be OP_READ or OP_WRITE (or both).
-            oldInterestOps = _desiredInterestOps;
-            newInterestOps = oldInterestOps & ~readyOps;
-            _desiredInterestOps = newInterestOps;
-        }
-
-        boolean fillable = (readyOps & SelectionKey.OP_READ) != 0;
-        boolean flushable = (readyOps & SelectionKey.OP_WRITE) != 0;
-
-        if (LOG.isDebugEnabled())
-            LOG.debug("onSelected {}->{} r={} w={} for {}", oldInterestOps, newInterestOps, fillable, flushable, this);
-
-        // return task to complete the job
-        Runnable task = fillable
-            ? (flushable
-            ? _runCompleteWriteFillable
-            : _runFillable)
-            : (flushable
-            ? _runCompleteWrite
-            : null);
-
-        if (LOG.isDebugEnabled())
-            LOG.debug("task {}", task);
-        return task;
-    }
-
-    @Override
-    public void updateKey()
-    {
-        /**
-         * This method may run concurrently with {@link #changeInterests(int)}.
-         */
-
-        try
-        {
-            int oldInterestOps;
-            int newInterestOps;
-            synchronized (this)
-            {
-                _updatePending = false;
-                oldInterestOps = _currentInterestOps;
-                newInterestOps = _desiredInterestOps;
-                if (oldInterestOps != newInterestOps)
-                {
-                    _currentInterestOps = newInterestOps;
-                    _key.interestOps(newInterestOps);
-                }
-            }
-
-            if (LOG.isDebugEnabled())
-                LOG.debug("Key interests updated {} -> {} on {}", oldInterestOps, newInterestOps, this);
-        }
-        catch (CancelledKeyException x)
-        {
-            LOG.debug("Ignoring key update for concurrently closed channel {}", this);
-            close();
-        }
-        catch (Throwable x)
-        {
-            LOG.warn("Ignoring key update for " + this, x);
-            close();
-        }
-    }
-
-    private void changeInterests(int operation)
-    {
-        /**
-         * This method may run concurrently with
-         * {@link #updateKey()} and {@link #onSelected()}.
-         */
-
-        int oldInterestOps;
-        int newInterestOps;
-        boolean pending;
-        synchronized (this)
-        {
-            pending = _updatePending;
-            oldInterestOps = _desiredInterestOps;
-            newInterestOps = oldInterestOps | operation;
-            if (newInterestOps != oldInterestOps)
-                _desiredInterestOps = newInterestOps;
-        }
-
-        if (LOG.isDebugEnabled())
-            LOG.debug("changeInterests p={} {}->{} for {}", pending, oldInterestOps, newInterestOps, this);
-
-        if (!pending && _selector != null)
-            _selector.submit(_updateKeyAction);
-    }
-
-    @Override
-    public String toEndPointString()
-    {
-        // We do a best effort to print the right toString() and that's it.
-        return String.format("%s{io=%d/%d,kio=%d,kro=%d}",
-            super.toEndPointString(),
-            _currentInterestOps,
-            _desiredInterestOps,
-            ManagedSelector.safeInterestOps(_key),
-            ManagedSelector.safeReadyOps(_key));
-    }
-}

jetty-io/src/main/java/org/eclipse/jetty/io/ClientConnector.java

@@ -18,10 +18,10 @@
 
 package org.eclipse.jetty.io;
 
-import java.io.Closeable;
 import java.io.IOException;
 import java.net.SocketAddress;
 import java.net.SocketException;
+import java.net.StandardSocketOptions;
 import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
 import java.nio.channels.SocketChannel;
@@ -58,6 +58,7 @@ public class ClientConnector extends ContainerLifeCycle
     private Duration connectTimeout = Duration.ofSeconds(5);
     private Duration idleTimeout = Duration.ofSeconds(30);
     private SocketAddress bindAddress;
+    private boolean reuseAddress = true;
 
     public Executor getExecutor()
     {
@@ -165,6 +166,16 @@ public class ClientConnector extends ContainerLifeCycle
         this.bindAddress = bindAddress;
     }
 
+    public boolean getReuseAddress()
+    {
+        return reuseAddress;
+    }
+
+    public void setReuseAddress(boolean reuseAddress)
+    {
+        this.reuseAddress = reuseAddress;
+    }
+
     @Override
     protected void doStart() throws Exception
     {
@@ -219,8 +230,10 @@ public class ClientConnector extends ContainerLifeCycle
             SocketAddress bindAddress = getBindAddress();
             if (bindAddress != null)
             {
+                boolean reuseAddress = getReuseAddress();
                 if (LOG.isDebugEnabled())
-                    LOG.debug("Binding to {} to connect to {}", bindAddress, address);
+                    LOG.debug("Binding to {} to connect to {}{}", bindAddress, address, (reuseAddress ? " reusing address" : ""));
+                channel.setOption(StandardSocketOptions.SO_REUSEADDR, reuseAddress);
                 channel.bind(bindAddress);
             }
             configure(channel);
@@ -253,7 +266,7 @@ public class ClientConnector extends ContainerLifeCycle
             // exception is being thrown, so we attempt to provide a better error message.
             if (x.getClass() == SocketException.class)
                 x = new SocketException("Could not connect to " + address).initCause(x);
-            safeClose(channel);
+            IO.close(channel);
             connectFailed(x, context);
         }
     }
@@ -273,23 +286,23 @@ public class ClientConnector extends ContainerLifeCycle
         {
             if (LOG.isDebugEnabled())
                 LOG.debug("Could not accept {}", channel);
-            safeClose(channel);
+            IO.close(channel);
             Promise<?> promise = (Promise<?>)context.get(CONNECTION_PROMISE_CONTEXT_KEY);
             if (promise != null)
                 promise.failed(failure);
         }
     }
 
-    protected void safeClose(Closeable closeable)
-    {
-        IO.close(closeable);
-    }
-
     protected void configure(SocketChannel channel) throws IOException
     {
         channel.socket().setTcpNoDelay(true);
     }
 
+    protected EndPoint newEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey selectionKey)
+    {
+        return new SocketChannelEndPoint(channel, selector, selectionKey, getScheduler());
+    }
+
     protected void connectFailed(Throwable failure, Map<String, Object> context)
     {
         if (LOG.isDebugEnabled())
@@ -309,7 +322,7 @@ public class ClientConnector extends ContainerLifeCycle
         @Override
         protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey selectionKey)
         {
-            SocketChannelEndPoint endPoint = new SocketChannelEndPoint(channel, selector, selectionKey, getScheduler());
+            EndPoint endPoint = ClientConnector.this.newEndPoint((SocketChannel)channel, selector, selectionKey);
             endPoint.setIdleTimeout(getIdleTimeout().toMillis());
             return endPoint;
         }

jetty-io/src/main/java/org/eclipse/jetty/io/ManagedSelector.java

@@ -80,7 +80,7 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
     }
 
     private final AtomicBoolean _started = new AtomicBoolean(false);
-    private boolean _selecting = false;
+    private boolean _selecting;
     private final SelectorManager _selectorManager;
     private final int _id;
     private final ExecutionStrategy _strategy;
@@ -123,22 +123,6 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         start._started.await();
     }
 
-    protected void onSelectFailed(Throwable cause)
-    {
-        // override to change behavior
-    }
-
-    public int size()
-    {
-        Selector s = _selector;
-        if (s == null)
-            return 0;
-        Set<SelectionKey> keys = s.keys();
-        if (keys == null)
-            return 0;
-        return keys.size();
-    }
-
     @Override
     protected void doStop() throws Exception
     {
@@ -160,22 +144,119 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         super.doStop();
     }
 
+    protected int nioSelect(Selector selector, boolean now) throws IOException
+    {
+        return now ? selector.selectNow() : selector.select();
+    }
+
+    protected int select(Selector selector) throws IOException
+    {
+        try
+        {
+            int selected = nioSelect(selector, false);
+            if (selected == 0)
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Selector {} woken with none selected", selector);
+
+                if (Thread.interrupted() && !isRunning())
+                    throw new ClosedSelectorException();
+
+                if (FORCE_SELECT_NOW)
+                    selected = nioSelect(selector, true);
+            }
+            return selected;
+        }
+        catch (ClosedSelectorException x)
+        {
+            throw x;
+        }
+        catch (Throwable x)
+        {
+            handleSelectFailure(selector, x);
+            return 0;
+        }
+    }
+
+    protected void handleSelectFailure(Selector selector, Throwable failure) throws IOException
+    {
+        LOG.info("Caught select() failure, trying to recover: {}", failure.toString());
+        if (LOG.isDebugEnabled())
+            LOG.debug("", failure);
+
+        Selector newSelector = _selectorManager.newSelector();
+        for (SelectionKey oldKey : selector.keys())
+        {
+            SelectableChannel channel = oldKey.channel();
+            int interestOps = safeInterestOps(oldKey);
+            if (interestOps >= 0)
+            {
+                try
+                {
+                    Object attachment = oldKey.attachment();
+                    SelectionKey newKey = channel.register(newSelector, interestOps, attachment);
+                    if (attachment instanceof Selectable)
+                        ((Selectable)attachment).replaceKey(newKey);
+                    oldKey.cancel();
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("Transferred {} iOps={} att={}", channel, interestOps, attachment);
+                }
+                catch (Throwable t)
+                {
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("Could not transfer {}", channel, t);
+                    IO.close(channel);
+                }
+            }
+            else
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Invalid interestOps for {}", channel);
+                IO.close(channel);
+            }
+        }
+
+        IO.close(selector);
+        _selector = newSelector;
+    }
+
+    protected void onSelectFailed(Throwable cause)
+    {
+        // override to change behavior
+    }
+
+    public int size()
+    {
+        Selector s = _selector;
+        if (s == null)
+            return 0;
+        Set<SelectionKey> keys = s.keys();
+        if (keys == null)
+            return 0;
+        return keys.size();
+    }
+
     /**
      * Submit an {@link SelectorUpdate} to be acted on between calls to {@link Selector#select()}
      *
      * @param update The selector update to apply at next wakeup
      */
     public void submit(SelectorUpdate update)
+    {
+        submit(update, false);
+    }
+
+    private void submit(SelectorUpdate update, boolean lazy)
     {
         if (LOG.isDebugEnabled())
-            LOG.debug("Queued change {} on {}", update, this);
+            LOG.debug("Queued change lazy={} {} on {}", lazy, update, this);
 
         Selector selector = null;
         synchronized (ManagedSelector.this)
         {
             _updates.offer(update);
 
-            if (_selecting)
+            if (_selecting && !lazy)
             {
                 selector = _selector;
                 // To avoid the extra select wakeup.
@@ -223,7 +304,7 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         }
     }
 
-    private void processConnect(SelectionKey key, final Connect connect)
+    private void processConnect(SelectionKey key, Connect connect)
     {
         SelectableChannel channel = key.channel();
         try
@@ -271,7 +352,18 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         Object context = selectionKey.attachment();
         Connection connection = _selectorManager.newConnection(channel, endPoint, context);
         endPoint.setConnection(connection);
-        selectionKey.attach(endPoint);
+        submit(selector ->
+        {
+            SelectionKey key = selectionKey;
+            if (key.selector() != selector)
+            {
+                key = channel.keyFor(selector);
+                if (key != null && endPoint instanceof Selectable)
+                    ((Selectable)endPoint).replaceKey(key);
+            }
+            if (key != null)
+                key.attach(endPoint);
+        }, true);
         endPoint.onOpen();
         endPointOpened(endPoint);
         _selectorManager.connectionOpened(connection, context);
@@ -279,7 +371,7 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             LOG.debug("Created {}", endPoint);
     }
 
-    public void destroyEndPoint(final EndPoint endPoint, Throwable cause)
+    void destroyEndPoint(EndPoint endPoint, Throwable cause)
     {
         // Waking up the selector is necessary to clean the
         // cancelled-key set and tell the TCP stack that the
@@ -330,8 +422,8 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         Selector selector = _selector;
         if (selector != null && selector.isOpen())
         {
-            final DumpKeys dump = new DumpKeys();
+            DumpKeys dump = new DumpKeys();
-            final String updatesAt = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now());
+            String updatesAt = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now());
             synchronized (ManagedSelector.this)
             {
                 updates = new ArrayList<>(_updates);
@@ -387,6 +479,14 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
          * {@link ManagedSelector} for this endpoint have been processed.
          */
         void updateKey();
+
+        /**
+         * Callback method invoked when the SelectionKey is replaced
+         * because the channel has been moved to a new selector.
+         *
+         * @param newKey the new SelectionKey
+         */
+        void replaceKey(SelectionKey newKey);
     }
 
     private class SelectorProducer implements ExecutionStrategy.Producer
@@ -434,9 +534,9 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
                         LOG.debug("update {}", update);
                     update.update(_selector);
                 }
-                catch (Throwable th)
+                catch (Throwable x)
                 {
-                    LOG.warn("Cannot update selector {}", _selector, th);
+                    LOG.warn("Cannot update selector {}", ManagedSelector.this, x);
                 }
             }
             _updateable.clear();
@@ -466,39 +566,33 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             try
             {
                 Selector selector = _selector;
-                if (selector != null && selector.isOpen())
+                if (selector != null)
                 {
                     if (LOG.isDebugEnabled())
                         LOG.debug("Selector {} waiting with {} keys", selector, selector.keys().size());
-                    int selected = selector.select();
+                    int selected = ManagedSelector.this.select(selector);
-                    if (selected == 0)
+                    // The selector may have been recreated.
+                    selector = _selector;
+                    if (selector != null)
                     {
                         if (LOG.isDebugEnabled())
-                            LOG.debug("Selector {} woken with none selected", selector);
+                            LOG.debug("Selector {} woken up from select, {}/{}/{} selected", selector, selected, selector.selectedKeys().size(), selector.keys().size());
 
-                        if (Thread.interrupted() && !isRunning())
+                        int updates;
-                            throw new ClosedSelectorException();
+                        synchronized (ManagedSelector.this)
+                        {
+                            // finished selecting
+                            _selecting = false;
+                            updates = _updates.size();
+                        }
 
-                        if (FORCE_SELECT_NOW)
+                        _keys = selector.selectedKeys();
-                            selected = selector.selectNow();
+                        _cursor = _keys.isEmpty() ? Collections.emptyIterator() : _keys.iterator();
+                        if (LOG.isDebugEnabled())
+                            LOG.debug("Selector {} processing {} keys, {} updates", selector, _keys.size(), updates);
+
+                        return true;
                     }
-                    if (LOG.isDebugEnabled())
-                        LOG.debug("Selector {} woken up from select, {}/{}/{} selected", selector, selected, selector.selectedKeys().size(), selector.keys().size());
-
-                    int updates;
-                    synchronized (ManagedSelector.this)
-                    {
-                        // finished selecting
-                        _selecting = false;
-                        updates = _updates.size();
-                    }
-
-                    _keys = selector.selectedKeys();
-                    _cursor = _keys.isEmpty() ? Collections.emptyIterator() : _keys.iterator();
-                    if (LOG.isDebugEnabled())
-                        LOG.debug("Selector {} processing {} keys, {} updates", selector, _keys.size(), updates);
-
-                    return true;
                 }
             }
             catch (Throwable x)
@@ -514,7 +608,8 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
                 else
                 {
                     LOG.warn(x.toString());
-                    LOG.debug("select() failure", x);
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("select() failure", x);
                 }
             }
             return false;
@@ -525,9 +620,10 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             while (_cursor.hasNext())
             {
                 SelectionKey key = _cursor.next();
+                Object attachment = key.attachment();
+                SelectableChannel channel = key.channel();
                 if (key.isValid())
                 {
-                    Object attachment = key.attachment();
                     if (LOG.isDebugEnabled())
                         LOG.debug("selected {} {} {} ", safeReadyOps(key), key, attachment);
                     try
@@ -550,24 +646,21 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
                     }
                     catch (CancelledKeyException x)
                     {
-                        LOG.debug("Ignoring cancelled key for channel {}", key.channel());
+                        if (LOG.isDebugEnabled())
-                        if (attachment instanceof EndPoint)
+                            LOG.debug("Ignoring cancelled key for channel {}", channel);
-                            IO.close((EndPoint)attachment);
+                        IO.close(attachment instanceof EndPoint ? (EndPoint)attachment : channel);
                     }
                     catch (Throwable x)
                     {
-                        LOG.warn("Could not process key for channel " + key.channel(), x);
+                        LOG.warn("Could not process key for channel {}", channel, x);
-                        if (attachment instanceof EndPoint)
+                        IO.close(attachment instanceof EndPoint ? (EndPoint)attachment : channel);
-                            IO.close((EndPoint)attachment);
                     }
                 }
                 else
                 {
                     if (LOG.isDebugEnabled())
-                        LOG.debug("Selector loop ignoring invalid key for channel {}", key.channel());
+                        LOG.debug("Selector loop ignoring invalid key for channel {}", channel);
-                    Object attachment = key.attachment();
+                    IO.close(attachment instanceof EndPoint ? (EndPoint)attachment : channel);
-                    if (attachment instanceof EndPoint)
-                        IO.close((EndPoint)attachment);
                 }
             }
             return null;
@@ -616,7 +709,7 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
 
     private static class DumpKeys implements SelectorUpdate
     {
-        private CountDownLatch latch = new CountDownLatch(1);
+        private final CountDownLatch latch = new CountDownLatch(1);
         private List<String> keys;
 
         @Override
@@ -652,9 +745,9 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         private final SelectableChannel _channel;
         private SelectionKey _key;
 
-        public Acceptor(SelectableChannel channel)
+        Acceptor(SelectableChannel channel)
         {
-            this._channel = channel;
+            _channel = channel;
         }
 
         @Override
@@ -662,31 +755,26 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         {
             try
             {
-                if (_key == null)
+                _key = _channel.register(selector, SelectionKey.OP_ACCEPT, this);
-                {
-                    _key = _channel.register(selector, SelectionKey.OP_ACCEPT, this);
-                }
-
                 if (LOG.isDebugEnabled())
-                    LOG.debug("{} acceptor={}", this, _key);
+                    LOG.debug("{} acceptor={}", this, _channel);
             }
             catch (Throwable x)
             {
                 IO.close(_channel);
-                LOG.warn("Unable to register OP_ACCEPT on selector", x);
+                LOG.warn("Unable to register OP_ACCEPT on selector for {}", _channel, x);
             }
         }
 
         @Override
         public Runnable onSelected()
         {
-            SelectableChannel server = _key.channel();
             SelectableChannel channel = null;
             try
             {
                 while (true)
                 {
-                    channel = _selectorManager.doAccept(server);
+                    channel = _selectorManager.doAccept(_channel);
                     if (channel == null)
                         break;
                     _selectorManager.accepted(channel);
@@ -694,10 +782,9 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             }
             catch (Throwable x)
             {
+                LOG.warn("Accept failed for channel {}", channel, x);
                 IO.close(channel);
-                LOG.warn("Accept failed for channel " + channel, x);
             }
-
             return null;
         }
 
@@ -706,13 +793,18 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         {
         }
 
+        @Override
+        public void replaceKey(SelectionKey newKey)
+        {
+            _key = newKey;
+        }
+
         @Override
         public void close() throws IOException
         {
-            SelectionKey key = _key;
+            // May be called from any thread.
-            _key = null;
+            // Implements AbstractConnector.setAccepting(boolean).
-            if (key != null && key.isValid())
+            submit(selector -> _key.cancel());
-                key.cancel();
         }
     }
 
@@ -732,7 +824,8 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         @Override
         public void close()
         {
-            LOG.debug("closed accept of {}", channel);
+            if (LOG.isDebugEnabled())
+                LOG.debug("closed accept of {}", channel);
             IO.close(channel);
         }
 
@@ -748,7 +841,8 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             {
                 IO.close(channel);
                 _selectorManager.onAcceptFailed(channel, x);
-                LOG.debug("Unable to register update for accept", x);
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Could not register channel after accept {}", channel, x);
             }
         }
 
@@ -762,7 +856,6 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             }
             catch (Throwable x)
             {
-                LOG.debug("Unable to accept", x);
                 failed(x);
             }
         }
@@ -770,10 +863,17 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         protected void failed(Throwable failure)
         {
             IO.close(channel);
-            LOG.warn("ManagedSelector#Accept failure : {}", Objects.toString(failure));
+            LOG.warn("Could not accept {}: {}", channel, String.valueOf(failure));
-            LOG.debug("ManagedSelector#Accept failure", failure);
+            if (LOG.isDebugEnabled())
+                LOG.debug("", failure);
             _selectorManager.onAcceptFailed(channel, failure);
         }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s@%x[%s]", getClass().getSimpleName(), hashCode(), channel);
+        }
     }
 
     class Connect implements SelectorUpdate, Runnable
@@ -833,16 +933,15 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
 
     private class CloseConnections implements SelectorUpdate
     {
-        final Set<Closeable> _closed;
+        private final Set<Closeable> _closed;
-        final CountDownLatch _noEndPoints = new CountDownLatch(1);
+        private final CountDownLatch _complete = new CountDownLatch(1);
-        final CountDownLatch _complete = new CountDownLatch(1);
 
-        public CloseConnections()
+        private CloseConnections()
         {
             this(null);
         }
 
-        public CloseConnections(Set<Closeable> closed)
+        private CloseConnections(Set<Closeable> closed)
         {
             _closed = closed;
         }
@@ -852,7 +951,6 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         {
             if (LOG.isDebugEnabled())
                 LOG.debug("Closing {} connections on {}", selector.keys().size(), ManagedSelector.this);
-            boolean zero = true;
             for (SelectionKey key : selector.keys())
             {
                 if (key != null && key.isValid())
@@ -861,14 +959,9 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
                     Object attachment = key.attachment();
                     if (attachment instanceof EndPoint)
                     {
-                        EndPoint endp = (EndPoint)attachment;
+                        EndPoint endPoint = (EndPoint)attachment;
-                        if (!endp.isOutputShutdown())
+                        Connection connection = endPoint.getConnection();
-                            zero = false;
+                        closeable = Objects.requireNonNullElse(connection, endPoint);
-                        Connection connection = endp.getConnection();
-                        if (connection != null)
-                            closeable = connection;
-                        else
-                            closeable = endp;
                     }
 
                     if (closeable != null)
@@ -885,30 +978,26 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
                     }
                 }
             }
-
-            if (zero)
-                _noEndPoints.countDown();
             _complete.countDown();
         }
     }
 
     private class StopSelector implements SelectorUpdate
     {
-        CountDownLatch _stopped = new CountDownLatch(1);
+        private final CountDownLatch _stopped = new CountDownLatch(1);
 
         @Override
         public void update(Selector selector)
         {
             for (SelectionKey key : selector.keys())
             {
-                if (key != null && key.isValid())
+                // Key may be null when using the UnixSocket selector.
-                {
+                if (key == null)
-                    Object attachment = key.attachment();
+                    continue;
-                    if (attachment instanceof EndPoint)
+                Object attachment = key.attachment();
-                        IO.close((EndPoint)attachment);
+                if (attachment instanceof Closeable)
-                }
+                    IO.close((Closeable)attachment);
             }
-
             _selector = null;
             IO.close(selector);
             _stopped.countDown();
@@ -936,8 +1025,9 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             catch (Throwable failure)
             {
                 IO.close(_connect.channel);
-                LOG.warn("ManagedSelector#CreateEndpoint failure : {}", Objects.toString(failure));
+                LOG.warn("Could not create EndPoint {}: {}", _connect.channel, String.valueOf(failure));
-                LOG.debug("ManagedSelector#CreateEndpoint failure", failure);
+                if (LOG.isDebugEnabled())
+                    LOG.debug("", failure);
                 _connect.failed(failure);
             }
         }
@@ -945,7 +1035,7 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         @Override
         public String toString()
         {
-            return String.format("CreateEndPoint@%x{%s,%s}", hashCode(), _connect, _key);
+            return String.format("%s@%x{%s}", getClass().getSimpleName(), hashCode(), _connect);
         }
     }
 
@@ -954,7 +1044,7 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         private final EndPoint endPoint;
         private final Throwable cause;
 
-        public DestroyEndPoint(EndPoint endPoint, Throwable cause)
+        private DestroyEndPoint(EndPoint endPoint, Throwable cause)
         {
             this.endPoint = endPoint;
             this.cause = cause;

jetty-io/src/main/java/org/eclipse/jetty/io/NetworkTrafficSocketChannelEndPoint.java

@@ -20,8 +20,8 @@ package org.eclipse.jetty.io;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
 
 import org.eclipse.jetty.util.thread.Scheduler;
 import org.slf4j.Logger;
@@ -36,7 +36,7 @@ public class NetworkTrafficSocketChannelEndPoint extends SocketChannelEndPoint
 
     private final NetworkTrafficListener listener;
 
-    public NetworkTrafficSocketChannelEndPoint(SelectableChannel channel, ManagedSelector selectSet, SelectionKey key, Scheduler scheduler, long idleTimeout, NetworkTrafficListener listener)
+    public NetworkTrafficSocketChannelEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key, Scheduler scheduler, long idleTimeout, NetworkTrafficListener listener)
     {
         super(channel, selectSet, key, scheduler);
         setIdleTimeout(idleTimeout);
@@ -80,7 +80,7 @@ public class NetworkTrafficSocketChannelEndPoint extends SocketChannelEndPoint
         {
             try
             {
-                listener.opened(getSocket());
+                listener.opened(getChannel().socket());
             }
             catch (Throwable x)
             {
@@ -97,7 +97,7 @@ public class NetworkTrafficSocketChannelEndPoint extends SocketChannelEndPoint
         {
             try
             {
-                listener.closed(getSocket());
+                listener.closed(getChannel().socket());
             }
             catch (Throwable x)
             {
@@ -113,7 +113,7 @@ public class NetworkTrafficSocketChannelEndPoint extends SocketChannelEndPoint
             try
             {
                 ByteBuffer view = buffer.asReadOnlyBuffer();
-                listener.incoming(getSocket(), view);
+                listener.incoming(getChannel().socket(), view);
             }
             catch (Throwable x)
             {
@@ -128,7 +128,7 @@ public class NetworkTrafficSocketChannelEndPoint extends SocketChannelEndPoint
         {
             try
             {
-                listener.outgoing(getSocket(), view);
+                listener.outgoing(getChannel().socket(), view);
             }
             catch (Throwable x)
             {

jetty-io/src/main/java/org/eclipse/jetty/io/SelectorManager.java

@@ -195,7 +195,7 @@ public abstract class SelectorManager extends ContainerLifeCycle implements Dump
      */
     public void accept(SelectableChannel channel, Object attachment)
     {
-        final ManagedSelector selector = chooseSelector();
+        ManagedSelector selector = chooseSelector();
         selector.submit(selector.new Accept(channel, attachment));
     }
 
@@ -210,7 +210,7 @@ public abstract class SelectorManager extends ContainerLifeCycle implements Dump
      */
     public Closeable acceptor(SelectableChannel server)
     {
-        final ManagedSelector selector = chooseSelector();
+        ManagedSelector selector = chooseSelector();
         ManagedSelector.Acceptor acceptor = selector.new Acceptor(server);
         selector.submit(acceptor);
         return acceptor;

jetty-io/src/main/java/org/eclipse/jetty/io/SocketChannelEndPoint.java

@@ -18,53 +18,165 @@
 
 package org.eclipse.jetty.io;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.Socket;
-import java.nio.channels.SelectableChannel;
+import java.nio.ByteBuffer;
+import java.nio.channels.CancelledKeyException;
 import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
 import java.nio.channels.SocketChannel;
 
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.thread.Invocable;
 import org.eclipse.jetty.util.thread.Scheduler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class SocketChannelEndPoint extends ChannelEndPoint
+/**
+ * Channel End Point.
+ * <p>Holds the channel and socket for an NIO endpoint.
+ */
+public class SocketChannelEndPoint extends AbstractEndPoint implements ManagedSelector.Selectable
 {
     private static final Logger LOG = LoggerFactory.getLogger(SocketChannelEndPoint.class);
-    private final Socket _socket;
-    private final InetSocketAddress _local;
-    private final InetSocketAddress _remote;
 
-    public SocketChannelEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler)
+    private final SocketChannel _channel;
+    private final ManagedSelector _selector;
+    private SelectionKey _key;
+    private boolean _updatePending;
+    // The current value for interestOps.
+    private int _currentInterestOps;
+    // The desired value for interestOps.
+    private int _desiredInterestOps;
+
+    private abstract class RunnableTask implements Runnable, Invocable
     {
-        this((SocketChannel)channel, selector, key, scheduler);
+        final String _operation;
+
+        protected RunnableTask(String op)
+        {
+            _operation = op;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s:%s:%s", SocketChannelEndPoint.this, _operation, getInvocationType());
+        }
     }
 
+    private abstract class RunnableCloseable extends RunnableTask implements Closeable
+    {
+        protected RunnableCloseable(String op)
+        {
+            super(op);
+        }
+
+        @Override
+        public void close()
+        {
+            try
+            {
+                SocketChannelEndPoint.this.close();
+            }
+            catch (Throwable x)
+            {
+                LOG.warn("Unable to close {}", SocketChannelEndPoint.this, x);
+            }
+        }
+    }
+
+    private final ManagedSelector.SelectorUpdate _updateKeyAction = this::updateKeyAction;
+
+    private final Runnable _runFillable = new RunnableCloseable("runFillable")
+    {
+        @Override
+        public InvocationType getInvocationType()
+        {
+            return getFillInterest().getCallbackInvocationType();
+        }
+
+        @Override
+        public void run()
+        {
+            getFillInterest().fillable();
+        }
+    };
+
+    private final Runnable _runCompleteWrite = new RunnableCloseable("runCompleteWrite")
+    {
+        @Override
+        public InvocationType getInvocationType()
+        {
+            return getWriteFlusher().getCallbackInvocationType();
+        }
+
+        @Override
+        public void run()
+        {
+            getWriteFlusher().completeWrite();
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s:%s:%s->%s", SocketChannelEndPoint.this, _operation, getInvocationType(), getWriteFlusher());
+        }
+    };
+
+    private final Runnable _runCompleteWriteFillable = new RunnableCloseable("runCompleteWriteFillable")
+    {
+        @Override
+        public InvocationType getInvocationType()
+        {
+            InvocationType fillT = getFillInterest().getCallbackInvocationType();
+            InvocationType flushT = getWriteFlusher().getCallbackInvocationType();
+            if (fillT == flushT)
+                return fillT;
+
+            if (fillT == InvocationType.EITHER && flushT == InvocationType.NON_BLOCKING)
+                return InvocationType.EITHER;
+
+            if (fillT == InvocationType.NON_BLOCKING && flushT == InvocationType.EITHER)
+                return InvocationType.EITHER;
+
+            return InvocationType.BLOCKING;
+        }
+
+        @Override
+        public void run()
+        {
+            getWriteFlusher().completeWrite();
+            getFillInterest().fillable();
+        }
+    };
+
     public SocketChannelEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler)
     {
-        super(channel, selector, key, scheduler);
+        super(scheduler);
-
+        _channel = channel;
-        _socket = channel.socket();
+        _selector = selector;
-        _local = (InetSocketAddress)_socket.getLocalSocketAddress();
+        _key = key;
-        _remote = (InetSocketAddress)_socket.getRemoteSocketAddress();
-    }
-
-    public Socket getSocket()
-    {
-        return _socket;
     }
 
     @Override
     public InetSocketAddress getLocalAddress()
     {
-        return _local;
+        return (InetSocketAddress)_channel.socket().getLocalSocketAddress();
     }
 
     @Override
     public InetSocketAddress getRemoteAddress()
     {
-        return _remote;
+        return (InetSocketAddress)_channel.socket().getRemoteSocketAddress();
+    }
+
+    @Override
+    public boolean isOpen()
+    {
+        return _channel.isOpen();
     }
 
     @Override
@@ -72,12 +184,250 @@ public class SocketChannelEndPoint extends ChannelEndPoint
     {
         try
         {
-            if (!_socket.isOutputShutdown())
+            Socket socket = _channel.socket();
-                _socket.shutdownOutput();
+            if (!socket.isOutputShutdown())
+                socket.shutdownOutput();
+        }
+        catch (IOException e)
+        {
+            LOG.debug("Could not shutdown output for {}", _channel, e);
+        }
+    }
+
+    @Override
+    public void doClose()
+    {
+        if (LOG.isDebugEnabled())
+            LOG.debug("doClose {}", this);
+        try
+        {
+            _channel.close();
+        }
+        catch (IOException e)
+        {
+            LOG.debug("Unable to close channel", e);
+        }
+        finally
+        {
+            super.doClose();
+        }
+    }
+
+    @Override
+    public void onClose(Throwable cause)
+    {
+        try
+        {
+            super.onClose(cause);
+        }
+        finally
+        {
+            if (_selector != null)
+                _selector.destroyEndPoint(this, cause);
+        }
+    }
+
+    @Override
+    public int fill(ByteBuffer buffer) throws IOException
+    {
+        if (isInputShutdown())
+            return -1;
+
+        int pos = BufferUtil.flipToFill(buffer);
+        int filled;
+        try
+        {
+            filled = _channel.read(buffer);
+            if (filled > 0)
+                notIdle();
+            else if (filled == -1)
+                shutdownInput();
         }
         catch (IOException e)
         {
             LOG.debug("Unable to shutdown output", e);
+            shutdownInput();
+            filled = -1;
+        }
+        finally
+        {
+            BufferUtil.flipToFlush(buffer, pos);
+        }
+        if (LOG.isDebugEnabled())
+            LOG.debug("filled {} {}", filled, BufferUtil.toDetailString(buffer));
+        return filled;
+    }
+
+    @Override
+    public boolean flush(ByteBuffer... buffers) throws IOException
+    {
+        long flushed;
+        try
+        {
+            flushed = _channel.write(buffers);
+            if (LOG.isDebugEnabled())
+                LOG.debug("flushed {} {}", flushed, this);
+        }
+        catch (IOException e)
+        {
+            throw new EofException(e);
+        }
+
+        if (flushed > 0)
+            notIdle();
+
+        for (ByteBuffer b : buffers)
+        {
+            if (!BufferUtil.isEmpty(b))
+                return false;
+        }
+
+        return true;
+    }
+
+    public SocketChannel getChannel()
+    {
+        return _channel;
+    }
+
+    @Override
+    public Object getTransport()
+    {
+        return _channel;
+    }
+
+    @Override
+    protected void needsFillInterest()
+    {
+        changeInterests(SelectionKey.OP_READ);
+    }
+
+    @Override
+    protected void onIncompleteFlush()
+    {
+        changeInterests(SelectionKey.OP_WRITE);
+    }
+
+    @Override
+    public Runnable onSelected()
+    {
+        // This method runs from the selector thread,
+        // possibly concurrently with changeInterests(int).
+
+        int readyOps = _key.readyOps();
+        int oldInterestOps;
+        int newInterestOps;
+        synchronized (this)
+        {
+            _updatePending = true;
+            // Remove the readyOps, that here can only be OP_READ or OP_WRITE (or both).
+            oldInterestOps = _desiredInterestOps;
+            newInterestOps = oldInterestOps & ~readyOps;
+            _desiredInterestOps = newInterestOps;
+        }
+
+        boolean fillable = (readyOps & SelectionKey.OP_READ) != 0;
+        boolean flushable = (readyOps & SelectionKey.OP_WRITE) != 0;
+
+        if (LOG.isDebugEnabled())
+            LOG.debug("onSelected {}->{} r={} w={} for {}", oldInterestOps, newInterestOps, fillable, flushable, this);
+
+        // return task to complete the job
+        Runnable task = fillable
+            ? (flushable
+            ? _runCompleteWriteFillable
+            : _runFillable)
+            : (flushable
+            ? _runCompleteWrite
+            : null);
+
+        if (LOG.isDebugEnabled())
+            LOG.debug("task {}", task);
+        return task;
+    }
+
+    private void updateKeyAction(Selector selector)
+    {
+        updateKey();
+    }
+
+    @Override
+    public void updateKey()
+    {
+        // This method runs from the selector thread,
+        // possibly concurrently with changeInterests(int).
+
+        try
+        {
+            int oldInterestOps;
+            int newInterestOps;
+            synchronized (this)
+            {
+                _updatePending = false;
+                oldInterestOps = _currentInterestOps;
+                newInterestOps = _desiredInterestOps;
+                if (oldInterestOps != newInterestOps)
+                {
+                    _currentInterestOps = newInterestOps;
+                    _key.interestOps(newInterestOps);
+                }
+            }
+
+            if (LOG.isDebugEnabled())
+                LOG.debug("Key interests updated {} -> {} on {}", oldInterestOps, newInterestOps, this);
+        }
+        catch (CancelledKeyException x)
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Ignoring key update for cancelled key {}", this, x);
+            close();
+        }
+        catch (Throwable x)
+        {
+            LOG.warn("Ignoring key update for {}", this, x);
+            close();
         }
     }
+
+    @Override
+    public void replaceKey(SelectionKey newKey)
+    {
+        _key = newKey;
+    }
+
+    private void changeInterests(int operation)
+    {
+        // This method runs from any thread, possibly
+        // concurrently with updateKey() and onSelected().
+
+        int oldInterestOps;
+        int newInterestOps;
+        boolean pending;
+        synchronized (this)
+        {
+            pending = _updatePending;
+            oldInterestOps = _desiredInterestOps;
+            newInterestOps = oldInterestOps | operation;
+            if (newInterestOps != oldInterestOps)
+                _desiredInterestOps = newInterestOps;
+        }
+
+        if (LOG.isDebugEnabled())
+            LOG.debug("changeInterests p={} {}->{} for {}", pending, oldInterestOps, newInterestOps, this);
+
+        if (!pending && _selector != null)
+            _selector.submit(_updateKeyAction);
+    }
+
+    @Override
+    public String toEndPointString()
+    {
+        // We do a best effort to print the right toString() and that's it.
+        return String.format("%s{io=%d/%d,kio=%d,kro=%d}",
+            super.toEndPointString(),
+            _currentInterestOps,
+            _desiredInterestOps,
+            ManagedSelector.safeInterestOps(_key),
+            ManagedSelector.safeReadyOps(_key));
+    }
 }

jetty-io/src/test/java/org/eclipse/jetty/io/SelectorManagerTest.java

@@ -72,11 +72,11 @@ public class SelectorManagerTest
         SelectorManager selectorManager = new SelectorManager(executor, scheduler)
         {
             @Override
-            protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key) throws IOException
+            protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key)
             {
-                SocketChannelEndPoint endp = new SocketChannelEndPoint(channel, selector, key, getScheduler());
+                SocketChannelEndPoint endPoint = new SocketChannelEndPoint((SocketChannel)channel, selector, key, getScheduler());
-                endp.setIdleTimeout(connectTimeout / 2);
+                endPoint.setIdleTimeout(connectTimeout / 2);
-                return endp;
+                return endPoint;
             }
 
             @Override
@@ -96,7 +96,7 @@ public class SelectorManagerTest
             }
 
             @Override
-            public Connection newConnection(SelectableChannel channel, EndPoint endpoint, Object attachment) throws IOException
+            public Connection newConnection(SelectableChannel channel, EndPoint endpoint, Object attachment)
             {
                 ((Callback)attachment).succeeded();
                 return new AbstractConnection(endpoint, executor)

jetty-io/src/test/java/org/eclipse/jetty/io/SocketChannelEndPointInterestsTest.java

@@ -69,7 +69,7 @@ public class SocketChannelEndPointInterestsTest
             @Override
             protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key)
             {
-                SocketChannelEndPoint endp = new SocketChannelEndPoint(channel, selector, key, getScheduler())
+                SocketChannelEndPoint endp = new SocketChannelEndPoint((SocketChannel)channel, selector, key, getScheduler())
                 {
                     @Override
                     protected void onIncompleteFlush()

jetty-io/src/test/java/org/eclipse/jetty/io/SocketChannelEndPointTest.java

@@ -465,10 +465,10 @@ public class SocketChannelEndPointTest
             @Override
             protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey selectionKey)
             {
-                SocketChannelEndPoint endp = new SocketChannelEndPoint(channel, selector, selectionKey, getScheduler());
+                SocketChannelEndPoint endPoint = new SocketChannelEndPoint((SocketChannel)channel, selector, selectionKey, getScheduler());
-                _lastEndPoint = endp;
+                _lastEndPoint = endPoint;
                 _lastEndPointLatch.countDown();
-                return endp;
+                return endPoint;
             }
 
             @Override
@@ -580,11 +580,11 @@ public class SocketChannelEndPointTest
 
         protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key)
         {
-            SocketChannelEndPoint endp = new SocketChannelEndPoint(channel, selector, key, getScheduler());
+            SocketChannelEndPoint endPoint = new SocketChannelEndPoint((SocketChannel)channel, selector, key, getScheduler());
-            endp.setIdleTimeout(60000);
+            endPoint.setIdleTimeout(60000);
-            _lastEndPoint = endp;
+            _lastEndPoint = endPoint;
             _lastEndPointLatch.countDown();
-            return endp;
+            return endPoint;
         }
 
         @Override
@@ -743,7 +743,7 @@ public class SocketChannelEndPointTest
                 return;
             }
 
-            EndPoint endp = getEndPoint();
+            EndPoint endPoint = getEndPoint();
             try
             {
                 _last = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
@@ -756,17 +756,17 @@ public class SocketChannelEndPointTest
                     BufferUtil.compact(_in);
                     if (BufferUtil.isFull(_in))
                         throw new IllegalStateException("FULL " + BufferUtil.toDetailString(_in));
-                    int filled = endp.fill(_in);
+                    int filled = endPoint.fill(_in);
                     if (filled > 0)
                         progress = true;
 
                     // If the tests wants to block, then block
-                    while (_blockAt.get() > 0 && endp.isOpen() && _in.remaining() < _blockAt.get())
+                    while (_blockAt.get() > 0 && endPoint.isOpen() && _in.remaining() < _blockAt.get())
                     {
                         FutureCallback future = _blockingRead = new FutureCallback();
                         fillInterested();
                         future.get();
-                        filled = endp.fill(_in);
+                        filled = endPoint.fill(_in);
                         progress |= filled > 0;
                     }
 
@@ -782,18 +782,18 @@ public class SocketChannelEndPointTest
                         for (int i = 0; i < _writeCount.get(); i++)
                         {
                             FutureCallback blockingWrite = new FutureCallback();
-                            endp.write(blockingWrite, out.asReadOnlyBuffer());
+                            endPoint.write(blockingWrite, out.asReadOnlyBuffer());
                             blockingWrite.get();
                         }
                         progress = true;
                     }
 
                     // are we done?
-                    if (endp.isInputShutdown())
+                    if (endPoint.isInputShutdown())
-                        endp.shutdownOutput();
+                        endPoint.shutdownOutput();
                 }
 
-                if (endp.isOpen())
+                if (endPoint.isOpen())
                     fillInterested();
             }
             catch (ExecutionException e)
@@ -802,9 +802,9 @@ public class SocketChannelEndPointTest
                 try
                 {
                     FutureCallback blockingWrite = new FutureCallback();
-                    endp.write(blockingWrite, BufferUtil.toBuffer("EE: " + BufferUtil.toString(_in)));
+                    endPoint.write(blockingWrite, BufferUtil.toBuffer("EE: " + BufferUtil.toString(_in)));
                     blockingWrite.get();
-                    endp.shutdownOutput();
+                    endPoint.shutdownOutput();
                 }
                 catch (Exception e2)
                 {

jetty-jmx/src/main/java/org/eclipse/jetty/jmx/ConnectorServer.java

@@ -66,7 +66,7 @@ public class ConnectorServer extends AbstractLifeCycle
     private JMXServiceURL _jmxURL;
     private final Map<String, Object> _environment;
     private final String _objectName;
-    private final SslContextFactory _sslContextFactory;
+    private final SslContextFactory.Server _sslContextFactory;
     private int _registryPort;
     private int _rmiPort;
     private JMXConnectorServer _connectorServer;
@@ -98,7 +98,7 @@ public class ConnectorServer extends AbstractLifeCycle
         this(svcUrl, environment, name, null);
     }
 
-    public ConnectorServer(JMXServiceURL svcUrl, Map<String, ?> environment, String name, SslContextFactory sslContextFactory)
+    public ConnectorServer(JMXServiceURL svcUrl, Map<String, ?> environment, String name, SslContextFactory.Server sslContextFactory)
     {
         this._jmxURL = svcUrl;
         this._environment = environment == null ? new HashMap<>() : new HashMap<>(environment);
@@ -243,6 +243,7 @@ public class ConnectorServer extends AbstractLifeCycle
             if (_sslContextFactory == null)
             {
                 ServerSocket server = new ServerSocket();
+                server.setReuseAddress(true);
                 server.bind(new InetSocketAddress(address, port));
                 return server;
             }

jetty-jmx/src/main/java/org/eclipse/jetty/jmx/MBeanContainer.java

@@ -60,7 +60,7 @@ public class MBeanContainer implements Container.InheritedListener, Dumpable, De
 
     private final MBeanServer _mbeanServer;
     private final boolean _useCacheForOtherClassLoaders;
-    private final ConcurrentMap<Class, MetaData> _metaData = new ConcurrentHashMap<>();
+    private final ConcurrentMap<Class<?>, MetaData> _metaData = new ConcurrentHashMap<>();
     private final ConcurrentMap<Object, Container> _beans = new ConcurrentHashMap<>();
     private final ConcurrentMap<Object, ObjectName> _mbeans = new ConcurrentHashMap<>();
     private String _domain = null;

jetty-jmx/src/test/java/org/eclipse/jetty/jmx/ConnectorServerTest.java

@@ -231,7 +231,7 @@ public class ConnectorServerTest
     @Test
     public void testJMXOverTLS() throws Exception
     {
-        SslContextFactory sslContextFactory = new SslContextFactory.Server();
+        SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
         String keyStorePath = MavenTestingUtils.getTestResourcePath("keystore.p12").toString();
         String keyStorePassword = "storepwd";
         sslContextFactory.setKeyStorePath(keyStorePath);

jetty-memcached/jetty-memcached-sessions/src/test/java/org/eclipse/jetty/memcached/session/TestMemcachedSessions.java

@@ -20,7 +20,6 @@ package org.eclipse.jetty.memcached.session;
 
 import java.io.IOException;
 import java.io.PrintWriter;
-import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -29,6 +28,7 @@ import javax.servlet.http.HttpSession;
 import org.eclipse.jetty.client.HttpClient;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.server.NetworkConnector;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.session.CachingSessionDataStore;
@@ -40,7 +40,6 @@ import org.junit.jupiter.api.Test;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * TestMemcachedSessions
@@ -49,9 +48,8 @@ public class TestMemcachedSessions
 {
     public static class TestServlet extends HttpServlet
     {
-
         @Override
-        protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException
+        protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException
         {
             String arg = req.getParameter("action");
             if (arg == null)
@@ -117,16 +115,17 @@ public class TestMemcachedSessions
                 ContentResponse response = client.GET("http://localhost:" + port + contextPath + "?action=set&value=" + value);
                 assertEquals(HttpServletResponse.SC_OK, response.getStatus());
                 String sessionCookie = response.getHeaders().get("Set-Cookie");
-                assertTrue(sessionCookie != null);
+                assertNotNull(sessionCookie);
                 // Mangle the cookie, replacing Path with $Path, etc.
-                sessionCookie = sessionCookie.replaceFirst("(\\W)(P|p)ath=", "$1\\$Path=");
+                sessionCookie = sessionCookie.replaceFirst("(\\W)([Pp])ath=", "$1\\$Path=");
 
                 String resp = response.getContentAsString();
                 assertEquals(resp.trim(), String.valueOf(value));
 
                 // Be sure the session value is still there
+                HttpField cookie = new HttpField("Cookie", sessionCookie);
                 Request request = client.newRequest("http://localhost:" + port + contextPath + "?action=get");
-                request.header("Cookie", sessionCookie);
+                request.headers(headers -> headers.put(cookie));
                 response = request.send();
                 assertEquals(HttpServletResponse.SC_OK, response.getStatus());
 
@@ -135,13 +134,13 @@ public class TestMemcachedSessions
 
                 //Delete the session
                 request = client.newRequest("http://localhost:" + port + contextPath + "?action=del");
-                request.header("Cookie", sessionCookie);
+                request.headers(headers -> headers.put(cookie));
                 response = request.send();
                 assertEquals(HttpServletResponse.SC_OK, response.getStatus());
 
                 //Check that the session is gone
                 request = client.newRequest("http://localhost:" + port + contextPath + "?action=get");
-                request.header("Cookie", sessionCookie);
+                request.headers(headers -> headers.put(cookie));
                 response = request.send();
                 assertEquals(HttpServletResponse.SC_OK, response.getStatus());
                 resp = response.getContentAsString();

jetty-osgi/test-jetty-osgi/pom.xml

@@ -323,7 +323,7 @@
     </dependency>
     <dependency>
       <groupId>org.eclipse.jetty.websocket</groupId>
-      <artifactId>websocket-servlet</artifactId>
+      <artifactId>websocket-util-server</artifactId>
       <version>${project.version}</version>
       <scope>runtime</scope>
     </dependency>

jetty-osgi/test-jetty-osgi/src/test/java/org/eclipse/jetty/osgi/test/TestOSGiUtil.java

@@ -196,8 +196,8 @@ public class TestOSGiUtil
         res.add(mavenBundle().groupId("org.eclipse.jetty").artifactId("jetty-plus").versionAsInProject().start());
         res.add(mavenBundle().groupId("org.eclipse.jetty").artifactId("jetty-annotations").versionAsInProject().start());
         res.add(mavenBundle().groupId("org.eclipse.jetty.websocket").artifactId("websocket-core").versionAsInProject().start());
-        res.add(mavenBundle().groupId("org.eclipse.jetty.websocket").artifactId("websocket-servlet").versionAsInProject().start());
         res.add(mavenBundle().groupId("org.eclipse.jetty.websocket").artifactId("websocket-util").versionAsInProject().start());
+        res.add(mavenBundle().groupId("org.eclipse.jetty.websocket").artifactId("websocket-util-server").versionAsInProject().start());
         res.add(mavenBundle().groupId("org.eclipse.jetty.websocket").artifactId("websocket-jetty-api").versionAsInProject().start());
         res.add(mavenBundle().groupId("org.eclipse.jetty.websocket").artifactId("websocket-jetty-server").versionAsInProject().start());
         res.add(mavenBundle().groupId("org.eclipse.jetty.websocket").artifactId("websocket-jetty-client").versionAsInProject().start());

jetty-proxy/src/main/java/org/eclipse/jetty/proxy/AbstractProxyServlet.java

@@ -499,7 +499,7 @@ public abstract class AbstractProxyServlet extends HttpServlet
         if (_hostHeader != null)
             newHeaders.add(HttpHeader.HOST, _hostHeader);
 
-        proxyRequest.set(newHeaders);
+        proxyRequest.headers(headers -> headers.clear().add(newHeaders));
     }
 
     protected Set<String> findConnectionHeaders(HttpServletRequest clientRequest)
@@ -531,15 +531,19 @@ public abstract class AbstractProxyServlet extends HttpServlet
 
     protected void addViaHeader(Request proxyRequest)
     {
-        proxyRequest.header(HttpHeader.VIA, "http/1.1 " + getViaHost());
+        proxyRequest.headers(headers -> headers.add(HttpHeader.VIA, "http/1.1 " + getViaHost()));
     }
 
     protected void addXForwardedHeaders(HttpServletRequest clientRequest, Request proxyRequest)
     {
-        proxyRequest.header(HttpHeader.X_FORWARDED_FOR, clientRequest.getRemoteAddr());
+        proxyRequest.headers(headers -> headers.add(HttpHeader.X_FORWARDED_FOR, clientRequest.getRemoteAddr()));
-        proxyRequest.header(HttpHeader.X_FORWARDED_PROTO, clientRequest.getScheme());
+        proxyRequest.headers(headers -> headers.add(HttpHeader.X_FORWARDED_PROTO, clientRequest.getScheme()));
-        proxyRequest.header(HttpHeader.X_FORWARDED_HOST, clientRequest.getHeader(HttpHeader.HOST.asString()));
+        String hostHeader = clientRequest.getHeader(HttpHeader.HOST.asString());
-        proxyRequest.header(HttpHeader.X_FORWARDED_SERVER, clientRequest.getLocalName());
+        if (hostHeader != null)
+            proxyRequest.headers(headers -> headers.add(HttpHeader.X_FORWARDED_HOST, hostHeader));
+        String localName = clientRequest.getLocalName();
+        if (localName != null)
+            proxyRequest.headers(headers -> headers.add(HttpHeader.X_FORWARDED_SERVER, localName));
     }
 
     protected void sendProxyRequest(HttpServletRequest clientRequest, HttpServletResponse proxyResponse, Request proxyRequest)
@@ -633,13 +637,9 @@ public abstract class AbstractProxyServlet extends HttpServlet
                 }
                 builder.append(System.lineSeparator());
             }
-            _log.debug("{} proxying to downstream:{}{}{}{}{}",
+            _log.debug("{} proxying to downstream:{}{}",
                 getRequestId(clientRequest),
                 System.lineSeparator(),
-                serverResponse,
-                System.lineSeparator(),
-                serverResponse.getHeaders().toString().trim(),
-                System.lineSeparator(),
                 builder);
         }
     }

jetty-proxy/src/main/java/org/eclipse/jetty/proxy/AsyncMiddleManServlet.java

@@ -396,7 +396,7 @@ public class AsyncMiddleManServlet extends AbstractProxyServlet
                 clientRequest.setAttribute(PROXY_REQUEST_CONTENT_COMMITTED_ATTRIBUTE, true);
                 if (!expects100Continue)
                 {
-                    proxyRequest.header(HttpHeader.CONTENT_LENGTH, null);
+                    proxyRequest.headers(headers -> headers.remove(HttpHeader.CONTENT_LENGTH));
                     sendProxyRequest(clientRequest, proxyResponse, proxyRequest);
                 }
             }

jetty-proxy/src/main/java/org/eclipse/jetty/proxy/ConnectHandler.java

@@ -506,7 +506,7 @@ public class ConnectHandler extends HandlerWrapper
         @Override
         protected EndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key)
         {
-            SocketChannelEndPoint endPoint = new SocketChannelEndPoint(channel, selector, key, getScheduler());
+            SocketChannelEndPoint endPoint = new SocketChannelEndPoint((SocketChannel)channel, selector, key, getScheduler());
             endPoint.setIdleTimeout(getIdleTimeout());
             return endPoint;
         }

jetty-proxy/src/test/java/org/eclipse/jetty/proxy/AsyncMiddleManServletTest.java

@@ -226,7 +226,7 @@ public class AsyncMiddleManServletTest
         Request.Content gzipContent = new BytesRequestContent(gzipBytes);
 
         ContentResponse response = client.newRequest("localhost", serverConnector.getLocalPort())
-            .header(HttpHeader.CONTENT_ENCODING, "gzip")
+            .headers(headers -> headers.put(HttpHeader.CONTENT_ENCODING, HttpHeaderValue.GZIP))
             .body(gzipContent)
             .timeout(5, TimeUnit.SECONDS)
             .send();
@@ -301,7 +301,7 @@ public class AsyncMiddleManServletTest
         startClient();
 
         ContentResponse response = client.newRequest("localhost", serverConnector.getLocalPort())
-            .header(HttpHeader.CONTENT_ENCODING, "gzip")
+            .headers(headers -> headers.put(HttpHeader.CONTENT_ENCODING, HttpHeaderValue.GZIP))
             .body(new BytesRequestContent(gzip(bytes)))
             .timeout(5, TimeUnit.SECONDS)
             .send();
@@ -348,7 +348,7 @@ public class AsyncMiddleManServletTest
         startClient();
 
         ContentResponse response = client.newRequest("localhost", serverConnector.getLocalPort())
-            .header(HttpHeader.CONTENT_ENCODING, "gzip")
+            .headers(headers -> headers.put(HttpHeader.CONTENT_ENCODING, HttpHeaderValue.GZIP))
             .timeout(5, TimeUnit.SECONDS)
             .send();
 
@@ -393,7 +393,7 @@ public class AsyncMiddleManServletTest
         AsyncRequestContent content = new AsyncRequestContent();
         Request request = client.newRequest("localhost", serverConnector.getLocalPort());
         FutureResponseListener listener = new FutureResponseListener(request);
-        request.header(HttpHeader.CONTENT_ENCODING, "gzip")
+        request.headers(headers -> headers.put(HttpHeader.CONTENT_ENCODING, HttpHeaderValue.GZIP))
             .body(content)
             .send(listener);
         byte[] bytes = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".getBytes(StandardCharsets.UTF_8);
@@ -438,7 +438,7 @@ public class AsyncMiddleManServletTest
 
         byte[] bytes = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".getBytes(StandardCharsets.UTF_8);
         ContentResponse response = client.newRequest("localhost", serverConnector.getLocalPort())
-            .header(HttpHeader.CONTENT_ENCODING, "gzip")
+            .headers(headers -> headers.put(HttpHeader.CONTENT_ENCODING, HttpHeaderValue.GZIP))
             .body(new BytesRequestContent(gzip(bytes)))
             .timeout(5, TimeUnit.SECONDS)
             .send();
@@ -482,7 +482,7 @@ public class AsyncMiddleManServletTest
         startClient();
 
         ContentResponse response = client.newRequest("localhost", serverConnector.getLocalPort())
-            .header(HttpHeader.CONTENT_ENCODING, "gzip")
+            .headers(headers -> headers.put(HttpHeader.CONTENT_ENCODING, HttpHeaderValue.GZIP))
             .body(new BytesRequestContent(gzip(bytes)))
             .timeout(5, TimeUnit.SECONDS)
             .send();

jetty-proxy/src/test/java/org/eclipse/jetty/proxy/ForwardProxyTLSServerTest.java

@@ -251,12 +251,13 @@ public class ForwardProxyTLSServerTest
             assertEquals(body, content);
 
             content = "body=" + body;
+            int contentLength = content.length();
             ContentResponse response2 = httpClient.newRequest("localhost", serverConnector.getLocalPort())
                 .scheme(HttpScheme.HTTPS.asString())
                 .method(HttpMethod.POST)
                 .path("/echo")
-                .header(HttpHeader.CONTENT_TYPE, MimeTypes.Type.FORM_ENCODED.asString())
+                .headers(headers -> headers.put(HttpHeader.CONTENT_TYPE, MimeTypes.Type.FORM_ENCODED.asString()))
-                .header(HttpHeader.CONTENT_LENGTH, String.valueOf(content.length()))
+                .headers(headers -> headers.put(HttpHeader.CONTENT_LENGTH, String.valueOf(contentLength)))
                 .body(new StringRequestContent(content))
                 .timeout(5, TimeUnit.SECONDS)
                 .send();
@@ -318,8 +319,8 @@ public class ForwardProxyTLSServerTest
                 .scheme(HttpScheme.HTTPS.asString())
                 .method(HttpMethod.POST)
                 .path("/echo")
-                .header(HttpHeader.CONTENT_TYPE, MimeTypes.Type.FORM_ENCODED.asString())
+                .headers(headers -> headers.put(HttpHeader.CONTENT_TYPE, MimeTypes.Type.FORM_ENCODED.asString()))
-                .header(HttpHeader.CONTENT_LENGTH, String.valueOf(body2.length()))
+                .headers(headers -> headers.put(HttpHeader.CONTENT_LENGTH, String.valueOf(body2.length())))
                 .body(new StringRequestContent(body2));
 
             // Make sure the second connection can send the exchange via the tunnel

jetty-proxy/src/test/java/org/eclipse/jetty/proxy/ProxyServletTest.java

@@ -988,7 +988,7 @@ public class ProxyServletTest
 
         String value1 = "1";
         ContentResponse response1 = client.newRequest("localhost", serverConnector.getLocalPort())
-            .header(name, value1)
+            .headers(headers -> headers.put(name, value1))
             .timeout(5, TimeUnit.SECONDS)
             .send();
         assertEquals(200, response1.getStatus());
@@ -1003,7 +1003,7 @@ public class ProxyServletTest
         {
             String value2 = "2";
             ContentResponse response2 = client2.newRequest("localhost", serverConnector.getLocalPort())
-                .header(name, value2)
+                .headers(headers -> headers.put(name, value2))
                 .timeout(5, TimeUnit.SECONDS)
                 .send();
             assertEquals(200, response2.getStatus());
@@ -1236,10 +1236,7 @@ public class ProxyServletTest
         startClient();
 
         Request request = client.newRequest("localhost", serverConnector.getLocalPort());
-        for (Map.Entry<String, String> entry : hopHeaders.entrySet())
+        hopHeaders.forEach((key, value) -> request.headers(headers -> headers.add(key, value)));
-        {
-            request.header(entry.getKey(), entry.getValue());
-        }
         ContentResponse response = request
             .timeout(5, TimeUnit.SECONDS)
             .send();
@@ -1283,7 +1280,7 @@ public class ProxyServletTest
         CountDownLatch contentLatch = new CountDownLatch(1);
         CountDownLatch clientLatch = new CountDownLatch(1);
         client.newRequest("localhost", serverConnector.getLocalPort())
-            .header(HttpHeader.EXPECT, HttpHeaderValue.CONTINUE.asString())
+            .headers(headers -> headers.put(HttpHeader.EXPECT, HttpHeaderValue.CONTINUE.asString()))
             .body(new BytesRequestContent(content))
             .onRequestContent((request, buffer) -> contentLatch.countDown())
             .send(new BufferingResponseListener()
@@ -1340,7 +1337,7 @@ public class ProxyServletTest
         requestContent.offer(ByteBuffer.wrap(content, 0, chunk1));
         CountDownLatch clientLatch = new CountDownLatch(1);
         client.newRequest("localhost", serverConnector.getLocalPort())
-            .header(HttpHeader.EXPECT, HttpHeaderValue.CONTINUE.asString())
+            .headers(headers -> headers.put(HttpHeader.EXPECT, HttpHeaderValue.CONTINUE.asString()))
             .body(requestContent)
             .send(new BufferingResponseListener()
             {
@@ -1400,7 +1397,7 @@ public class ProxyServletTest
         requestContent.offer(ByteBuffer.wrap(content, 0, chunk1));
         CountDownLatch clientLatch = new CountDownLatch(1);
         client.newRequest("localhost", serverConnector.getLocalPort())
-            .header(HttpHeader.EXPECT, HttpHeaderValue.CONTINUE.asString())
+            .headers(headers -> headers.put(HttpHeader.EXPECT, HttpHeaderValue.CONTINUE.asString()))
             .body(requestContent)
             .send(result ->
             {
@@ -1448,7 +1445,7 @@ public class ProxyServletTest
         CountDownLatch contentLatch = new CountDownLatch(1);
         CountDownLatch clientLatch = new CountDownLatch(1);
         client.newRequest("localhost", serverConnector.getLocalPort())
-            .header(HttpHeader.EXPECT, HttpHeaderValue.CONTINUE.asString())
+            .headers(headers -> headers.put(HttpHeader.EXPECT, HttpHeaderValue.CONTINUE.asString()))
             .body(new BytesRequestContent(content))
             .onRequestContent((request, buffer) -> contentLatch.countDown())
             .send(result ->

jetty-server/pom.xml

@@ -37,6 +37,14 @@
           <onlyAnalyze>org.eclipse.jetty.server.*</onlyAnalyze>
         </configuration>
       </plugin>
+      <plugin>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <argLine>
+            @{argLine} ${jetty.surefire.argLine} --add-opens org.eclipse.jetty.server/org.eclipse.jetty.server=ALL-UNNAMED
+          </argLine>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 

jetty-server/src/main/config/etc/jetty-http.xml

@@ -38,6 +38,10 @@
         <Set name="idleTimeout"><Property name="jetty.http.idleTimeout" default="30000"/></Set>
         <Set name="acceptorPriorityDelta" property="jetty.http.acceptorPriorityDelta" />
         <Set name="acceptQueueSize" property="jetty.http.acceptQueueSize" />
+        <Set name="reuseAddress"><Property name="jetty.http.reuseAddress" default="true"/></Set>
+        <Set name="acceptedTcpNoDelay"><Property name="jetty.http.acceptedTcpNoDelay" default="true"/></Set>
+        <Set name="acceptedReceiveBufferSize" property="jetty.http.acceptedReceiveBufferSize" />
+        <Set name="acceptedSendBufferSize" property="jetty.http.acceptedSendBufferSize" />
         <Get name="SelectorManager">
           <Set name="connectTimeout"><Property name="jetty.http.connectTimeout" default="15000"/></Set>
         </Get>

jetty-server/src/main/config/etc/jetty-ssl.xml

@@ -31,6 +31,10 @@
         <Set name="idleTimeout"><Property name="jetty.ssl.idleTimeout" default="30000"/></Set>
         <Set name="acceptorPriorityDelta" property="jetty.ssl.acceptorPriorityDelta"/>
         <Set name="acceptQueueSize" property="jetty.ssl.acceptQueueSize"/>
+        <Set name="reuseAddress"><Property name="jetty.ssl.reuseAddress" default="true"/></Set>
+        <Set name="acceptedTcpNoDelay"><Property name="jetty.ssl.acceptedTcpNoDelay" default="true"/></Set>
+        <Set name="acceptedReceiveBufferSize" property="jetty.ssl.acceptedReceiveBufferSize" />
+        <Set name="acceptedSendBufferSize" property="jetty.ssl.acceptedSendBufferSize" />
         <Get name="SelectorManager">
           <Set name="connectTimeout" property="jetty.ssl.connectTimeout"/>
         </Get>

jetty-server/src/main/config/modules/http.mod

@@ -39,5 +39,20 @@ etc/jetty-http.xml
 ## Thread priority delta to give to acceptor threads
 # jetty.http.acceptorPriorityDelta=0
 
+## The requested maximum length of the queue of incoming connections.
+# jetty.http.acceptQueueSize=0
+
+## Enable/disable the SO_REUSEADDR socket option.
+# jetty.http.reuseAddress=true
+
+## Enable/disable TCP_NODELAY on accepted sockets.
+# jetty.http.acceptedTcpNoDelay=true
+
+## The SO_RCVBUF option to set on accepted sockets. A value of -1 indicates that it is left to its default value.
+# jetty.http.acceptedReceiveBufferSize=-1
+
+## The SO_SNDBUF option to set on accepted sockets. A value of -1 indicates that it is left to its default value.
+# jetty.http.acceptedSendBufferSize=-1
+
 ## Connect Timeout in milliseconds
 # jetty.http.connectTimeout=15000

jetty-server/src/main/config/modules/ssl.mod

@@ -40,6 +40,21 @@ etc/jetty-ssl-context.xml
 ## Thread priority delta to give to acceptor threads
 # jetty.ssl.acceptorPriorityDelta=0
 
+## The requested maximum length of the queue of incoming connections.
+# jetty.ssl.acceptQueueSize=0
+
+## Enable/disable the SO_REUSEADDR socket option.
+# jetty.ssl.reuseAddress=true
+
+## Enable/disable TCP_NODELAY on accepted sockets.
+# jetty.ssl.acceptedTcpNoDelay=true
+
+## The SO_RCVBUF option to set on accepted sockets. A value of -1 indicates that it is left to its default value.
+# jetty.ssl.acceptedReceiveBufferSize=-1
+
+## The SO_SNDBUF option to set on accepted sockets. A value of -1 indicates that it is left to its default value.
+# jetty.ssl.acceptedSendBufferSize=-1
+
 ## Connect Timeout in milliseconds
 # jetty.ssl.connectTimeout=15000
 

jetty-server/src/main/java/org/eclipse/jetty/server/NetworkTrafficServerConnector.java

@@ -23,10 +23,10 @@ import java.nio.channels.SocketChannel;
 import java.util.concurrent.Executor;
 
 import org.eclipse.jetty.io.ByteBufferPool;
-import org.eclipse.jetty.io.ChannelEndPoint;
 import org.eclipse.jetty.io.ManagedSelector;
 import org.eclipse.jetty.io.NetworkTrafficListener;
 import org.eclipse.jetty.io.NetworkTrafficSocketChannelEndPoint;
+import org.eclipse.jetty.io.SocketChannelEndPoint;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.Scheduler;
 
@@ -81,7 +81,7 @@ public class NetworkTrafficServerConnector extends ServerConnector
     }
 
     @Override
-    protected ChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key)
+    protected SocketChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key)
     {
         return new NetworkTrafficSocketChannelEndPoint(channel, selectSet, key, getScheduler(), getIdleTimeout(), getNetworkTrafficListener());
     }

jetty-server/src/main/java/org/eclipse/jetty/server/ProxyConnectionFactory.java

@@ -641,6 +641,10 @@ public class ProxyConnectionFactory extends DetectorConnectionFactory
                     if (LOG.isDebugEnabled())
                         LOG.debug("Proxy v2 {} {}", getEndPoint(), proxyEndPoint.toString());
                 }
+                else
+                {
+                    _buffer.position(_buffer.position() + _length);
+                }
 
                 if (LOG.isDebugEnabled())
                     LOG.debug("Proxy v2 parsing dynamic packet part is now done, upgrading to {}", _nextProtocol);
@@ -777,7 +781,7 @@ public class ProxyConnectionFactory extends DetectorConnectionFactory
         {
             return _tlvs != null ? _tlvs.get(type) : null;
         }
- 
+
         @Override
         public void close(Throwable cause)
         {

jetty-server/src/main/java/org/eclipse/jetty/server/ProxyCustomizer.java

@@ -0,0 +1,108 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.server;
+
+import java.net.InetSocketAddress;
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.servlet.ServletRequest;
+
+import org.eclipse.jetty.io.EndPoint;
+import org.eclipse.jetty.util.Attributes;
+
+/**
+ * <p>Customizer that extracts the real local and remote address:port pairs from a {@link ProxyConnectionFactory}
+ * and sets them on the request with {@link ServletRequest#setAttribute(String, Object)}.
+ */
+public class ProxyCustomizer implements HttpConfiguration.Customizer
+{
+    /**
+     * The remote address attribute name.
+     */
+    public static final String REMOTE_ADDRESS_ATTRIBUTE_NAME = "org.eclipse.jetty.proxy.remote.address";
+
+    /**
+     * The remote port attribute name.
+     */
+    public static final String REMOTE_PORT_ATTRIBUTE_NAME = "org.eclipse.jetty.proxy.remote.port";
+
+    /**
+     * The local address attribute name.
+     */
+    public static final String LOCAL_ADDRESS_ATTRIBUTE_NAME = "org.eclipse.jetty.proxy.local.address";
+
+    /**
+     * The local port attribute name.
+     */
+    public static final String LOCAL_PORT_ATTRIBUTE_NAME = "org.eclipse.jetty.proxy.local.port";
+
+    @Override
+    public void customize(Connector connector, HttpConfiguration channelConfig, Request request)
+    {
+        EndPoint endPoint = request.getHttpChannel().getEndPoint();
+        if (endPoint instanceof ProxyConnectionFactory.ProxyEndPoint)
+        {
+            EndPoint underlyingEndpoint = ((ProxyConnectionFactory.ProxyEndPoint)endPoint).unwrap();
+            request.setAttributes(new ProxyAttributes(underlyingEndpoint.getRemoteAddress(), underlyingEndpoint.getLocalAddress(), request.getAttributes()));
+        }
+    }
+
+    private static class ProxyAttributes extends Attributes.Wrapper
+    {
+        private final InetSocketAddress remoteAddress;
+        private final InetSocketAddress localAddress;
+
+        private ProxyAttributes(InetSocketAddress remoteAddress, InetSocketAddress localAddress, Attributes attributes)
+        {
+            super(attributes);
+            this.remoteAddress = remoteAddress;
+            this.localAddress = localAddress;
+        }
+
+        @Override
+        public Object getAttribute(String name)
+        {
+            switch (name)
+            {
+                case REMOTE_ADDRESS_ATTRIBUTE_NAME:
+                    return remoteAddress.getAddress().getHostAddress();
+                case REMOTE_PORT_ATTRIBUTE_NAME:
+                    return remoteAddress.getPort();
+                case LOCAL_ADDRESS_ATTRIBUTE_NAME:
+                    return localAddress.getAddress().getHostAddress();
+                case LOCAL_PORT_ATTRIBUTE_NAME:
+                    return localAddress.getPort();
+                default:
+                    return super.getAttribute(name);
+            }
+        }
+
+        @Override
+        public Set<String> getAttributeNameSet()
+        {
+            Set<String> names = new HashSet<>(_attributes.getAttributeNameSet());
+            names.add(REMOTE_ADDRESS_ATTRIBUTE_NAME);
+            names.add(REMOTE_PORT_ATTRIBUTE_NAME);
+            names.add(LOCAL_ADDRESS_ATTRIBUTE_NAME);
+            names.add(LOCAL_PORT_ATTRIBUTE_NAME);
+            return names;
+        }
+    }
+}

jetty-server/src/main/java/org/eclipse/jetty/server/PushBuilderImpl.java

@@ -18,6 +18,8 @@
 
 package org.eclipse.jetty.server;
 
+import java.util.EnumSet;
+import java.util.Objects;
 import java.util.Set;
 import javax.servlet.http.PushBuilder;
 
@@ -27,6 +29,7 @@ import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpURI;
 import org.eclipse.jetty.http.MetaData;
+import org.eclipse.jetty.util.StringUtil;
 import org.eclipse.jetty.util.URIUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,7 +41,14 @@ public class PushBuilderImpl implements PushBuilder
 {
     private static final Logger LOG = LoggerFactory.getLogger(PushBuilderImpl.class);
 
-    private static final HttpField JettyPush = new HttpField("x-http2-push", "PushBuilder");
+    private static final HttpField JETTY_PUSH = new HttpField("x-http2-push", "PushBuilder");
+    private static EnumSet<HttpMethod> UNSAFE_METHODS = EnumSet.of(
+        HttpMethod.POST,
+        HttpMethod.PUT,
+        HttpMethod.DELETE,
+        HttpMethod.CONNECT,
+        HttpMethod.OPTIONS,
+        HttpMethod.TRACE);
 
     private final Request _request;
     private final HttpFields.Mutable _fields;
@@ -56,7 +66,7 @@ public class PushBuilderImpl implements PushBuilder
         _method = method;
         _queryString = queryString;
         _sessionId = sessionId;
-        _fields.add(JettyPush);
+        _fields.add(JETTY_PUSH);
         if (LOG.isDebugEnabled())
             LOG.debug("PushBuilder({} {}?{} s={} c={})", _method, _request.getRequestURI(), _queryString, _sessionId);
     }
@@ -70,6 +80,10 @@ public class PushBuilderImpl implements PushBuilder
     @Override
     public PushBuilder method(String method)
     {
+        Objects.requireNonNull(method);
+        
+        if (StringUtil.isBlank(method) || UNSAFE_METHODS.contains(HttpMethod.fromString(method)))
+            throw new IllegalArgumentException("Method not allowed for push: " + method);
         _method = method;
         return this;
     }
@@ -149,9 +163,6 @@ public class PushBuilderImpl implements PushBuilder
     @Override
     public void push()
     {
-        if (HttpMethod.POST.is(_method) || HttpMethod.PUT.is(_method))
-            throw new IllegalStateException("Bad Method " + _method);
-
         if (_path == null || _path.length() == 0)
             throw new IllegalStateException("Bad Path " + _path);
 

jetty-server/src/main/java/org/eclipse/jetty/server/Request.java

@@ -71,6 +71,7 @@ import org.eclipse.jetty.http.BadMessageException;
 import org.eclipse.jetty.http.ComplianceViolation;
 import org.eclipse.jetty.http.HostPortHttpField;
 import org.eclipse.jetty.http.HttpCookie;
+import org.eclipse.jetty.http.HttpCookie.SetCookieHttpField;
 import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
@@ -393,6 +394,11 @@ public class Request implements HttpServletRequest
 
         HttpFields.Mutable fields = HttpFields.build(getHttpFields(), NOT_PUSHED_HEADERS);
 
+        HttpField authField = getHttpFields().getField(HttpHeader.AUTHORIZATION);
+        //TODO check what to do for digest etc etc
+        if (getUserPrincipal() != null && authField.getValue().startsWith("Basic"))
+            fields.add(authField);
+
         String id;
         try
         {
@@ -410,12 +416,47 @@ public class Request implements HttpServletRequest
             id = getRequestedSessionId();
         }
 
+        Map<String,String> cookies = new HashMap<>();
+        Cookie[] existingCookies = getCookies();
+        if (existingCookies != null)
+        {        
+            for (Cookie c: getCookies())
+            {
+                cookies.put(c.getName(), c.getValue());
+            }
+        }
+
+        //Any Set-Cookies that were set on the response must be set as Cookies on the
+        //PushBuilder, unless the max-age of the cookie is <= 0
+        HttpFields responseFields = getResponse().getHttpFields();
+        for (HttpField field : responseFields)
+        {
+            HttpHeader header = field.getHeader();
+            if (header == HttpHeader.SET_COOKIE)
+            {
+                HttpCookie cookie = ((SetCookieHttpField)field).getHttpCookie();
+                if (cookie.getMaxAge() > 0)
+                    cookies.put(cookie.getName(), cookie.getValue());
+                else
+                    cookies.remove(cookie.getName());
+            }
+        }
+
+        if (!cookies.isEmpty())
+        {
+            StringBuilder buff = new StringBuilder();
+            for (Map.Entry<String,String> entry : cookies.entrySet())
+            {
+                if (buff.length() > 0)
+                    buff.append("; ");
+                buff.append(entry.getKey()).append('=').append(entry.getValue());
+            }
+            fields.add(new HttpField(HttpHeader.COOKIE, buff.toString()));
+        }
+        
         PushBuilder builder = new PushBuilderImpl(this, fields, getMethod(), getQueryString(), id);
         builder.addHeader("referer", getRequestURL().toString());
 
-        // TODO process any set cookies
-        // TODO process any user_identity
-
         return builder;
     }
 

jetty-server/src/main/java/org/eclipse/jetty/server/ServerConnector.java

@@ -35,7 +35,6 @@ import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.eclipse.jetty.io.ByteBufferPool;
-import org.eclipse.jetty.io.ChannelEndPoint;
 import org.eclipse.jetty.io.Connection;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.io.ManagedSelector;
@@ -83,6 +82,9 @@ public class ServerConnector extends AbstractNetworkConnector
     private volatile int _localPort = -1;
     private volatile int _acceptQueueSize = 0;
     private volatile boolean _reuseAddress = true;
+    private volatile boolean _acceptedTcpNoDelay = true;
+    private volatile int _acceptedReceiveBufferSize = -1;
+    private volatile int _acceptedSendBufferSize = -1;
 
     /**
      * <p>Construct a ServerConnector with a private instance of {@link HttpConnectionFactory} as the only factory.</p>
@@ -397,7 +399,11 @@ public class ServerConnector extends AbstractNetworkConnector
     {
         try
         {
-            socket.setTcpNoDelay(true);
+            socket.setTcpNoDelay(_acceptedTcpNoDelay);
+            if (_acceptedReceiveBufferSize > -1)
+                socket.setReceiveBufferSize(_acceptedReceiveBufferSize);
+            if (_acceptedSendBufferSize > -1)
+                socket.setSendBufferSize(_acceptedSendBufferSize);
         }
         catch (SocketException e)
         {
@@ -424,7 +430,7 @@ public class ServerConnector extends AbstractNetworkConnector
         return _localPort;
     }
 
-    protected ChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key) throws IOException
+    protected SocketChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key) throws IOException
     {
         SocketChannelEndPoint endpoint = new SocketChannelEndPoint(channel, selectSet, key, getScheduler());
         endpoint.setIdleTimeout(getIdleTimeout());
@@ -452,6 +458,7 @@ public class ServerConnector extends AbstractNetworkConnector
      * @return whether the server socket reuses addresses
      * @see ServerSocket#getReuseAddress()
      */
+    @ManagedAttribute("Server Socket SO_REUSEADDR")
     public boolean getReuseAddress()
     {
         return _reuseAddress;
@@ -466,6 +473,67 @@ public class ServerConnector extends AbstractNetworkConnector
         _reuseAddress = reuseAddress;
     }
 
+    /**
+     * @return whether the accepted socket gets {@link java.net.SocketOptions#TCP_NODELAY TCP_NODELAY} enabled.
+     * @see Socket#getTcpNoDelay()
+     */
+    @ManagedAttribute("Accepted Socket TCP_NODELAY")
+    public boolean getAcceptedTcpNoDelay()
+    {
+        return _acceptedTcpNoDelay;
+    }
+
+    /**
+     * @param tcpNoDelay whether {@link java.net.SocketOptions#TCP_NODELAY TCP_NODELAY} gets enabled on the the accepted socket.
+     * @see Socket#setTcpNoDelay(boolean)
+     */
+    public void setAcceptedTcpNoDelay(boolean tcpNoDelay)
+    {
+        this._acceptedTcpNoDelay = tcpNoDelay;
+    }
+
+    /**
+     * @return the {@link java.net.SocketOptions#SO_RCVBUF SO_RCVBUF} size to set onto the accepted socket.
+     * A value of -1 indicates that it is left to its default value.
+     * @see Socket#getReceiveBufferSize()
+     */
+    @ManagedAttribute("Accepted Socket SO_RCVBUF")
+    public int getAcceptedReceiveBufferSize()
+    {
+        return _acceptedReceiveBufferSize;
+    }
+
+    /**
+     * @param receiveBufferSize the {@link java.net.SocketOptions#SO_RCVBUF SO_RCVBUF} size to set onto the accepted socket.
+     * A value of -1 indicates that it is left to its default value.
+     * @see Socket#setReceiveBufferSize(int)
+     */
+    public void setAcceptedReceiveBufferSize(int receiveBufferSize)
+    {
+        this._acceptedReceiveBufferSize = receiveBufferSize;
+    }
+
+    /**
+     * @return the {@link java.net.SocketOptions#SO_SNDBUF SO_SNDBUF} size to set onto the accepted socket.
+     * A value of -1 indicates that it is left to its default value.
+     * @see Socket#getSendBufferSize()
+     */
+    @ManagedAttribute("Accepted Socket SO_SNDBUF")
+    public int getAcceptedSendBufferSize()
+    {
+        return _acceptedSendBufferSize;
+    }
+
+    /**
+     * @param sendBufferSize the {@link java.net.SocketOptions#SO_SNDBUF SO_SNDBUF} size to set onto the accepted socket.
+     * A value of -1 indicates that it is left to its default value.
+     * @see Socket#setSendBufferSize(int)
+     */
+    public void setAcceptedSendBufferSize(int sendBufferSize)
+    {
+        this._acceptedSendBufferSize = sendBufferSize;
+    }
+
     @Override
     public void setAccepting(boolean accepting)
     {
@@ -511,9 +579,9 @@ public class ServerConnector extends AbstractNetworkConnector
         }
 
         @Override
-        protected ChannelEndPoint newEndPoint(SelectableChannel channel, ManagedSelector selectSet, SelectionKey selectionKey) throws IOException
+        protected SocketChannelEndPoint newEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey selectionKey) throws IOException
         {
-            return ServerConnector.this.newEndPoint((SocketChannel)channel, selectSet, selectionKey);
+            return ServerConnector.this.newEndPoint((SocketChannel)channel, selector, selectionKey);
         }
 
         @Override

jetty-server/src/main/java/org/eclipse/jetty/server/SocketCustomizationListener.java

@@ -59,18 +59,18 @@ public class SocketCustomizationListener implements Listener
     @Override
     public void onOpened(Connection connection)
     {
-        EndPoint endp = connection.getEndPoint();
+        EndPoint endPoint = connection.getEndPoint();
         boolean ssl = false;
 
-        if (_ssl && endp instanceof DecryptedEndPoint)
+        if (_ssl && endPoint instanceof DecryptedEndPoint)
         {
-            endp = ((DecryptedEndPoint)endp).getSslConnection().getEndPoint();
+            endPoint = ((DecryptedEndPoint)endPoint).getSslConnection().getEndPoint();
             ssl = true;
         }
 
-        if (endp instanceof SocketChannelEndPoint)
+        if (endPoint instanceof SocketChannelEndPoint)
         {
-            Socket socket = ((SocketChannelEndPoint)endp).getSocket();
+            Socket socket = ((SocketChannelEndPoint)endPoint).getChannel().socket();
             customize(socket, connection.getClass(), ssl);
         }
     }

jetty-server/src/main/java/org/eclipse/jetty/server/handler/gzip/GzipHttpOutputInterceptor.java

@@ -223,7 +223,15 @@ public class GzipHttpOutputInterceptor implements HttpOutput.Interceptor
             LOG.debug("{} compressing {}", this, _deflater);
             _state.set(GZState.COMPRESSING);
 
-            gzip(content, complete, callback);
+            if (BufferUtil.isEmpty(content))
+            {
+                // We are committing, but have no content to compress, so flush empty buffer to write headers.
+                _interceptor.write(BufferUtil.EMPTY_BUFFER, complete, callback);
+            }
+            else
+            {
+                gzip(content, complete, callback);
+            }
         }
         else
             callback.failed(new WritePendingException());
@@ -406,7 +414,7 @@ public class GzipHttpOutputInterceptor implements HttpOutput.Interceptor
         @Override
         public String toString()
         {
-            return String.format("%s[content=%s last=%b copy=%s buffer=%s deflate=%s",
+            return String.format("%s[content=%s last=%b copy=%s buffer=%s deflate=%s %s]",
                 super.toString(),
                 BufferUtil.toDetailString(_content),
                 _last,

jetty-server/src/test/java/org/eclipse/jetty/server/AsyncCompletionTest.java

@@ -45,7 +45,6 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.http.tools.HttpTester;
-import org.eclipse.jetty.io.ChannelEndPoint;
 import org.eclipse.jetty.io.Connection;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.io.ManagedSelector;
@@ -132,7 +131,7 @@ public class AsyncCompletionTest extends HttpServerTestFixture
         })
         {
             @Override
-            protected ChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key) throws IOException
+            protected SocketChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key)
             {
                 return new ExtendedEndPoint(channel, selectSet, key, getScheduler());
             }

jetty-server/src/test/java/org/eclipse/jetty/server/ExtendedServerTest.java

@@ -30,7 +30,6 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.http.HttpVersion;
-import org.eclipse.jetty.io.ChannelEndPoint;
 import org.eclipse.jetty.io.Connection;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.io.ManagedSelector;
@@ -61,7 +60,7 @@ public class ExtendedServerTest extends HttpServerTestBase
         })
         {
             @Override
-            protected ChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key) throws IOException
+            protected SocketChannelEndPoint newEndPoint(SocketChannel channel, ManagedSelector selectSet, SelectionKey key)
             {
                 return new ExtendedEndPoint(channel, selectSet, key, getScheduler());
             }