Merge remote-tracking branch 'eclipse/jetty-10.0.x' into jetty-10.0.x-4407-JavaxAnnotatedConfigDefault

Signed-off-by: Lachlan Roberts <lachlan@webtide.com>

.github/stale.yml

@@ -13,7 +13,7 @@ staleLabel: Stale
 # Comment to post when marking an issue as stale. Set to `false` to disable
 markComment: >
   This issue has been automatically marked as stale because it has been a
-  full year without activit. It will be closed if no further activity occurs.
+  full year without activity. It will be closed if no further activity occurs.
   Thank you for your contributions.
 # Comment to post when closing a stale issue. Set to `false` to disable
 closeComment: >

VERSION.txt

@@ -1,7 +1,169 @@
 jetty-10.0.0-SNAPSHOT
 
-jetty-10.0.0-alpha0 - 11 July
+jetty-10.0.0.alpha1 - 26 November 2019
-   2019
+ + 97 Permanent UnavailableException thrown during servlet request handling
+   should cause servlet destroy
+ + 137 Support OAuth
+ + 155 No way to set keystore for JSR 356 websocket clients, needed for SSL
+   client authentication
+ + 250 Implement HTTP CONNECT for HTTP/2
+ + 995 UrlEncoded.encodeString should skip more characters
+ + 1036 Allow easy configuration of Scheduler-Threads and name them more
+   appropriate
+ + 1485 Add systemd service file
+ + 1743 Refactor jetty maven plugin goals to be more orthogonal
+ + 2266 Jetty maven plugin reload is triggered each time the
+   `scanIntervalSeconds` pass
+ + 2340 Remove raw ServletHandler usage examples from documentation
+ + 2429 Review HttpClient backpressure semantic
+ + 2578 Use addEventListener(EventListener listener)
+ + 2709 current default for headerCacheSize is not large enough for many
+   requests
+ + 2815 hpack fields are opaque octets
+ + 3040 Allow RFC6265 Cookies to include optional SameSite attribute.
+ + 3083 The ini-template for jetty.console-capture.dir does not match the
+   default value
+ + 3106 Websocket connection stats and request stats
+ + 3558 Error notifications can be received after a successful websocket close
+ + 3601 HTTP2 stall on reset streams
+ + 3705 Review ClientUpgradeRequest exception handling
+ + 3734 websocket suspend when input closed
+ + 3747 Make Jetty Demo work with JPMS
+ + 3787 Jetty client sometimes returns EOFException instead of
+   SSLHandshakeException on certificate errors.
+ + 3804 Weld/CDI XML backwards compat?
+ + 3806 Error Page handling Async race with ProxyServlet
+ + 3822 trustAll will not work on some servers
+ + 3829 Avoid sending empty trailer frames for http/2 responses
+ + 3840 Byte-range request performance problems with large files
+ + 3856 Different behaviour with maxFormContentSize=0 if Content-Length header
+   is present/missing
+ + 3863 Enforce use of SNI
+ + 3869 Update to ASM 7.2 for jdk 13
+ + 3872 Review exposure of JavaxWebSocketServletContainerInitializer
+ + 3876 WebSocketPartialListener is only called for initial frames, not for
+   continuation frames
+ + 3884 @WebSocket without @OnWebSocketMessage handler fails when receiving a
+   continuation frame
+ + 3888 BufferUtil.toBuffer(Resource resource,boolean direct) does not like
+   large (4G+) Resources
+ + 3906 Fix for #3840 breaks Path encapsulation in PathResource
+ + 3913 Clustered HttpSession IllegalStateException: Invalid for read
+ + 3929 Deadlock between new HTTP2Connection() and Server.stop()
+ + 3936 Race condition when modifying session + sendRedirect()
+ + 3940 Double initialization of Log
+ + 3951 Consider adding demand API to HTTP/2
+ + 3952 Server configuration for direct/heap ByteBuffers
+ + 3956 Remove and warn on use of illegal HTTP/2 response headers
+ + 3957 CustomRequestLog bad usage of MethodHandles.lookup()
+ + 3960 Fix HttpConfiguration copy constructor
+ + 3964 Improve efficiency of listeners
+ + 3968 WebSocket sporadic ReadPendingException using suspend/resume
+ + 3969 X-Forwarded-Port header customization isn't possible
+ + 3978 HTTP/2 fixes for robustly handling abnormal traffic and resource
+   exhaustion
+ + 3983 JarFileResource incorrectly lists the contents of directories with
+   spaces
+ + 3985 Improve lenient Cookie parsing
+ + 3989 Inform custom ManagedSelector of dead selector via optional
+   onFailedSelect()
+ + 4000 Add SameFileAliasChecker to help with FileSystem static file access
+   normalization on Mac and Windows
+ + 4003 Quickstart broken in jetty-10
+ + 4007 Getting NullPointerException while trying to run jetty start.run on
+   Windows
+ + 4009 ServletContextHandler setSecurityHandler broke handler chain
+ + 4020 Revert WebSocket ExtensionFactory change to interface
+ + 4022 Servlet which is added by ServletRegistration can't be started
+ + 4025 Provide more write-through behaviours for DefaultSessionCache
+ + 4027 Ensure AbstractSessionDataStore cannot be used unless it is started
+ + 4033 Ignore bad percent encodings in paths during
+   URIUtil.equalsIgnoreEncodings()
+ + 4047 Gracefully stopped Jetty not flushing all response data
+ + 4048 Multiple values in X-Forwarded-Port throw NumberFormatException
+ + 4057 NullPointerException in o.e.j.h.HttpFields
+ + 4058 Review Locker
+ + 4064 java.lang.NullPointerException initializing embedded servlet
+ + 4075 Do not fail on servlet-mapping with url-pattern /On*
+ + 4076 Restarting quickstarted webapp throws IllegalStateException:
+   ServletContainerInitializersStarter already exists
+ + 4082 Debug logging causes NullPointerException in client
+ + 4084 Use of HttpConfiguration.setBlockingTimeout(long) in jetty.xml produces
+   warning on jetty-home startup
+ + 4096 Thread in ReservedThreadExecutor does not exit when stopped
+ + 4104 Frames are sent through ExtensionStack even if WebSocket Session is
+   closed
+ + 4105 QueuedThreadPool increased thread usage and no idle thread decay
+ + 4113 HttpClient fails with JDK 13 and TLS 1.3
+ + 4115 Drop HTTP/2 pseudo headers
+ + 4121 QueuedThreadPool should support ThreadFactory behaviors
+ + 4122 QueuedThreadPool should reset thread interrupted on failed run
+ + 4124 Run websocket autobahn tests with jetty and javax apis instead of just
+   with core.
+ + 4128 OpenIdCredentials can't decode JWT ID token
+ + 4132 Should be possible to use OIDC without metadata
+ + 4138 OpenID module should use HttpClient instead of HttpURLConnection
+ + 4141 ClassCastException with non-async Servlet + async Filter +
+   HttpServletRequestWrapper
+ + 4142 Configurable HTTP/2 RateControl
+ + 4144 Naked cast to Request should be avoided
+ + 4150 Module org.eclipse.jetty.alpn.client not found, required by
+   org.eclipse.jetty.proxy
+ + 4152 WebSocket autoFragment does not fragment based on maxFrameSize
+ + 4156 IllegalStateException when forwarding to jsp with new session
+ + 4161 Regression: EofException: request lifecycle violation 
+ + 4170 Client-side alias selection based on SSLEngine
+ + 4173 NullPointerException warning in log from WebInfConfiguration after
+   upgrade
+ + 4174 ConcurrentModificationException when stopping jetty:run-war
+ + 4176 Should not set header if sendError has been called 
+ + 4177 Configure HTTP proxy with SslContextFactory
+ + 4179 Improve HttpChannel$SendCallback references for GC
+ + 4183 Jetty considers bootstrap injected class to be a "server class"
+ + 4188 Spin in HttpOutput.close
+ + 4190 Jetty hangs after thread blocked in SharedBlockingCallback.block()
+   called by HttpOutput.close
+ + 4191 Increase GzipHandler minGzipSize default value
+ + 4193 InetAccessHandler - new includeConnectors/excludeConnectors not quite
+   correct anymore
+ + 4201 Throw SSLHandshakeException in case of TLS handshake failures
+ + 4203 Some Transfer-Encoding and Content-Length combinations do not result in
+   expected 400 Bad Request
+ + 4204 Transfer-Encoding behavior does not follow RFC7230
+ + 4208 304 response with Content-Length fails, not conform to RFC7230
+ + 4209 Unused TLS connection is not closed in Java 11
+ + 4217 SslConnection.DecryptedEnpoint.flush eternal busy loop
+ + 4222 Major/Minor Version wrong (jetty 10 is servlet 4)
+ + 4227 First authorization request produced by OIDC module fails due to
+   inclusion of sessionid
+ + 4236 clean up redirect code calculation for OpenIdAuthenticator
+ + 4237 simplify openid module configuration
+ + 4240 CGI form post results in 500 response if no character encoding
+ + 4243 ErrorHandler produces invalid json error response
+ + 4247 Cookie security attributes are going to mandated by Google Chrome
+ + 4248 Websocket client UpgradeListener never reports success
+ + 4251 Http 2.0 clients cannot upgrade protocol
+ + 4258 RateControl should be per-connection
+ + 4264 Spring Boot BasicErrorController no longer invoked
+ + 4265 HttpChannel SEND_ERROR should use ErrorHandler.doError()
+ + 4277 Reading streamed gzipped body never terminates
+ + 4279 Regression: ResponseWriter#close blocks indefinitely
+ + 4282 Review HttpParser handling in case of no content
+ + 4283 Wrong package for OpenJDK8ClientALPNProcessor 
+ + 4284 Possible NullPointerException in Main.java when stopped from command
+   line
+ + 4287 Move getUriLastPathSegment(URI uri) to URIUtil
+ + 4296 Unable to create WebSocket connect if the query string of the URL has %
+   symbol.
+ + 4301 Demand beforeContent is not forwarded
+ + 4305 Jetty server ALPN shall alert fatal no_application_protocol if no
+   client application protocol is supported
+ + 4325 Deprecate SniX509ExtendedKeyManager constructor without
+   SslContextFactory$Server
+ + 4334 Better test ErrorHandler changes
+ + 4342 OpenID module cannot create HttpClient in Jetty 10
+
+jetty-10.0.0-alpha0 - 11 July 2019
  + 113 Add support for NCSA Extended Log File Format
  + 114 Bring back overlay deployer
  + 132 ClientConnector abstraction
@@ -193,6 +355,12 @@ jetty-10.0.0-alpha0 - 11 July
  + 3849 ClosedChannelException from jetty-test-webapp javax websocket chat
    example
 
+jetty-9.4.24.v20191120 - 20 November 2019
+ + 3083 The ini-template for jetty.console-capture.dir does not match the
+   default value
+ + 4128 OpenIdCredetials can't decode JWT ID token
+ + 4334 Better test ErrorHandler changes
+
 jetty-9.4.23.v20191118 - 18 November 2019
  + 1485 Add systemd service file
  + 2266 Jetty maven plugin reload is triggered each time the

jetty-client/pom.xml

@@ -142,6 +142,13 @@
       <version>1.1.1</version>
       <scope>test</scope>
     </dependency>
+    <!-- transitive dependency defined as a range and we don't want that -->
+    <dependency>
+      <groupId>net.minidev</groupId>
+      <artifactId>json-smart</artifactId>
+      <version>2.3</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>

jetty-client/src/main/java/org/eclipse/jetty/client/HttpClient.java

@@ -1101,7 +1101,7 @@ public class HttpClient extends ContainerLifeCycle
             return port == 80;
     }
 
-    static boolean isSchemeSecure(String scheme)
+    public static boolean isSchemeSecure(String scheme)
     {
         return HttpScheme.HTTPS.is(scheme) || HttpScheme.WSS.is(scheme);
     }

jetty-client/src/main/java/org/eclipse/jetty/client/HttpContent.java

@@ -26,6 +26,7 @@ import java.util.Iterator;
 import org.eclipse.jetty.client.api.ContentProvider;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.IO;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 
@@ -218,15 +219,8 @@ public class HttpContent implements Callback, Closeable
     @Override
     public void close()
     {
-        try
+        if (iterator instanceof Closeable)
-        {
+            IO.close((Closeable)iterator);
-            if (iterator instanceof Closeable)
-                ((Closeable)iterator).close();
-        }
-        catch (Throwable x)
-        {
-            LOG.ignore(x);
-        }
     }
 
     @Override

jetty-client/src/main/java/org/eclipse/jetty/client/HttpRequest.java

@@ -88,6 +88,7 @@ public class HttpRequest implements Request
     private List<RequestListener> requestListeners;
     private BiFunction<Request, Request, Response.CompleteListener> pushListener;
     private Supplier<HttpFields> trailers;
+    private String upgradeProtocol;
 
     protected HttpRequest(HttpClient client, HttpConversation conversation, URI uri)
     {
@@ -635,6 +636,12 @@ public class HttpRequest implements Request
         return this;
     }
 
+    public HttpRequest upgradeProtocol(String upgradeProtocol)
+    {
+        this.upgradeProtocol = upgradeProtocol;
+        return this;
+    }
+
     @Override
     public ContentProvider getContent()
     {
@@ -791,6 +798,11 @@ public class HttpRequest implements Request
         return trailers;
     }
 
+    public String getUpgradeProtocol()
+    {
+        return upgradeProtocol;
+    }
+
     @Override
     public boolean abort(Throwable cause)
     {

jetty-client/src/main/java/org/eclipse/jetty/client/HttpUpgrader.java

@@ -0,0 +1,34 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.client;
+
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.io.EndPoint;
+
+public interface HttpUpgrader
+{
+    public void prepare(HttpRequest request);
+
+    public void upgrade(HttpResponse response, EndPoint endPoint);
+
+    public interface Factory
+    {
+        public HttpUpgrader newHttpUpgrader(HttpVersion version);
+    }
+}

jetty-client/src/main/java/org/eclipse/jetty/client/Socks4Proxy.java

@@ -19,6 +19,7 @@
 package org.eclipse.jetty.client;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
@@ -29,6 +30,7 @@ import java.util.regex.Pattern;
 import org.eclipse.jetty.client.api.Connection;
 import org.eclipse.jetty.io.AbstractConnection;
 import org.eclipse.jetty.io.ClientConnectionFactory;
+import org.eclipse.jetty.io.ClientConnector;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.Callback;
@@ -195,6 +197,9 @@ public class Socks4Proxy extends ProxyConfiguration.Proxy
             try
             {
                 HttpDestination destination = (HttpDestination)context.get(HttpClientTransport.HTTP_DESTINATION_CONTEXT_KEY);
+                // Don't want to do DNS resolution here.
+                InetSocketAddress address = InetSocketAddress.createUnresolved(destination.getHost(), destination.getPort());
+                context.put(ClientConnector.REMOTE_SOCKET_ADDRESS_CONTEXT_KEY, address);
                 ClientConnectionFactory connectionFactory = this.connectionFactory;
                 if (destination.isSecure())
                     connectionFactory = destination.newSslClientConnectionFactory(null, connectionFactory);

jetty-client/src/main/java/org/eclipse/jetty/client/dynamic/HttpClientTransportDynamic.java

@@ -29,6 +29,7 @@ import java.util.stream.Collectors;
 import org.eclipse.jetty.alpn.client.ALPNClientConnection;
 import org.eclipse.jetty.alpn.client.ALPNClientConnectionFactory;
 import org.eclipse.jetty.client.AbstractConnectorHttpClientTransport;
+import org.eclipse.jetty.client.HttpClient;
 import org.eclipse.jetty.client.HttpClientTransport;
 import org.eclipse.jetty.client.HttpDestination;
 import org.eclipse.jetty.client.HttpRequest;
@@ -36,7 +37,6 @@ import org.eclipse.jetty.client.MultiplexConnectionPool;
 import org.eclipse.jetty.client.MultiplexHttpDestination;
 import org.eclipse.jetty.client.Origin;
 import org.eclipse.jetty.client.http.HttpClientConnectionFactory;
-import org.eclipse.jetty.http.HttpScheme;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.io.ClientConnectionFactory;
 import org.eclipse.jetty.io.ClientConnector;
@@ -121,7 +121,7 @@ public class HttpClientTransportDynamic extends AbstractConnectorHttpClientTrans
     @Override
     public HttpDestination.Key newDestinationKey(HttpRequest request, Origin origin)
     {
-        boolean ssl = HttpScheme.HTTPS.is(request.getScheme());
+        boolean ssl = HttpClient.isSchemeSecure(request.getScheme());
         String http1 = "http/1.1";
         String http2 = ssl ? "h2" : "h2c";
         List<String> protocols = List.of();

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpChannelOverHTTP.java

@@ -26,6 +26,7 @@ import org.eclipse.jetty.client.HttpExchange;
 import org.eclipse.jetty.client.HttpRequest;
 import org.eclipse.jetty.client.HttpResponse;
 import org.eclipse.jetty.client.HttpResponseException;
+import org.eclipse.jetty.client.HttpUpgrader;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.client.api.Result;
 import org.eclipse.jetty.http.HttpFields;
@@ -98,29 +99,24 @@ public class HttpChannelOverHTTP extends HttpChannel
             return result;
 
         HttpResponse response = exchange.getResponse();
-
+        if (response.getVersion() == HttpVersion.HTTP_1_1 && response.getStatus() == HttpStatus.SWITCHING_PROTOCOLS_101)
-        if ((response.getVersion() == HttpVersion.HTTP_1_1) &&
-            (response.getStatus() == HttpStatus.SWITCHING_PROTOCOLS_101))
         {
-            String nextConnection = response.getHeaders().get(HttpHeader.CONNECTION);
+            String header = response.getHeaders().get(HttpHeader.CONNECTION);
-            if ((nextConnection == null) || !nextConnection.toLowerCase(Locale.US).contains("upgrade"))
+            if (header == null || !header.toLowerCase(Locale.US).contains("upgrade"))
-            {
+                return new Result(result, new HttpResponseException("101 response without 'Connection: Upgrade'", response));
-                return new Result(result, new HttpResponseException("101 Switching Protocols without Connection: Upgrade not supported", response));
-            }
 
-            // Upgrade Response
             HttpRequest request = exchange.getRequest();
-            HttpConnectionUpgrader upgrader = (HttpConnectionUpgrader)request.getConversation().getAttribute(HttpConnectionUpgrader.class.getName());
+            HttpUpgrader upgrader = (HttpUpgrader)request.getConversation().getAttribute(HttpUpgrader.class.getName());
-            if (upgrader != null)
+            if (upgrader == null)
+                return new Result(result, new HttpResponseException("101 response without " + HttpUpgrader.class.getSimpleName(), response));
+
+            try
             {
-                try
+                upgrader.upgrade(response, getHttpConnection().getEndPoint());
-                {
+            }
-                    upgrader.upgrade(response, getHttpConnection());
+            catch (Throwable x)
-                }
+            {
-                catch (Throwable x)
+                return new Result(result, new HttpResponseException("Could not upgrade to WebSocket", response, x));
-                {
-                    return new Result(result, x);
-                }
             }
         }
 

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpConnectionOverHTTP.java

@@ -30,11 +30,14 @@ import org.eclipse.jetty.client.HttpConnection;
 import org.eclipse.jetty.client.HttpDestination;
 import org.eclipse.jetty.client.HttpExchange;
 import org.eclipse.jetty.client.HttpProxy;
+import org.eclipse.jetty.client.HttpRequest;
+import org.eclipse.jetty.client.HttpUpgrader;
 import org.eclipse.jetty.client.IConnection;
 import org.eclipse.jetty.client.SendFailure;
 import org.eclipse.jetty.client.api.Connection;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.io.AbstractConnection;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.util.Promise;
@@ -258,6 +261,12 @@ public class HttpConnectionOverHTTP extends AbstractConnection implements IConne
                 request.timeout(connectTimeout, TimeUnit.MILLISECONDS)
                         .idleTimeout(2 * connectTimeout, TimeUnit.MILLISECONDS);
             }
+            if (request instanceof HttpUpgrader.Factory)
+            {
+                HttpUpgrader upgrader = ((HttpUpgrader.Factory)request).newHttpUpgrader(HttpVersion.HTTP_1_1);
+                ((HttpRequest)request).getConversation().setAttribute(HttpUpgrader.class.getName(), upgrader);
+                upgrader.prepare((HttpRequest)request);
+            }
         }
 
         @Override

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpReceiverOverHTTP.java

@@ -338,8 +338,7 @@ public class HttpReceiverOverHTTP extends HttpReceiver implements HttpParser.Res
         if (status == HttpStatus.SWITCHING_PROTOCOLS_101)
             return true;
 
-        if (HttpMethod.CONNECT.is(exchange.getRequest().getMethod()) &&
+        if (HttpMethod.CONNECT.is(exchange.getRequest().getMethod()) && status == HttpStatus.OK_200)
-            status == HttpStatus.OK_200)
             return true;
 
         return false;

jetty-client/src/main/java/org/eclipse/jetty/client/util/MultiPartContentProvider.java

@@ -38,6 +38,7 @@ import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.io.RuntimeIOException;
 import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.IO;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 
@@ -345,10 +346,16 @@ public class MultiPartContentProvider extends AbstractTypedContentProvider imple
                         if (iterator.hasNext())
                             return iterator.next();
                         ++index;
-                        if (index == parts.size())
+                        if (index < parts.size())
-                            state = State.LAST_BOUNDARY;
+                        {
-                        else
                             state = State.MIDDLE_BOUNDARY;
+                            if (iterator instanceof Closeable)
+                                IO.close((Closeable)iterator);
+                        }
+                        else
+                        {
+                            state = State.LAST_BOUNDARY;
+                        }
                         break;
                     }
                     case MIDDLE_BOUNDARY:
@@ -383,14 +390,14 @@ public class MultiPartContentProvider extends AbstractTypedContentProvider imple
         @Override
         public void succeeded()
         {
-            if (iterator instanceof Callback)
+            if (state == State.CONTENT && iterator instanceof Callback)
                 ((Callback)iterator).succeeded();
         }
 
         @Override
         public void failed(Throwable x)
         {
-            if (iterator instanceof Callback)
+            if (state == State.CONTENT && iterator instanceof Callback)
                 ((Callback)iterator).failed(x);
         }
 

jetty-client/src/test/java/org/eclipse/jetty/client/Socks4ProxyTest.java

@@ -18,6 +18,8 @@
 
 package org.eclipse.jetty.client;
 
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
 import java.nio.channels.ServerSocketChannel;
@@ -25,7 +27,14 @@ import java.nio.channels.SocketChannel;
 import java.nio.charset.StandardCharsets;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocket;
 
+import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
+import org.eclipse.jetty.http.HttpScheme;
+import org.eclipse.jetty.io.ClientConnector;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -44,7 +53,13 @@ public class Socks4ProxyTest
         server = ServerSocketChannel.open();
         server.bind(new InetSocketAddress("localhost", 0));
 
-        client = new HttpClient();
+        ClientConnector connector = new ClientConnector();
+        QueuedThreadPool clientThreads = new QueuedThreadPool();
+        clientThreads.setName("client");
+        connector.setExecutor(clientThreads);
+        connector.setSslContextFactory(new SslContextFactory.Client());
+        client = new HttpClient(new HttpClientTransportOverHTTP(connector));
+        client.setExecutor(clientThreads);
         client.start();
     }
 
@@ -61,7 +76,7 @@ public class Socks4ProxyTest
         int proxyPort = server.socket().getLocalPort();
         client.getProxyConfiguration().getProxies().add(new Socks4Proxy("localhost", proxyPort));
 
-        final CountDownLatch latch = new CountDownLatch(1);
+        CountDownLatch latch = new CountDownLatch(1);
 
         byte ip1 = 127;
         byte ip2 = 0;
@@ -111,7 +126,7 @@ public class Socks4ProxyTest
                     "Content-Length: 0\r\n" +
                     "Connection: close\r\n" +
                     "\r\n";
-            channel.write(ByteBuffer.wrap(response.getBytes("UTF-8")));
+            channel.write(ByteBuffer.wrap(response.getBytes(StandardCharsets.UTF_8)));
 
             assertTrue(latch.await(5, TimeUnit.SECONDS));
         }
@@ -123,7 +138,7 @@ public class Socks4ProxyTest
         int proxyPort = server.socket().getLocalPort();
         client.getProxyConfiguration().getProxies().add(new Socks4Proxy("localhost", proxyPort));
 
-        final CountDownLatch latch = new CountDownLatch(1);
+        CountDownLatch latch = new CountDownLatch(1);
 
         String serverHost = "127.0.0.13"; // Test expects an IP address.
         int serverPort = proxyPort + 1; // Any port will do
@@ -169,7 +184,92 @@ public class Socks4ProxyTest
                     "Content-Length: 0\r\n" +
                     "Connection: close\r\n" +
                     "\r\n";
-            channel.write(ByteBuffer.wrap(response.getBytes("UTF-8")));
+            channel.write(ByteBuffer.wrap(response.getBytes(StandardCharsets.UTF_8)));
+
+            assertTrue(latch.await(5, TimeUnit.SECONDS));
+        }
+    }
+
+    @Test
+    public void testSocks4ProxyWithTLSServer() throws Exception
+    {
+        String proxyHost = "localhost";
+        int proxyPort = server.socket().getLocalPort();
+
+        String serverHost = "127.0.0.13"; // Server host different from proxy host.
+        int serverPort = proxyPort + 1; // Any port will do.
+
+        SslContextFactory clientTLS = client.getSslContextFactory();
+        clientTLS.reload(ssl ->
+        {
+            // The client keystore contains the trustedCertEntry for the
+            // self-signed server certificate, so it acts as a truststore.
+            ssl.setTrustStorePath("src/test/resources/client_keystore.jks");
+            ssl.setTrustStorePassword("storepwd");
+            // Disable TLS hostname verification, but
+            // enable application hostname verification.
+            ssl.setEndpointIdentificationAlgorithm(null);
+            // The hostname must be that of the server, not of the proxy.
+            ssl.setHostnameVerifier((hostname, session) -> serverHost.equals(hostname));
+        });
+        client.getProxyConfiguration().getProxies().add(new Socks4Proxy(proxyHost, proxyPort));
+
+        CountDownLatch latch = new CountDownLatch(1);
+        client.newRequest(serverHost, serverPort)
+            .scheme(HttpScheme.HTTPS.asString())
+            .path("/path")
+            .send(result ->
+            {
+                if (result.isSucceeded())
+                    latch.countDown();
+                else
+                    result.getFailure().printStackTrace();
+            });
+
+        try (SocketChannel channel = server.accept())
+        {
+            int socks4MessageLength = 9;
+            ByteBuffer buffer = ByteBuffer.allocate(socks4MessageLength);
+            int read = channel.read(buffer);
+            assertEquals(socks4MessageLength, read);
+
+            // Socks4 response.
+            channel.write(ByteBuffer.wrap(new byte[]{0, 0x5A, 0, 0, 0, 0, 0, 0}));
+
+            // Wrap the socket with TLS.
+            SslContextFactory.Server serverTLS = new SslContextFactory.Server();
+            serverTLS.setKeyStorePath("src/test/resources/keystore.jks");
+            serverTLS.setKeyStorePassword("storepwd");
+            serverTLS.start();
+            SSLContext sslContext = serverTLS.getSslContext();
+            SSLSocket sslSocket = (SSLSocket)sslContext.getSocketFactory().createSocket(channel.socket(), serverHost, serverPort, false);
+            sslSocket.setUseClientMode(false);
+
+            // Read the request.
+            int crlfs = 0;
+            InputStream input = sslSocket.getInputStream();
+            while (true)
+            {
+                read = input.read();
+                if (read < 0)
+                    break;
+                if (read == '\r' || read == '\n')
+                    ++crlfs;
+                else
+                    crlfs = 0;
+                if (crlfs == 4)
+                    break;
+            }
+
+            // Send the response.
+            String response =
+                "HTTP/1.1 200 OK\r\n" +
+                    "Content-Length: 0\r\n" +
+                    "Connection: close\r\n" +
+                    "\r\n";
+            OutputStream output = sslSocket.getOutputStream();
+            output.write(response.getBytes(StandardCharsets.UTF_8));
+            output.flush();
 
             assertTrue(latch.await(5, TimeUnit.SECONDS));
         }

jetty-client/src/test/java/org/eclipse/jetty/client/util/MultiPartContentProviderTest.java

@@ -20,6 +20,7 @@ package org.eclipse.jetty.client.util;
 
 import java.io.BufferedWriter;
 import java.io.ByteArrayInputStream;
+import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -31,10 +32,12 @@ import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.servlet.MultipartConfigElement;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
@@ -435,6 +438,46 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
         assertTrue(responseLatch.await(5, TimeUnit.SECONDS));
     }
 
+    @ParameterizedTest
+    @ArgumentsSource(ScenarioProvider.class)
+    public void testEachPartIsClosed(Scenario scenario) throws Exception
+    {
+        String name1 = "field1";
+        String value1 = "value1";
+        String name2 = "field2";
+        String value2 = "value2";
+        start(scenario, new AbstractMultiPartHandler()
+        {
+            @Override
+            protected void handle(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
+            {
+                Collection<Part> parts = request.getParts();
+                assertEquals(2, parts.size());
+                Iterator<Part> iterator = parts.iterator();
+                Part part1 = iterator.next();
+                assertEquals(name1, part1.getName());
+                assertEquals(value1, IO.toString(part1.getInputStream()));
+                Part part2 = iterator.next();
+                assertEquals(name2, part2.getName());
+                assertEquals(value2, IO.toString(part2.getInputStream()));
+            }
+        });
+
+        AtomicInteger closeCount = new AtomicInteger();
+        MultiPartContentProvider multiPart = new MultiPartContentProvider();
+        multiPart.addFieldPart(name1, new CloseableStringContentProvider(value1, closeCount::incrementAndGet), null);
+        multiPart.addFieldPart(name2, new CloseableStringContentProvider(value2, closeCount::incrementAndGet), null);
+        multiPart.close();
+        ContentResponse response = client.newRequest("localhost", connector.getLocalPort())
+            .scheme(scenario.getScheme())
+            .method(HttpMethod.POST)
+            .content(multiPart)
+            .send();
+
+        assertEquals(200, response.getStatus());
+        assertEquals(2, closeCount.get());
+    }
+
     private abstract static class AbstractMultiPartHandler extends AbstractHandler
     {
         @Override
@@ -448,4 +491,49 @@ public class MultiPartContentProviderTest extends AbstractHttpClientServerTest
 
         protected abstract void handle(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException;
     }
+
+    private static class CloseableStringContentProvider extends StringContentProvider
+    {
+        private final Runnable closeFn;
+
+        private CloseableStringContentProvider(String content, Runnable closeFn)
+        {
+            super(content);
+            this.closeFn = closeFn;
+        }
+
+        @Override
+        public Iterator<ByteBuffer> iterator()
+        {
+            return new CloseableIterator<>(super.iterator());
+        }
+
+        private class CloseableIterator<T> implements Iterator<T>, Closeable
+        {
+            private final Iterator<T> iterator;
+
+            public CloseableIterator(Iterator<T> iterator)
+            {
+                this.iterator = iterator;
+            }
+
+            @Override
+            public boolean hasNext()
+            {
+                return iterator.hasNext();
+            }
+
+            @Override
+            public T next()
+            {
+                return iterator.next();
+            }
+
+            @Override
+            public void close()
+            {
+                closeFn.run();
+            }
+        }
+    }
 }

jetty-documentation/src/main/asciidoc/distribution-guide/http2/enabling-http2.adoc

@@ -27,11 +27,12 @@ A demo Jetty base that supports HTTP/1, HTTPS/1 and deployment from a webapps di
 $ JETTY_BASE=http2-demo
 $ mkdir $JETTY_BASE
 $ cd $JETTY_BASE
-$ java -jar $JETTY_HOME/start.jar --add-to-start=http,https,deploy
+$ java -jar $JETTY_HOME/start.jar --add-to-start=http,https,deploy,test-keystore
 ....
 
 The commands above create a `$JETTY_BASE` directory called `http2-demo`, and initializes the `http,` `https` and `deploy` modules (and their dependencies) to run a typical Jetty Server on port 8080 (for HTTP/1) and 8443 (for HTTPS/1).
-Note that the HTTPS module downloads a demo keystore file with a self signed certificate, which needs to be replaced by a Certificate Authority issued certificate for real deployment.
+Note that the `test-keystore` module downloads a demo keystore file with a self signed certificate, which needs to be replaced by a Certificate Authority issued certificate for real deployment.
+A keystore can also be added by enabling and configuring the `ssl` module.
 
 To add HTTP/2 to this demo base, it is just a matter of enabling the `http2` module with the following command:
 

jetty-documentation/src/main/asciidoc/distribution-guide/jsp/configuring-jsp.adoc

@@ -36,6 +36,12 @@ include::{SRCDIR}/jetty-home/src/main/resources/modules/jsp.mod[]
 Note that the availability of some JSP features may depend on which JSP container implementation you are using.
 Note also that it may not be possible to precompile your JSPs with one container and deploy to the other.
 
+===== Logging
+
+The Apache Jasper logging system is bridged to the jetty logging system.
+Thus, you can enable logging for jsps in the same way you have setup for your webapp.
+For example, assuming you are using Jetty's default StdErr logger, you would enable DEBUG level logging for jsps by adding the system property `-Dorg.apache.jasper.LEVEL=DEBUG` to the command line.
+
 ===== JSPs and Embedding
 
 If you have an embedded setup for your webapp and wish to use JSPs, you will need to ensure that a JSP engine is correctly initialized.
@@ -79,89 +85,91 @@ If you wish to use a different compiler, you will need to configure the `compile
 [cols=",,,",options="header",]
 |=======================================================================
 |init param |Description |Default |`webdefault.xml`
-|classpath |`Classpath used for jsp compilation. Only used if
+
-                org.apache.catalina.jsp_classpath context attribute is not
+|checkInterval |If non-zero and `development` is `false`, background jsp recompilation is enabled. This value is the interval in seconds between background recompile checks.
-                set, which it is in Jetty.` |- |–
+               |0 |–
+|classpath |The classpath is dynamically generated if the context has a URL classloader. The `org.apache.catalina.jsp_classpath`
+context attribute is used to add to the classpath, but if this is not set, this `classpath` configuration item is added to the classpath instead.` |- |–
 
 |classdebuginfo |Include debugging info in class file. |TRUE |–
 
-|checkInterval |Interval in seconds between background recompile checks.
+|compilerClassName |If not set, defaults to the Eclipse jdt compiler. |- |–
-Only relevant if `
-                development=false`. |0 |–
 
-|development |`development=true`, recompilation checks occur on each
+|compiler |Used if the Eclipse jdt compiler cannot be found on the
-request. See also `
+classpath. It is the classname of a compiler that Ant should invoke. |–
-                modificationTestInterval`. |TRUE |–
+|–
+
+|compilerTargetVM |Target vm to compile for. |1.8 |1.8
+
+|compilerSourceVM |Sets source compliance level for the jdt compiler.
+|1.8 |1.8
+
+|development |If `true` recompilation checks occur at the frequency governed by `modificationTestInterval`. |TRUE |–
 
 |displaySourceFragment |Should a source fragment be included in
 exception messages |TRUE |–
 
+|dumpSmap |Dump SMAP JSR45 info to a file. |FALSE |–
+
+|enablePooling |Determines whether tag handler pooling is enabled. |TRUE |–
+
+|engineOptionsClass |Allows specifying the Options class used to
+configure Jasper. If not present, the default EmbeddedServletOptions
+will be used. |- |–
+
 |errorOnUseBeanInvalidClassAttribute |Should Jasper issue an error when
 the value of the class attribute in an useBean action is not a valid
 bean class |TRUE |–
 
-|fork |Should Ant fork its Java compiles of JSP pages? |TRUE |FALSE
+|fork |Only relevant if you use Ant to compile jsps: by default Jetty will use the Eclipse jdt compiler.|TRUE |-
+
+|genStrAsCharArray |Option for generating Strings as char arrays. |FALSE |–
+
+|ieClassId |The class-id value to be sent to Internet Explorer when
+using <jsp:plugin> tags. |clsid:8AD9C840-044E-11D1-B3E9-00805F499D93 |–
+
+|javaEncoding |Pass through the encoding to use for the compilation.
+|UTF8 |–
+
+|jspIdleTimeout |The amount of time in seconds a JSP can be idle before
+it is unloaded. A value of zero or less indicates never unload. |-1 |–
 
 |keepgenerated |Do you want to keep the generated Java files around?
 |TRUE |–
 
-|trimSpaces |Should white spaces between directives or actions be
-trimmed? |FALSE |–
-
-|enablePooling |Determines whether tag handler pooling is enabled. |TRUE
-|–
-
-|engineOptionsClass |Allows specifying the Options class used to
-configure Jasper. If not present, the default EmbeddedServletOptions
-will be used. |–
-
 |mappedFile |Support for mapped Files. Generates a servlet that has a
 print statement per line of the JSP file  |TRUE |–
 
-|suppressSmap |Generation of SMAP info for JSR45 debugging. |FALSE |–
-
-|dumpSmap |Dump SMAP JSR45 info to a file. |FALSE |–
-
-|genStrAsCharArray |Option for generating Strings. |FALSE |–
-
-|ieClassId |The class-id value to be sent to Internet Explorer when
-using <jsp:plugin> tags. |clsid:8AD9C840-044E-11D1-B3E9-00805F499D93 |–
-
 |maxLoadedJsps |The maximum number of JSPs that will be loaded for a web
 application. If more than this number of JSPs are loaded, the least
 recently used JSPs will be unloaded so that the number of JSPs loaded at
 any one time does not exceed this limit. A value of zero or less
 indicates no limit. |-1 |–
 
-|jspIdleTimeout |The amount of time in seconds a JSP can be idle before
-it is unloaded. A value of zero or less indicates never unload. |-1 |–
-
-|scratchDir |Directory where servlets are generated. See |– |–
-
-|compilerClassName |If not set, defaults to the Eclipse jdt compiler. |–
-
-|compiler |Used if the Eclipse jdt compiler cannot be found on the
-classpath. It is the classname of a compiler that Ant should invoke. |–
-|–
-
-|compilerTargetVM |Target vm to compile for. |1.7 |–
-
-|compilerSourceVM |Sets source compliance level for the jdt compiler.
-|1.7 |–
-
-|javaEncoding |Pass through the encoding to use for the compilation.
-|UTF8 |–
-
 |modificationTestInterval |If `development=true`, interval between
 recompilation checks, triggered by a request. |4 |–
 
-|xpoweredBy |Generate an X-Powered-By response header. |FALSE |FALSE
+|quoteAttributeEL | When EL is used in an attribute value on a JSP page, should the rules for quoting of attributes described in JSP.1.6 be applied to the expression
+                 |TRUE |-
 
 |recompileOnFail |If a JSP compilation fails should the
 modificationTestInterval be ignored and the next access trigger a
 re-compilation attempt? Used in development mode only and is disabled by
 default as compilation may be expensive and could lead to excessive
-resource usage. |- |–
+resource usage. |FALSE |–
+
+|scratchDir |Directory where servlets are generated. The default is the value of the context attribute `javax.servlet.context.tempdir`, or the system property `java.io.tmpdir` if the context attribute is not set. |– |–
+
+|strictQuoteEscaping |Should the quote escaping required by section JSP.1.6 of the JSP specification be applied to scriplet expression.
+             |TRUE|-
+
+|suppressSmap |Generation of SMAP info for JSR45 debugging. |FALSE |–
+
+|trimSpaces |Should template text that consists entirely of whitespace be removed from the output (true), replaced with a single space (single) or left unchanged (false)? Note that if a JSP page or tag file specifies a trimDirectiveWhitespaces value of true, that will take precedence over this configuration setting for that page/tag.
+trimmed? |FALSE |–
+
+|xpoweredBy |Generate an X-Powered-By response header. |FALSE |FALSE
+
 |=======================================================================
 
 [[configuring-jsp-for-jetty]]
@@ -171,7 +179,7 @@ The JSP engine has many configuration parameters.
 Some parameters affect only precompilation, and some affect runtime recompilation checking.
 Parameters also differ among the various versions of the JSP engine.
 This page lists the configuration parameters, their meanings, and their default settings.
-Set all parameters on the `org.apache.jasper.servlet.JspServlet` instance defined in the link:#webdefault-xml[`webdefault.xml`] file.
+Set all parameters on the `org.eclipse.jetty.jsp.JettyJspServlet` instance defined in the link:#webdefault-xml[`webdefault.xml`] file.
 
 ____
 [NOTE]
@@ -225,18 +233,10 @@ You can use the entry in link:#webdefault-xml[{$jetty.home}/etc/webdefault.xml]
 ----
  <servlet id="jsp">
      <servlet-name>jsp</servlet-name>
-     <servlet-class>org.apache.jasper.servlet.JspServlet</servlet-class>
+     <servlet-class>org.eclipse.jetty.jsp.JettyJspServlet</servlet-class>
-     <init-param>
-         <param-name>logVerbosityLevel</param-name>
-         <param-value>DEBUG</param-value>
-     </init-param>
-     <init-param>
-         <param-name>fork</param-name>
-         <param-value>>false</param-value>
-     </init-param>
      <init-param>
          <param-name>keepgenerated</param-name>
-         <param-value>>true</param-value>
+         <param-value>true</param-value>
      </init-param>
      ...
 

jetty-fcgi/fcgi-server/src/main/java/org/eclipse/jetty/fcgi/server/HttpChannelOverFCGI.java

@@ -125,6 +125,14 @@ public class HttpChannelOverFCGI extends HttpChannel
         dispatcher.dispatch();
     }
 
+    public boolean onIdleTimeout(Throwable timeout)
+    {
+        boolean handle = getRequest().getHttpInput().onIdleTimeout(timeout);
+        if (handle)
+            execute(this);
+        return !handle;
+    }
+
     private static class Dispatcher implements Runnable
     {
         private final AtomicReference<State> state = new AtomicReference<>(State.IDLE);

jetty-fcgi/fcgi-server/src/main/java/org/eclipse/jetty/fcgi/server/ServerFCGIConnection.java

@@ -105,6 +105,14 @@ public class ServerFCGIConnection extends AbstractConnection
         }
     }
 
+    @Override
+    protected boolean onReadTimeout(Throwable timeout)
+    {
+        return channels.values().stream()
+            .mapToInt(channel -> channel.onIdleTimeout(timeout) ? 0 : 1)
+            .sum() == 0;
+    }
+
     private void parse(ByteBuffer buffer)
     {
         while (buffer.hasRemaining())

jetty-http/src/main/java/org/eclipse/jetty/http/GZIPContentDecoder.java

@@ -37,13 +37,16 @@ import org.eclipse.jetty.util.component.Destroyable;
  */
 public class GZIPContentDecoder implements Destroyable
 {
+    // Unsigned Integer Max == 2^32
+    private static final long UINT_MAX = 0xFFFFFFFFL;
+
     private final List<ByteBuffer> _inflateds = new ArrayList<>();
     private final Inflater _inflater = new Inflater(true);
     private final ByteBufferPool _pool;
     private final int _bufferSize;
     private State _state;
     private int _size;
-    private int _value;
+    private long _value;
     private byte _flags;
     private ByteBuffer _inflated;
 
@@ -375,11 +378,12 @@ public class GZIPContentDecoder implements Destroyable
                     }
                     case ISIZE:
                     {
-                        _value += (currByte & 0xFF) << 8 * _size;
+                        _value = _value | ((currByte & 0xFFL) << (8 * _size));
                         ++_size;
                         if (_size == 4)
                         {
-                            if (_value != _inflater.getBytesWritten())
+                            // RFC 1952: Section 2.3.1; ISIZE is the input size modulo 2^32
+                            if (_value != (_inflater.getBytesWritten() & UINT_MAX))
                                 throw new ZipException("Invalid input size");
 
                             // TODO ByteBuffer result = output == null ? BufferUtil.EMPTY_BUFFER : ByteBuffer.wrap(output);

jetty-http/src/main/java/org/eclipse/jetty/http/HttpURI.java

@@ -608,6 +608,15 @@ public class HttpURI
         return _param;
     }
 
+    public void setParam(String param)
+    {
+        _param = param;
+        if (_path != null && !_path.contains(_param))
+        {
+            _path += ";" + _param;
+        }
+    }
+
     public String getQuery()
     {
         return _query;

jetty-http/src/main/java/org/eclipse/jetty/http/MetaData.java

@@ -250,6 +250,11 @@ public class MetaData implements Iterable<HttpField>
         private String _protocol;
 
         public ConnectRequest(HttpScheme scheme, HostPortHttpField authority, String path, HttpFields fields, String protocol)
+        {
+            this(scheme == null ? null : scheme.asString(), authority, path, fields, protocol);
+        }
+
+        public ConnectRequest(String scheme, HostPortHttpField authority, String path, HttpFields fields, String protocol)
         {
             super(HttpMethod.CONNECT.asString(), scheme, authority, path, HttpVersion.HTTP_2, fields, Long.MIN_VALUE);
             _protocol = protocol;

jetty-http/src/test/java/org/eclipse/jetty/http/GZIPContentDecoderTest.java

@@ -20,6 +20,8 @@ package org.eclipse.jetty.http;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -30,7 +32,11 @@ import org.eclipse.jetty.io.ArrayByteBufferPool;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -351,4 +357,81 @@ public class GZIPContentDecoderTest
         assertTrue(buffer.hasRemaining());
         assertEquals(data2, StandardCharsets.UTF_8.decode(buffer).toString());
     }
+
+    // Signed Integer Max
+    final long INT_MAX = Integer.MAX_VALUE;
+
+    // Unsigned Integer Max == 2^32
+    final long UINT_MAX = 0xFFFFFFFFL;
+
+    @ParameterizedTest
+    @ValueSource(longs = {INT_MAX, INT_MAX + 1, UINT_MAX, UINT_MAX + 1})
+    public void testLargeGzipStream(long origSize) throws IOException
+    {
+        // Size chosen for trade off between speed of I/O vs speed of Gzip
+        final int BUFSIZE = 1024 * 1024;
+
+        // Create a buffer to use over and over again to produce the uncompressed input
+        byte[] cbuf = "0123456789ABCDEFGHIJKLMOPQRSTUVWXYZ".getBytes(StandardCharsets.UTF_8);
+        byte[] buf = new byte[BUFSIZE];
+        for (int off = 0; off < buf.length; )
+        {
+            int len = Math.min(cbuf.length, buf.length - off);
+            System.arraycopy(cbuf, 0, buf, off, len);
+            off += len;
+        }
+
+        GZIPDecoderOutputStream out = new GZIPDecoderOutputStream(new GZIPContentDecoder(BUFSIZE));
+        GZIPOutputStream outputStream = new GZIPOutputStream(out, BUFSIZE);
+
+        for (long bytesLeft = origSize; bytesLeft > 0; )
+        {
+            int len = buf.length;
+            if (bytesLeft < buf.length)
+            {
+                len = (int)bytesLeft;
+            }
+            outputStream.write(buf, 0, len);
+            bytesLeft -= len;
+        }
+
+        // Close GZIPOutputStream to have it generate gzip trailer.
+        // This can cause more writes of unflushed gzip buffers
+        outputStream.close();
+
+        // out.decodedByteCount is only valid after close
+        assertThat("Decoded byte count", out.decodedByteCount, is(origSize));
+    }
+
+    public static class GZIPDecoderOutputStream extends OutputStream
+    {
+        private final GZIPContentDecoder decoder;
+        public long decodedByteCount = 0L;
+
+        public GZIPDecoderOutputStream(GZIPContentDecoder decoder)
+        {
+            this.decoder = decoder;
+        }
+
+        @Override
+        public void write(byte[] b, int off, int len) throws IOException
+        {
+            ByteBuffer buf = ByteBuffer.wrap(b, off, len);
+            while (buf.hasRemaining())
+            {
+                ByteBuffer decoded = decoder.decode(buf);
+                if (decoded.hasRemaining())
+                {
+                    decodedByteCount += decoded.remaining();
+                }
+                decoder.release(decoded);
+            }
+        }
+
+        @Override
+        public void write(int b) throws IOException
+        {
+            write(new byte[]{(byte)b}, 0, 1);
+        }
+    }
 }

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Session.java

@@ -97,6 +97,7 @@ public abstract class HTTP2Session extends ContainerLifeCycle implements ISessio
     private int initialSessionRecvWindow;
     private int writeThreshold;
     private boolean pushEnabled;
+    private boolean connectProtocolEnabled;
     private long idleTime;
     private GoAwayFrame closeFrame;
 
@@ -370,6 +371,14 @@ public abstract class HTTP2Session extends ContainerLifeCycle implements ISessio
                     generator.setMaxHeaderListSize(value);
                     break;
                 }
+                case SettingsFrame.ENABLE_CONNECT_PROTOCOL:
+                {
+                    boolean enabled = value == 1;
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("{} CONNECT protocol for {}", enabled ? "Enabling" : "Disabling", this);
+                    connectProtocolEnabled = enabled;
+                    break;
+                }
                 default:
                 {
                     if (LOG.isDebugEnabled())
@@ -906,6 +915,17 @@ public abstract class HTTP2Session extends ContainerLifeCycle implements ISessio
         return pushEnabled;
     }
 
+    @ManagedAttribute(value = "Whether CONNECT requests supports a protocol", readonly = true)
+    public boolean isConnectProtocolEnabled()
+    {
+        return connectProtocolEnabled;
+    }
+
+    public void setConnectProtocolEnabled(boolean connectProtocolEnabled)
+    {
+        this.connectProtocolEnabled = connectProtocolEnabled;
+    }
+
     /**
      * A typical close by a remote peer involves a GO_AWAY frame followed by TCP FIN.
      * This method is invoked when the TCP FIN is received, or when an exception is

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/HTTP2Stream.java

@@ -372,16 +372,18 @@ public class HTTP2Stream extends IdleTimeout implements IStream, Callback, Dumpa
                 dataProcess = proceed = dataDemand > 0;
             }
         }
-        if (LOG.isDebugEnabled())
-            LOG.debug("{} data processing of {} for {}", initial ? "Starting" : proceed ? "Proceeding" : "Stalling", frame, this);
         if (initial)
         {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Starting data processing of {} for {}", frame, this);
             notifyBeforeData(this);
             try (AutoLock l = lock.lock())
             {
                 dataProcess = proceed = dataDemand > 0;
             }
         }
+        if (LOG.isDebugEnabled())
+            LOG.debug("{} data processing of {} for {}", proceed ? "Proceeding" : "Stalling", frame, this);
         if (proceed)
             processData();
     }

jetty-http2/http2-common/src/main/java/org/eclipse/jetty/http2/frames/SettingsFrame.java

@@ -30,6 +30,7 @@ public class SettingsFrame extends Frame
     public static final int INITIAL_WINDOW_SIZE = 4;
     public static final int MAX_FRAME_SIZE = 5;
     public static final int MAX_HEADER_LIST_SIZE = 6;
+    public static final int ENABLE_CONNECT_PROTOCOL = 8;
 
     private final Map<Integer, Integer> settings;
     private final boolean reply;

jetty-http2/http2-hpack/src/main/java/org/eclipse/jetty/http2/hpack/MetaDataBuilder.java

@@ -177,22 +177,22 @@ public class MetaDataBuilder
                     _contentLength = field.getLongValue();
                     _fields.add(field);
                     break;
-                
+
                 case TE:
                     if ("trailers".equalsIgnoreCase(value))
                         _fields.add(field);
                     else
                         streamException("Unsupported TE value '%s'", value);
                     break;
-            
+
                 case CONNECTION:
                     if ("TE".equalsIgnoreCase(value))
                         _fields.add(field);
                     else
                         streamException("Connection specific field '%s'", header);
-                    break;                
+                    break;
 
-                default:               
+                default:
                     if (name.charAt(0) == ':')
                         streamException("Unknown pseudo header '%s'", name);
                     else
@@ -238,7 +238,7 @@ public class MetaDataBuilder
             _streamException.addSuppressed(new Throwable());
             throw _streamException;
         }
-        
+
         if (_request && _response)
             throw new HpackException.StreamException("Request and Response headers");
 
@@ -268,7 +268,7 @@ public class MetaDataBuilder
                     throw new HpackException.StreamException("No Status");
                 return new MetaData.Response(HttpVersion.HTTP_2, _status, fields, _contentLength);
             }
-                
+
             return new MetaData(HttpVersion.HTTP_2, fields, _contentLength);
         }
         finally

jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpConnectionOverHTTP2.java

@@ -32,7 +32,9 @@ import org.eclipse.jetty.client.HttpConnection;
 import org.eclipse.jetty.client.HttpDestination;
 import org.eclipse.jetty.client.HttpExchange;
 import org.eclipse.jetty.client.HttpRequest;
+import org.eclipse.jetty.client.HttpUpgrader;
 import org.eclipse.jetty.client.SendFailure;
+import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.http2.ErrorCode;
 import org.eclipse.jetty.http2.IStream;
@@ -88,6 +90,18 @@ public class HttpConnectionOverHTTP2 extends HttpConnection implements Sweeper.S
         return send(channel, exchange);
     }
 
+    @Override
+    protected void normalizeRequest(Request request)
+    {
+        super.normalizeRequest(request);
+        if (request instanceof HttpUpgrader.Factory)
+        {
+            HttpUpgrader upgrader = ((HttpUpgrader.Factory)request).newHttpUpgrader(HttpVersion.HTTP_2);
+            ((HttpRequest)request).getConversation().setAttribute(HttpUpgrader.class.getName(), upgrader);
+            upgrader.prepare((HttpRequest)request);
+        }
+    }
+
     protected HttpChannelOverHTTP2 acquireHttpChannel()
     {
         HttpChannelOverHTTP2 channel = idleChannels.poll();

jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpReceiverOverHTTP2.java

@@ -27,10 +27,12 @@ import java.util.Queue;
 import java.util.function.BiFunction;
 
 import org.eclipse.jetty.client.HttpChannel;
+import org.eclipse.jetty.client.HttpConversation;
 import org.eclipse.jetty.client.HttpExchange;
 import org.eclipse.jetty.client.HttpReceiver;
 import org.eclipse.jetty.client.HttpRequest;
 import org.eclipse.jetty.client.HttpResponse;
+import org.eclipse.jetty.client.HttpUpgrader;
 import org.eclipse.jetty.client.api.Request;
 import org.eclipse.jetty.client.api.Response;
 import org.eclipse.jetty.http.HttpField;
@@ -110,7 +112,11 @@ public class HttpReceiverOverHTTP2 extends HttpReceiver implements HTTP2Channel.
                     if (LOG.isDebugEnabled())
                         LOG.debug("Successful HTTP2 tunnel on {} via {}", stream, endPoint);
                     ((IStream)stream).setAttachment(endPoint);
-                    httpRequest.getConversation().setAttribute(EndPoint.class.getName(), endPoint);
+                    HttpConversation conversation = httpRequest.getConversation();
+                    conversation.setAttribute(EndPoint.class.getName(), endPoint);
+                    HttpUpgrader upgrader = (HttpUpgrader)conversation.getAttribute(HttpUpgrader.class.getName());
+                    if (upgrader != null)
+                        upgrader.upgrade(httpResponse, endPoint);
                 }
 
                 if (responseHeaders(exchange))

jetty-http2/http2-http-client-transport/src/main/java/org/eclipse/jetty/http2/client/http/HttpSenderOverHTTP2.java

@@ -59,7 +59,16 @@ public class HttpSenderOverHTTP2 extends HttpSender
         MetaData.Request metaData;
         if (isTunnel)
         {
-            metaData = new MetaData.Request(request.getMethod(), null, new HostPortHttpField(request.getPath()), null, HttpVersion.HTTP_2, request.getHeaders());
+            String upgradeProtocol = request.getUpgradeProtocol();
+            if (upgradeProtocol == null)
+            {
+                metaData = new MetaData.ConnectRequest((String)null, new HostPortHttpField(request.getPath()), null, request.getHeaders(), null);
+            }
+            else
+            {
+                HostPortHttpField authority = new HostPortHttpField(request.getHost(), request.getPort());
+                metaData = new MetaData.ConnectRequest(request.getScheme(), authority, request.getPath(), request.getHeaders(), upgradeProtocol);
+            }
         }
         else
         {

jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/AbstractHTTP2ServerConnectionFactory.java

@@ -60,6 +60,7 @@ public abstract class AbstractHTTP2ServerConnectionFactory extends AbstractConne
     private int maxHeaderBlockFragment = 0;
     private int maxFrameLength = Frame.DEFAULT_MAX_LENGTH;
     private int maxSettingsKeys = SettingsFrame.DEFAULT_MAX_KEYS;
+    private boolean connectProtocolEnabled = true;
     private RateControl.Factory rateControlFactory = new WindowRateControl.Factory(20);
     private FlowControlStrategy.Factory flowControlStrategyFactory = () -> new BufferingFlowControlStrategy(0.5F);
     private long streamIdleTimeout;
@@ -185,6 +186,17 @@ public abstract class AbstractHTTP2ServerConnectionFactory extends AbstractConne
         this.maxSettingsKeys = maxSettingsKeys;
     }
 
+    @ManagedAttribute("Whether CONNECT requests supports a protocol")
+    public boolean isConnectProtocolEnabled()
+    {
+        return connectProtocolEnabled;
+    }
+
+    public void setConnectProtocolEnabled(boolean connectProtocolEnabled)
+    {
+        this.connectProtocolEnabled = connectProtocolEnabled;
+    }
+
     /**
      * @return the factory that creates RateControl objects
      */
@@ -237,6 +249,7 @@ public abstract class AbstractHTTP2ServerConnectionFactory extends AbstractConne
         if (maxConcurrentStreams >= 0)
             settings.put(SettingsFrame.MAX_CONCURRENT_STREAMS, maxConcurrentStreams);
         settings.put(SettingsFrame.MAX_HEADER_LIST_SIZE, getHttpConfiguration().getRequestHeaderSize());
+        settings.put(SettingsFrame.ENABLE_CONNECT_PROTOCOL, isConnectProtocolEnabled() ? 1 : 0);
         return settings;
     }
 
@@ -259,6 +272,7 @@ public abstract class AbstractHTTP2ServerConnectionFactory extends AbstractConne
             session.setStreamIdleTimeout(streamIdleTimeout);
         session.setInitialSessionRecvWindow(getInitialSessionRecvWindow());
         session.setWriteThreshold(getHttpConfiguration().getOutputBufferSize());
+        session.setConnectProtocolEnabled(isConnectProtocolEnabled());
 
         ServerParser parser = newServerParser(connector, session, getRateControlFactory().newRateControl(endPoint));
         parser.setMaxFrameLength(getMaxFrameLength());

jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HTTP2ServerConnectionFactory.java

@@ -102,6 +102,15 @@ public class HTTP2ServerConnectionFactory extends AbstractHTTP2ServerConnectionF
             return this;
         }
 
+        @Override
+        public void onBeforeData(Stream stream)
+        {
+            // Do not notify DATA frame listeners until demanded.
+            // This allows CONNECT requests with pseudo header :protocol
+            // (e.g. WebSocket over HTTP/2) to buffer DATA frames
+            // until they upgrade and are ready to process them.
+        }
+
         @Override
         public boolean onIdleTimeout(Session session)
         {

jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HTTP2ServerSession.java

@@ -108,6 +108,16 @@ public class HTTP2ServerSession extends HTTP2Session implements ServerParser.Lis
                     if (stream != null)
                     {
                         onStreamOpened(stream);
+
+                        if (metaData instanceof MetaData.ConnectRequest)
+                        {
+                            if (!isConnectProtocolEnabled() && ((MetaData.ConnectRequest)metaData).getProtocol() != null)
+                            {
+                                stream.reset(new ResetFrame(streamId, ErrorCode.PROTOCOL_ERROR.code), Callback.NOOP);
+                                return;
+                            }
+                        }
+
                         stream.process(frame, Callback.NOOP);
                         Stream.Listener listener = notifyNewStream(stream, frame);
                         stream.setListener(listener);

jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpChannelOverHTTP2.java

@@ -29,7 +29,6 @@ import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpGenerator;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpHeaderValue;
-import org.eclipse.jetty.http.HttpMethod;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.http.MetaData;
 import org.eclipse.jetty.http.PreEncodedHttpField;
@@ -140,8 +139,13 @@ public class HttpChannelOverHTTP2 extends HttpChannel implements Closeable, Writ
                 onRequestComplete();
             }
 
+            boolean connect = request instanceof MetaData.ConnectRequest;
             _delayedUntilContent = getHttpConfiguration().isDelayDispatchUntilContent() &&
-                    !endStream && !_expect100Continue && !HttpMethod.CONNECT.is(request.getMethod());
+                    !endStream && !_expect100Continue && !connect;
+
+            // Delay the demand of DATA frames for CONNECT with :protocol.
+            if (!connect || request.getProtocol() == null)
+                getStream().demand(1);
 
             if (LOG.isDebugEnabled())
             {

jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java

@@ -318,12 +318,19 @@ public class HttpTransportOverHTTP2 implements HttpTransport
     {
         HttpChannelOverHTTP2 channel = (HttpChannelOverHTTP2)stream.getAttachment();
         Request request = channel.getRequest();
+        if (request.getHttpInput().hasContent())
+            return channel.sendErrorOrAbort("Unexpected content in CONNECT request");
         Connection connection = (Connection)request.getAttribute(UPGRADE_CONNECTION_ATTRIBUTE);
         EndPoint endPoint = connection.getEndPoint();
         endPoint.upgrade(connection);
         stream.setAttachment(endPoint);
-        if (request.getHttpInput().hasContent())
+        // Only now that we have switched the attachment,
-            return channel.sendErrorOrAbort("Unexpected content in CONNECT request");
+        // we can demand DATA frames to process them.
+        stream.demand(1);
+
+        if (LOG.isDebugEnabled())
+            LOG.debug("Upgrading to {}", connection);
+
         return false;
     }
 
@@ -333,6 +340,8 @@ public class HttpTransportOverHTTP2 implements HttpTransport
         Object attachment = stream.getAttachment();
         if (attachment instanceof HttpChannelOverHTTP2)
         {
+            // TODO: we used to "fake" a 101 response to upgrade the endpoint
+            //  but we don't anymore, so this code should be deleted.
             HttpChannelOverHTTP2 channel = (HttpChannelOverHTTP2)attachment;
             if (channel.getResponse().getStatus() == HttpStatus.SWITCHING_PROTOCOLS_101)
             {

jetty-maven-plugin/pom.xml

@@ -119,6 +119,11 @@
     </plugins>
   </build>
   <dependencies>
+    <dependency>
+      <groupId>org.apache.maven.shared</groupId>
+      <artifactId>maven-artifact-transfer</artifactId>
+      <version>0.11.0</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.maven</groupId>
       <artifactId>maven-plugin-api</artifactId>
@@ -140,11 +145,6 @@
       <artifactId>maven-plugin-annotations</artifactId>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.maven.shared</groupId>
-      <artifactId>maven-artifact-transfer</artifactId>
-      <version>0.9.1</version>
-    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-util</artifactId>

jetty-maven-plugin/src/it/jetty-start-mojo-it/jetty-simple-webapp/pom.xml

@@ -69,6 +69,7 @@
           </includes>
           <systemPropertyVariables>
             <jetty.port.file>${jetty.port.file}</jetty.port.file>
+            <context.path>/setbycontextxml</context.path>
             <pingServlet>true</pingServlet>
             <helloServlet>true</helloServlet>
             <contentCheck>Counter accessed 1 times.</contentCheck>
@@ -91,6 +92,7 @@
               <goal>start</goal>
             </goals>
             <configuration>
+              <contextXml>${basedir}/src/config/context.xml</contextXml>
               <systemProperties>
                   <jetty.port.file>${jetty.port.file}</jetty.port.file>
                   <jetty.deployMode>EMBED</jetty.deployMode>

jetty-maven-plugin/src/it/jetty-start-mojo-it/jetty-simple-webapp/src/config/context.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure_9_3.dtd">
+
+<Configure class="org.eclipse.jetty.webapp.WebAppContext">
+
+  <Set name="contextPath">/setbycontextxml</Set>
+
+</Configure>

jetty-maven-plugin/src/main/java/org/eclipse/jetty/maven/plugin/AbstractWebAppMojo.java

@@ -50,7 +50,7 @@ import org.apache.maven.plugin.descriptor.PluginDescriptor;
 import org.apache.maven.plugins.annotations.Component;
 import org.apache.maven.plugins.annotations.Parameter;
 import org.apache.maven.project.MavenProject;
-import org.apache.maven.shared.artifact.resolve.ArtifactResolver;
+import org.apache.maven.shared.transfer.artifact.resolve.ArtifactResolver;
 import org.codehaus.plexus.util.StringUtils;
 import org.eclipse.jetty.maven.plugin.utils.MavenProjectHelper;
 import org.eclipse.jetty.security.LoginService;

jetty-maven-plugin/src/main/java/org/eclipse/jetty/maven/plugin/utils/MavenProjectHelper.java

@@ -36,9 +36,9 @@ import org.apache.maven.execution.MavenSession;
 import org.apache.maven.project.DefaultProjectBuildingRequest;
 import org.apache.maven.project.MavenProject;
 import org.apache.maven.project.ProjectBuildingRequest;
-import org.apache.maven.shared.artifact.DefaultArtifactCoordinate;
+import org.apache.maven.shared.transfer.artifact.DefaultArtifactCoordinate;
-import org.apache.maven.shared.artifact.resolve.ArtifactResolver;
+import org.apache.maven.shared.transfer.artifact.resolve.ArtifactResolver;
-import org.apache.maven.shared.artifact.resolve.ArtifactResolverException;
+import org.apache.maven.shared.transfer.artifact.resolve.ArtifactResolverException;
 import org.eclipse.jetty.maven.plugin.OverlayManager;
 import org.eclipse.jetty.maven.plugin.WarPluginInfo;
 

jetty-maven-plugin/src/test/java/org/eclipse/jetty/maven/plugin/it/IntegrationTestGetContent.java

@@ -43,6 +43,10 @@ public class IntegrationTestGetContent
     {
         int port = getPort();
         assertTrue(port > 0);
+        String contextPath = getContextPath();
+        if (contextPath.endsWith("/"))
+            contextPath = contextPath.substring(0, contextPath.lastIndexOf('/'));
+
         HttpClient httpClient = new HttpClient();
         try
         {
@@ -50,16 +54,16 @@ public class IntegrationTestGetContent
 
             if (Boolean.getBoolean("helloServlet"))
             {
-                String response = httpClient.GET("http://localhost:" + port + "/hello?name=beer").getContentAsString();
+                String response = httpClient.GET("http://localhost:" + port + contextPath + "/hello?name=beer").getContentAsString();
                 assertEquals("Hello beer", response.trim(), "it test " + System.getProperty("maven.it.name"));
-                response = httpClient.GET("http://localhost:" + port + "/hello?name=foo").getContentAsString();
+                response = httpClient.GET("http://localhost:" + port + contextPath + "/hello?name=foo").getContentAsString();
                 assertEquals("Hello foo", response.trim(), "it test " + System.getProperty("maven.it.name"));
                 System.out.println("helloServlet");
             }
             if (Boolean.getBoolean("pingServlet"))
             {
                 System.out.println("pingServlet");
-                String response = httpClient.GET("http://localhost:" + port + "/ping?name=beer").getContentAsString();
+                String response = httpClient.GET("http://localhost:" + port + contextPath + "/ping?name=beer").getContentAsString();
                 assertEquals("pong beer", response.trim(), "it test " + System.getProperty("maven.it.name"));
                 System.out.println("pingServlet ok");
             }
@@ -67,7 +71,7 @@ public class IntegrationTestGetContent
             String pathToCheck = System.getProperty("pathToCheck");
             if (StringUtils.isNotBlank(contentCheck))
             {
-                String url = "http://localhost:" + port;
+                String url = "http://localhost:" + port + contextPath;
                 if (pathToCheck != null)
                 {
                     url += pathToCheck;
@@ -79,9 +83,9 @@ public class IntegrationTestGetContent
             }
             if (Boolean.getBoolean("helloTestServlet"))
             {
-                String response = httpClient.GET("http://localhost:" + port + "/testhello?name=beer").getContentAsString();
+                String response = httpClient.GET("http://localhost:" + port + contextPath + "/testhello?name=beer").getContentAsString();
                 assertEquals("Hello from test beer", response.trim(), "it test " + System.getProperty("maven.it.name"));
-                response = httpClient.GET("http://localhost:" + port + "/testhello?name=foo").getContentAsString();
+                response = httpClient.GET("http://localhost:" + port + contextPath + "/testhello?name=foo").getContentAsString();
                 assertEquals("Hello from test foo", response.trim(), "it test " + System.getProperty("maven.it.name"));
                 System.out.println("helloServlet");
             }
@@ -92,6 +96,11 @@ public class IntegrationTestGetContent
         }
     }
 
+    public static String getContextPath()
+    {
+        return System.getProperty("context.path", "/");
+    }
+
     public static int getPort()
         throws Exception
     {

jetty-osgi/jetty-osgi-boot/src/main/java/org/eclipse/jetty/osgi/boot/internal/serverfactory/DefaultJettyAtJettyHomeHelper.java

@@ -26,6 +26,8 @@ import java.util.Dictionary;
 import java.util.Enumeration;
 import java.util.Hashtable;
 import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 import java.util.StringTokenizer;
 
 import org.eclipse.jetty.osgi.boot.JettyBootstrapActivator;
@@ -148,9 +150,8 @@ public class DefaultJettyAtJettyHomeHelper
             LOG.warn("No default jetty created.");
             return null;
         }
-
+        
-        //configure the server here rather than letting the JettyServerServiceTracker do it, because we want to be able to
+        //resolve the jetty xml config files
-        //configure the ThreadPool, which can only be done via the constructor, ie from within the xml configuration processing
         List<URL> configURLs = jettyHomeDir != null ? getJettyConfigurationURLs(jettyHomeDir) : getJettyConfigurationURLs(jettyHomeBundle, properties);
 
         LOG.info("Configuring the default jetty server with {}", configURLs);
@@ -174,14 +175,36 @@ public class DefaultJettyAtJettyHomeHelper
             }
             Thread.currentThread().setContextClassLoader(cl);
 
-            // these properties usually are the ones passed to this type of
+            //the default server name
-            // configuration.
             properties.put(OSGiServerConstants.MANAGED_JETTY_SERVER_NAME, OSGiServerConstants.MANAGED_JETTY_SERVER_DEFAULT_NAME);
-            Util.setProperty(properties, OSGiServerConstants.JETTY_HOST, System.getProperty(OSGiServerConstants.JETTY_HOST, System.getProperty("jetty.host")));
+            
-            Util.setProperty(properties, OSGiServerConstants.JETTY_PORT, System.getProperty(OSGiServerConstants.JETTY_PORT, System.getProperty("jetty.port")));
+            //Always set home and base
-            Util.setProperty(properties, OSGiServerConstants.JETTY_PORT_SSL, System.getProperty(OSGiServerConstants.JETTY_PORT_SSL, System.getProperty("ssl.port")));
             Util.setProperty(properties, OSGiServerConstants.JETTY_HOME, home);
             Util.setProperty(properties, OSGiServerConstants.JETTY_BASE, base);
+            
+            // copy all system properties starting with "jetty." to service properties for the jetty server service.
+            // these will be used as xml configuration properties.
+            for (Map.Entry<Object, Object> prop : System.getProperties().entrySet())
+            {
+                if (prop.getKey() instanceof String)
+                {
+                    String skey = (String)prop.getKey();
+                    //never copy the jetty xml config files into the properties as we pass them explicitly into
+                    //the call to configure, also we set home and base explicitly
+                    if (OSGiServerConstants.MANAGED_JETTY_XML_CONFIG_URLS.equals(skey) ||
+                        OSGiServerConstants.JETTY_HOME.equals(skey) ||
+                        OSGiServerConstants.JETTY_BASE.equals(skey))
+                        continue;
+                    
+                    if (skey.startsWith("jetty."))
+                    {
+                        Util.setProperty(properties, skey, prop.getValue());
+                    }
+                }
+            }
+
+            //configure the server here rather than letting the JettyServerServiceTracker do it, because we want to be able to
+            //configure the ThreadPool, which can only be done via the constructor, ie from within the xml configuration processing
             Server server = ServerInstanceWrapper.configure(null, configURLs, properties);
 
             //Register the default Server instance as an OSGi service.

jetty-osgi/test-jetty-osgi/src/test/java/org/eclipse/jetty/osgi/test/TestJettyOSGiBootWithAnnotations.java

@@ -18,6 +18,7 @@
 
 package org.eclipse.jetty.osgi.test;
 
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import javax.inject.Inject;
@@ -25,6 +26,8 @@ import javax.inject.Inject;
 import org.eclipse.jetty.client.HttpClient;
 import org.eclipse.jetty.client.api.ContentResponse;
 import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.util.MultiPartContentProvider;
+import org.eclipse.jetty.client.util.StringContentProvider;
 import org.eclipse.jetty.http.HttpStatus;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -131,6 +134,11 @@ public class TestJettyOSGiBootWithAnnotations
             assertEquals("Response status code", HttpStatus.OK_200, response.getStatus());
             content = response.getContentAsString();
             TestOSGiUtil.assertContains("Response contents", content, "<h1>FRAGMENT</h1>");
+            MultiPartContentProvider multiPart = new MultiPartContentProvider();
+            multiPart.addFieldPart("field", new StringContentProvider("foo"), null);
+            response = client.newRequest("http://127.0.0.1:" + port + "/multi").method("POST")
+                .content(multiPart).send();
+            assertEquals(HttpStatus.OK_200, response.getStatus());
         }
         finally
         {

jetty-rewrite/src/main/java/org/eclipse/jetty/rewrite/handler/RuleContainer.java

@@ -22,8 +22,10 @@ import java.io.IOException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.eclipse.jetty.http.HttpURI;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.util.ArrayUtil;
+import org.eclipse.jetty.util.StringUtil;
 import org.eclipse.jetty.util.URIUtil;
 import org.eclipse.jetty.util.component.Dumpable;
 import org.eclipse.jetty.util.log.Log;
@@ -185,7 +187,18 @@ public class RuleContainer extends Rule implements Dumpable
                     if (rule instanceof Rule.ApplyURI)
                         ((Rule.ApplyURI)rule).applyURI(baseRequest, baseRequest.getRequestURI(), encoded);
                     else
-                        baseRequest.setURIPathQuery(encoded);
+                    {
+                        String uriPathQuery = encoded;
+                        HttpURI baseUri = baseRequest.getHttpURI();
+                        // Copy path params from original URI if present
+                        if ((baseUri != null) && StringUtil.isNotBlank(baseUri.getParam()))
+                        {
+                            HttpURI uri = new HttpURI(uriPathQuery);
+                            uri.setParam(baseUri.getParam());
+                            uriPathQuery = uri.toString();
+                        }
+                        baseRequest.setURIPathQuery(uriPathQuery);
+                    }
                 }
 
                 if (_rewritePathInfo)

jetty-rewrite/src/test/java/org/eclipse/jetty/rewrite/handler/CookiePatternRuleTest.java

@@ -37,7 +37,6 @@ import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.eclipse.jetty.server.handler.HandlerList;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -48,7 +47,6 @@ import static org.junit.jupiter.api.Assertions.fail;
 
 public class CookiePatternRuleTest
 {
-
     private Server server;
     private LocalConnector localConnector;
 
@@ -150,7 +148,6 @@ public class CookiePatternRuleTest
     }
 
     @Test
-    @Disabled("See #2675 for details") // TODO: needs to be fixed in RuleContainer
     public void testUrlParameter() throws Exception
     {
         CookiePatternRule rule = new CookiePatternRule();
@@ -170,7 +167,6 @@ public class CookiePatternRuleTest
         HttpTester.Response response = HttpTester.parseResponse(rawResponse);
 
         String responseContent = response.getContent();
-        System.out.println(responseContent);
         assertResponseContentLine(responseContent, "baseRequest.requestUri=", "/other;fruit=apple");
 
         // verify

jetty-server/src/main/java/org/eclipse/jetty/server/HttpChannel.java

@@ -1228,10 +1228,7 @@ public class HttpChannel implements Runnable, HttpOutput.Interceptor
             if (_length > 0)
                 _combinedListener.onResponseContent(_request, _content);
             if (_complete && _state.completeResponse())
-            {
-                _response.getHttpOutput().closed();
                 _combinedListener.onResponseEnd(_request);
-            }
             super.succeeded();
         }
 
@@ -1255,7 +1252,6 @@ public class HttpChannel implements Runnable, HttpOutput.Interceptor
                     @Override
                     public void failed(Throwable th)
                     {
-                        _response.getHttpOutput().closed();
                         abort(x);
                         super.failed(x);
                     }

jetty-server/src/main/java/org/eclipse/jetty/server/HttpChannelOverHttp.java

@@ -412,6 +412,13 @@ public class HttpChannelOverHttp extends HttpChannel implements HttpParser.Reque
         return true;
     }
 
+    @Override
+    protected boolean checkAndPrepareUpgrade()
+    {
+        // TODO: move the code from HttpConnection.upgrade() here?
+        return false;
+    }
+
     /**
      * <p>Attempts to perform an HTTP/1.1 upgrade.</p>
      * <p>The upgrade looks up a {@link ConnectionFactory.Upgrading} from the connector

jetty-server/src/main/java/org/eclipse/jetty/server/HttpChannelState.java

@@ -28,6 +28,7 @@ import javax.servlet.UnavailableException;
 
 import org.eclipse.jetty.http.BadMessageException;
 import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.io.QuietException;
 import org.eclipse.jetty.server.handler.ContextHandler;
 import org.eclipse.jetty.server.handler.ContextHandler.Context;
 import org.eclipse.jetty.server.handler.ErrorHandler;
@@ -406,8 +407,6 @@ public class HttpChannelState
      */
     protected Action unhandle()
     {
-        boolean readInterested = false;
-
         synchronized (this)
         {
             if (LOG.isDebugEnabled())
@@ -737,8 +736,10 @@ public class HttpChannelState
             }
             else
             {
-                LOG.warn(failure.toString());
+                if (!(failure instanceof QuietException))
-                LOG.debug(failure);
+                    LOG.warn(failure.toString());
+                if (LOG.isDebugEnabled())
+                    LOG.debug(failure);
             }
         }
 
@@ -969,6 +970,9 @@ public class HttpChannelState
             }
         }
 
+        // release any aggregate buffer from a closing flush
+        _channel.getResponse().getHttpOutput().closed();
+
         if (event != null)
         {
             cancelTimeout(event);
@@ -1341,7 +1345,7 @@ public class HttpChannelState
      * but that a handling thread may need to produce (fill/parse)
      * it.  Typically called by the async read success callback.
      *
-     * @return <code>true</code> if more content may be available
+     * @return {@code true} if more content may be available
      */
     public boolean onReadPossible()
     {
@@ -1373,7 +1377,7 @@ public class HttpChannelState
      * Called to signal that a read has read -1.
      * Will wake if the read was called while in ASYNC_WAIT state
      *
-     * @return <code>true</code> if woken
+     * @return {@code true} if woken
      */
     public boolean onReadEof()
     {

jetty-server/src/main/java/org/eclipse/jetty/server/HttpConfiguration.java

@@ -315,7 +315,7 @@ public class HttpConfiguration implements Dumpable
     }
 
     /**
-     * @param delay if true, delay the application dispatch until content is available (default false)
+     * @param delay if true, delays the application dispatch until content is available (defaults to true)
      */
     public void setDelayDispatchUntilContent(boolean delay)
     {

jetty-server/src/main/java/org/eclipse/jetty/server/HttpInput.java

@@ -732,22 +732,29 @@ public class HttpInput extends ServletInputStream implements Runnable
 
                 _listener = Objects.requireNonNull(readListener);
 
-                Content content = produceNextContext();
+                if (isError())
-                if (content != null)
                 {
-                    _state = ASYNC;
                     woken = _channelState.onReadReady();
                 }
-                else if (_state == EOF)
-                {
-                    _state = AEOF;
-                    woken = _channelState.onReadEof();
-                }
                 else
                 {
-                    _state = ASYNC;
+                    Content content = produceNextContext();
-                    _channelState.onReadUnready();
+                    if (content != null)
-                    _waitingForContent = true;
+                    {
+                        _state = ASYNC;
+                        woken = _channelState.onReadReady();
+                    }
+                    else if (_state == EOF)
+                    {
+                        _state = AEOF;
+                        woken = _channelState.onReadEof();
+                    }
+                    else
+                    {
+                        _state = ASYNC;
+                        _channelState.onReadUnready();
+                        _waitingForContent = true;
+                    }
                 }
             }
         }

jetty-server/src/main/java/org/eclipse/jetty/server/HttpOutput.java

@@ -364,13 +364,6 @@ public class HttpOutput extends ServletOutputStream implements Runnable
             State state = _state.get();
             switch (state)
             {
-                case CLOSING:
-                {
-                    if (!_state.compareAndSet(state, State.CLOSED))
-                        break;
-                    releaseBuffer();
-                    return;
-                }
                 case CLOSED:
                 {
                     return;
@@ -378,7 +371,12 @@ public class HttpOutput extends ServletOutputStream implements Runnable
                 case UNREADY:
                 {
                     if (_state.compareAndSet(state, State.ERROR))
-                        _writeListener.onError(_onError == null ? new EofException("Async closed") : _onError);
+                    {
+                        if (_onError == null)
+                            _onError = new EofException("Async closed");
+                        releaseBuffer();
+                        return;
+                    }
                     break;
                 }
                 default:
@@ -484,6 +482,8 @@ public class HttpOutput extends ServletOutputStream implements Runnable
         {
             write(content, complete, blocker);
             blocker.block();
+            if (complete)
+                closed();
         }
         catch (Exception failure)
         {

jetty-server/src/main/java/org/eclipse/jetty/server/MultiPartCleanerListener.java

@@ -1,63 +0,0 @@
-//
-//  ========================================================================
-//  Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd.
-//  ------------------------------------------------------------------------
-//  All rights reserved. This program and the accompanying materials
-//  are made available under the terms of the Eclipse Public License v1.0
-//  and Apache License v2.0 which accompanies this distribution.
-//
-//      The Eclipse Public License is available at
-//      http://www.eclipse.org/legal/epl-v10.html
-//
-//      The Apache License v2.0 is available at
-//      http://www.opensource.org/licenses/apache2.0.php
-//
-//  You may elect to redistribute this code under either of these licenses.
-//  ========================================================================
-//
-
-package org.eclipse.jetty.server;
-
-import javax.servlet.ServletRequestEvent;
-import javax.servlet.ServletRequestListener;
-
-import org.eclipse.jetty.server.handler.ContextHandler;
-
-public class MultiPartCleanerListener implements ServletRequestListener
-{
-    public static final MultiPartCleanerListener INSTANCE = new MultiPartCleanerListener();
-
-    protected MultiPartCleanerListener()
-    {
-    }
-
-    @Override
-    public void requestDestroyed(ServletRequestEvent sre)
-    {
-        //Clean up any tmp files created by MultiPartInputStream
-        MultiParts parts = (MultiParts)sre.getServletRequest().getAttribute(Request.__MULTIPARTS);
-        if (parts != null)
-        {
-            ContextHandler.Context context = parts.getContext();
-
-            //Only do the cleanup if we are exiting from the context in which a servlet parsed the multipart files
-            if (context == sre.getServletContext())
-            {
-                try
-                {
-                    parts.close();
-                }
-                catch (Throwable e)
-                {
-                    sre.getServletContext().log("Errors deleting multipart tmp files", e);
-                }
-            }
-        }
-    }
-
-    @Override
-    public void requestInitialized(ServletRequestEvent sre)
-    {
-        //nothing to do, multipart config set up by ServletHolder.handle()
-    }
-}

jetty-server/src/main/java/org/eclipse/jetty/server/MultiParts.java

@@ -1,88 +0,0 @@
-//
-//  ========================================================================
-//  Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd.
-//  ------------------------------------------------------------------------
-//  All rights reserved. This program and the accompanying materials
-//  are made available under the terms of the Eclipse Public License v1.0
-//  and Apache License v2.0 which accompanies this distribution.
-//
-//      The Eclipse Public License is available at
-//      http://www.eclipse.org/legal/epl-v10.html
-//
-//      The Apache License v2.0 is available at
-//      http://www.opensource.org/licenses/apache2.0.php
-//
-//  You may elect to redistribute this code under either of these licenses.
-//  ========================================================================
-//
-
-package org.eclipse.jetty.server;
-
-import java.io.Closeable;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collection;
-import javax.servlet.MultipartConfigElement;
-import javax.servlet.http.Part;
-
-import org.eclipse.jetty.http.MultiPartFormInputStream;
-import org.eclipse.jetty.server.handler.ContextHandler;
-import org.eclipse.jetty.server.handler.ContextHandler.Context;
-
-/*
- * Used to switch between the old and new implementation of MultiPart Form InputStream Parsing.
- * The new implementation is preferred will be used as default unless specified otherwise constructor.
- */
-public interface MultiParts extends Closeable
-{
-    Collection<Part> getParts() throws IOException;
-
-    Part getPart(String name) throws IOException;
-
-    boolean isEmpty();
-
-    ContextHandler.Context getContext();
-
-    class MultiPartsHttpParser implements MultiParts
-    {
-        private final MultiPartFormInputStream _httpParser;
-        private final ContextHandler.Context _context;
-
-        public MultiPartsHttpParser(InputStream in, String contentType, MultipartConfigElement config, File contextTmpDir, Request request) throws IOException
-        {
-            _httpParser = new MultiPartFormInputStream(in, contentType, config, contextTmpDir);
-            _context = request.getContext();
-        }
-
-        @Override
-        public Collection<Part> getParts() throws IOException
-        {
-            return _httpParser.getParts();
-        }
-
-        @Override
-        public Part getPart(String name) throws IOException
-        {
-            return _httpParser.getPart(name);
-        }
-
-        @Override
-        public void close()
-        {
-            _httpParser.deleteParts();
-        }
-
-        @Override
-        public boolean isEmpty()
-        {
-            return _httpParser.isEmpty();
-        }
-
-        @Override
-        public Context getContext()
-        {
-            return _context;
-        }
-    }
-}

jetty-server/src/main/java/org/eclipse/jetty/server/Request.java

@@ -81,6 +81,7 @@ import org.eclipse.jetty.http.HttpURI;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.http.MetaData;
 import org.eclipse.jetty.http.MimeTypes;
+import org.eclipse.jetty.http.MultiPartFormInputStream;
 import org.eclipse.jetty.http.pathmap.PathSpec;
 import org.eclipse.jetty.http.pathmap.ServletPathSpec;
 import org.eclipse.jetty.io.RuntimeIOException;
@@ -141,7 +142,6 @@ import org.eclipse.jetty.util.log.Logger;
 public class Request implements HttpServletRequest
 {
     public static final String __MULTIPART_CONFIG_ELEMENT = "org.eclipse.jetty.multipartConfig";
-    public static final String __MULTIPARTS = "org.eclipse.jetty.multiParts";
 
     private static final Logger LOG = Log.getLogger(Request.class);
     private static final Collection<Locale> __defaultLocale = Collections.singleton(Locale.getDefault());
@@ -228,7 +228,7 @@ public class Request implements HttpServletRequest
     private HttpSession _session;
     private SessionHandler _sessionHandler;
     private long _timeStamp;
-    private MultiParts _multiParts; //if the request is a multi-part mime
+    private MultiPartFormInputStream _multiParts; //if the request is a multi-part mime
     private AsyncContextState _async;
     private List<Session> _sessions; //list of sessions used during lifetime of request
 
@@ -1504,6 +1504,19 @@ public class Request implements HttpServletRequest
             for (Session s:_sessions)
                 leaveSession(s);
         }
+
+        //Clean up any tmp files created by MultiPartInputStream
+        if (_multiParts != null)
+        {
+            try
+            {
+                _multiParts.deleteParts();
+            }
+            catch (Throwable e)
+            {
+                LOG.warn("Errors deleting multipart tmp files", e);
+            }
+        }
     }
     
     /**
@@ -2299,15 +2312,12 @@ public class Request implements HttpServletRequest
     {
         String contentType = getContentType();
         if (contentType == null || !MimeTypes.Type.MULTIPART_FORM_DATA.is(HttpFields.valueParameters(contentType, null)))
-            throw new ServletException("Content-Type != multipart/form-data");
+            throw new ServletException("Unsupported Content-Type [" + contentType + "], expected [multipart/form-data]");
         return getParts(null);
     }
 
     private Collection<Part> getParts(MultiMap<String> params) throws IOException
     {
-        if (_multiParts == null)
-            _multiParts = (MultiParts)getAttribute(__MULTIPARTS);
-
         if (_multiParts == null)
         {
             MultipartConfigElement config = (MultipartConfigElement)getAttribute(__MULTIPART_CONFIG_ELEMENT);
@@ -2315,7 +2325,6 @@ public class Request implements HttpServletRequest
                 throw new IllegalStateException("No multipart config for servlet");
 
             _multiParts = newMultiParts(config);
-            setAttribute(__MULTIPARTS, _multiParts);
             Collection<Part> parts = _multiParts.getParts();
 
             String formCharset = null;
@@ -2377,10 +2386,10 @@ public class Request implements HttpServletRequest
         return _multiParts.getParts();
     }
 
-    private MultiParts newMultiParts(MultipartConfigElement config) throws IOException
+    private MultiPartFormInputStream newMultiParts(MultipartConfigElement config) throws IOException
     {
-        return new MultiParts.MultiPartsHttpParser(getInputStream(), getContentType(), config,
+        return new MultiPartFormInputStream(getInputStream(), getContentType(), config,
-            (_context != null ? (File)_context.getAttribute("javax.servlet.context.tempdir") : null), this);
+            (_context != null ? (File)_context.getAttribute("javax.servlet.context.tempdir") : null));
     }
 
     @Override

jetty-server/src/main/java/org/eclipse/jetty/server/ResponseWriter.java

@@ -128,10 +128,13 @@ public class ResponseWriter extends PrintWriter
     private void isOpen() throws IOException
     {
         if (_ioException != null)
-            throw new RuntimeIOException(_ioException);
+            throw _ioException;
 
         if (_isClosed)
-            throw new EofException("Stream closed");
+        {
+            _ioException = new EofException("Stream closed");
+            throw _ioException;
+        }
     }
 
     @Override

jetty-server/src/main/java/org/eclipse/jetty/server/handler/ErrorHandler.java

@@ -21,6 +21,7 @@ package org.eclipse.jetty.server.handler;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.io.Writer;
 import java.nio.BufferOverflowException;
 import java.nio.ByteBuffer;
@@ -474,24 +475,24 @@ public class ErrorHandler extends AbstractHandler
         writer.append(json.entrySet().stream()
                 .map(e -> QuotedStringTokenizer.quote(e.getKey()) +
                         ":" +
-                        QuotedStringTokenizer.quote((e.getValue())))
+                    QuotedStringTokenizer.quote(StringUtil.sanitizeXmlString((e.getValue()))))
                 .collect(Collectors.joining(",\n", "{\n", "\n}")));
-
-
     }
 
     protected void writeErrorPageStacks(HttpServletRequest request, Writer writer)
         throws IOException
     {
         Throwable th = (Throwable)request.getAttribute(RequestDispatcher.ERROR_EXCEPTION);
-        if (_showStacks && th != null)
+        if (th != null)
         {
-            PrintWriter pw = writer instanceof PrintWriter ? (PrintWriter)writer : new PrintWriter(writer);
+            writer.write("<h3>Caused by:</h3><pre>");
-            pw.write("<pre>");
+            // You have to pre-generate and then use #write(writer, String)
-            while (th != null)
+            try (StringWriter sw = new StringWriter();
+                 PrintWriter pw = new PrintWriter(sw))
             {
                 th.printStackTrace(pw);
-                th = th.getCause();
+                pw.flush();
+                write(writer, sw.getBuffer().toString()); // sanitize
             }
             writer.write("</pre>\n");
         }

jetty-server/src/main/java/org/eclipse/jetty/server/session/Session.java

@@ -378,7 +378,7 @@ public class Session implements SessionHandler.SessionIf
         try 
         {
             HttpSessionEvent event = new HttpSessionEvent(this);
-        for (String name : _sessionData.getKeys())
+            for (String name : _sessionData.getKeys())
             {
                 Object value = _sessionData.getAttribute(name);
                 if (value instanceof HttpSessionActivationListener)

jetty-server/src/test/java/org/eclipse/jetty/server/AsyncCompletionTest.java

@@ -33,6 +33,12 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Stream;
+import javax.servlet.AsyncContext;
+import javax.servlet.ServletException;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.WriteListener;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.http.HttpCompliance;
 import org.eclipse.jetty.http.tools.HttpTester;
@@ -41,6 +47,7 @@ import org.eclipse.jetty.io.Connection;
 import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.io.ManagedSelector;
 import org.eclipse.jetty.io.SocketChannelEndPoint;
+import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.thread.Scheduler;
 import org.hamcrest.Matchers;
@@ -89,7 +96,7 @@ public class AsyncCompletionTest extends HttpServerTestFixture
                 _delay.get(10, TimeUnit.SECONDS);
                 getCallback().succeeded();
             }
-            catch(Throwable th)
+            catch (Throwable th)
             {
                 th.printStackTrace();
                 getCallback().failed(th);
@@ -97,7 +104,6 @@ public class AsyncCompletionTest extends HttpServerTestFixture
         }
     }
 
-
     @BeforeEach
     public void init() throws Exception
     {
@@ -153,7 +159,7 @@ public class AsyncCompletionTest extends HttpServerTestFixture
         @Override
         public void onCompleted()
         {
-            COMPLETE.compareAndSet(false,true);
+            COMPLETE.compareAndSet(false, true);
             super.onCompleted();
         }
     }
@@ -163,7 +169,8 @@ public class AsyncCompletionTest extends HttpServerTestFixture
     {
         List<Object[]> tests = new ArrayList<>();
         tests.add(new Object[]{new HelloWorldHandler(), 200, "Hello world"});
-        tests.add(new Object[]{new SendErrorHandler(499,"Test async sendError"), 499, "Test async sendError"});
+        tests.add(new Object[]{new SendErrorHandler(499, "Test async sendError"), 499, "Test async sendError"});
+        tests.add(new Object[]{new AsyncReadyCompleteHandler(), 200, AsyncReadyCompleteHandler.data});
         return tests.stream().map(Arguments::of);
     }
 
@@ -197,7 +204,7 @@ public class AsyncCompletionTest extends HttpServerTestFixture
 
             // wait for threads to return to base level
             long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
-            while(_threadPool.getBusyThreads() != base)
+            while (_threadPool.getBusyThreads() != base)
             {
                 if (System.nanoTime() > end)
                     throw new TimeoutException();
@@ -210,7 +217,7 @@ public class AsyncCompletionTest extends HttpServerTestFixture
             // proceed with the completion
             delay.proceed();
 
-            while(!COMPLETE.get())
+            while (!COMPLETE.get())
             {
                 if (System.nanoTime() > end)
                     throw new TimeoutException();
@@ -218,4 +225,46 @@ public class AsyncCompletionTest extends HttpServerTestFixture
             }
         }
     }
+
+    private static class AsyncReadyCompleteHandler extends AbstractHandler
+    {
+        static String data = "Now is the time for all good men to come to the aid of the party";
+
+        @Override
+        public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
+        {
+            AsyncContext context = request.startAsync();
+            ServletOutputStream out = response.getOutputStream();
+            out.setWriteListener(new WriteListener()
+            {
+                byte[] bytes = data.getBytes(StandardCharsets.ISO_8859_1);
+
+                @Override
+                public void onWritePossible() throws IOException
+                {
+                    while (out.isReady())
+                    {
+                        if (bytes != null)
+                        {
+                            response.setContentType("text/plain");
+                            response.setContentLength(bytes.length);
+                            out.write(bytes);
+                            bytes = null;
+                        }
+                        else
+                        {
+                            context.complete();
+                            return;
+                        }
+                    }
+                }
+
+                @Override
+                public void onError(Throwable t)
+                {
+                    t.printStackTrace();
+                }
+            });
+        }
+    }
 }

jetty-server/src/test/java/org/eclipse/jetty/server/ErrorHandlerTest.java

@@ -19,8 +19,9 @@
 package org.eclipse.jetty.server;
 
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.Map;
-
+import java.util.Set;
 import javax.servlet.DispatcherType;
 import javax.servlet.RequestDispatcher;
 import javax.servlet.ServletException;
@@ -28,30 +29,38 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.http.BadMessageException;
-import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.tools.HttpTester;
 import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.eclipse.jetty.util.ajax.JSON;
+import org.eclipse.jetty.util.log.StacklessLogging;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.startsWith;
+import static org.hamcrest.Matchers.nullValue;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class ErrorHandlerTest
 {
+    static StacklessLogging stacklessLogging;
     static Server server;
     static LocalConnector connector;
 
     @BeforeAll
     public static void before() throws Exception
     {
+        stacklessLogging = new StacklessLogging(HttpChannel.class);
         server = new Server();
         connector = new LocalConnector(server);
         server.addConnector(connector);
@@ -64,7 +73,7 @@ public class ErrorHandlerTest
                 if (baseRequest.getDispatcherType() == DispatcherType.ERROR)
                 {
                     baseRequest.setHandled(true);
-                    response.sendError(((Integer)request.getAttribute(RequestDispatcher.ERROR_STATUS_CODE)).intValue());
+                    response.sendError((Integer)request.getAttribute(RequestDispatcher.ERROR_STATUS_CODE));
                     return;
                 }
 
@@ -78,7 +87,40 @@ public class ErrorHandlerTest
 
                 if (target.startsWith("/badmessage/"))
                 {
-                    throw new ServletException(new BadMessageException(Integer.parseInt(target.substring(12))));
+                    int code = Integer.parseInt(target.substring(target.lastIndexOf('/') + 1));
+                    throw new ServletException(new BadMessageException(code));
+                }
+
+                // produce an exception with an JSON formatted cause message
+                if (target.startsWith("/jsonmessage/"))
+                {
+                    String message = "\"}, \"glossary\": {\n \"title\": \"example\"\n }\n {\"";
+                    throw new ServletException(new RuntimeException(message));
+                }
+
+                // produce an exception with an XML cause message
+                if (target.startsWith("/xmlmessage/"))
+                {
+                    String message =
+                        "<!DOCTYPE glossary PUBLIC \"-//OASIS//DTD DocBook V3.1//EN\">\n" +
+                            " <glossary>\n" +
+                            "  <title>example glossary</title>\n" +
+                            " </glossary>";
+                    throw new ServletException(new RuntimeException(message));
+                }
+
+                // produce an exception with an HTML cause message
+                if (target.startsWith("/htmlmessage/"))
+                {
+                    String message = "<hr/><script>alert(42)</script>%3Cscript%3E";
+                    throw new ServletException(new RuntimeException(message));
+                }
+
+                // produce an exception with a UTF-8 cause message
+                if (target.startsWith("/utf8message/"))
+                {
+                    String message = "Euro is &euro; and \u20AC and %E2%82%AC";
+                    throw new ServletException(new RuntimeException(message));
                 }
             }
         });
@@ -89,193 +131,238 @@ public class ErrorHandlerTest
     public static void after() throws Exception
     {
         server.stop();
+        stacklessLogging.close();
     }
 
     @Test
     public void test404NoAccept() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Host: Localhost\r\n" +
                 "\r\n");
 
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
-        assertThat(response, not(containsString("Content-Length: 0")));
+
-        assertThat(response, containsString("Content-Type: text/html;charset=iso-8859-1"));
+        assertThat("Response status code", response.getStatus(), is(404));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html;charset=ISO-8859-1"));
+
+        assertContent(response);
     }
 
     @Test
     public void test404EmptyAccept() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Accept: \r\n" +
                 "Host: Localhost\r\n" +
                 "\r\n");
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
-        assertThat(response, containsString("Content-Length: 0"));
+
-        assertThat(response, not(containsString("Content-Type")));
+        assertThat("Response status code", response.getStatus(), is(404));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), is(0));
+        assertThat("Response Content-Type", response.getField(HttpHeader.CONTENT_TYPE), is(nullValue()));
     }
 
     @Test
     public void test404UnAccept() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Accept: text/*;q=0\r\n" +
                 "Host: Localhost\r\n" +
                 "\r\n");
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
 
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        assertThat("Response status code", response.getStatus(), is(404));
-        assertThat(response, containsString("Content-Length: 0"));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), is(0));
-        assertThat(response, not(containsString("Content-Type")));
+        assertThat("Response Content-Type", response.getField(HttpHeader.CONTENT_TYPE), is(nullValue()));
     }
 
     @Test
     public void test404AllAccept() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Host: Localhost\r\n" +
                 "Accept: */*\r\n" +
                 "\r\n");
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
 
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        assertThat("Response status code", response.getStatus(), is(404));
-        assertThat(response, not(containsString("Content-Length: 0")));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
-        assertThat(response, containsString("Content-Type: text/html;charset=iso-8859-1"));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html;charset=ISO-8859-1"));
+
+        assertContent(response);
     }
 
     @Test
     public void test404HtmlAccept() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Host: Localhost\r\n" +
                 "Accept: text/html\r\n" +
                 "\r\n");
 
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
-        assertThat(response, not(containsString("Content-Length: 0")));
+
-        assertThat(response, containsString("Content-Type: text/html;charset=iso-8859-1"));
+        assertThat("Response status code", response.getStatus(), is(404));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html;charset=ISO-8859-1"));
+
+        assertContent(response);
     }
 
     @Test
     public void testMoreSpecificAccept() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Host: Localhost\r\n" +
                 "Accept: text/html, some/other;specific=true\r\n" +
                 "\r\n");
 
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
-        assertThat(response, not(containsString("Content-Length: 0")));
+
-        assertThat(response, containsString("Content-Type: text/html;charset=iso-8859-1"));
+        assertThat("Response status code", response.getStatus(), is(404));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html;charset=ISO-8859-1"));
+
+        assertContent(response);
     }
 
     @Test
     public void test404HtmlAcceptAnyCharset() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Host: Localhost\r\n" +
                 "Accept: text/html\r\n" +
                 "Accept-Charset: *\r\n" +
                 "\r\n");
 
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
-        assertThat(response, not(containsString("Content-Length: 0")));
+
-        assertThat(response, containsString("Content-Type: text/html;charset=utf-8"));
+        assertThat("Response status code", response.getStatus(), is(404));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html;charset=UTF-8"));
+
+        assertContent(response);
     }
 
     @Test
     public void test404HtmlAcceptUtf8Charset() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Host: Localhost\r\n" +
                 "Accept: text/html\r\n" +
                 "Accept-Charset: utf-8\r\n" +
                 "\r\n");
 
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
-        assertThat(response, not(containsString("Content-Length: 0")));
+
-        assertThat(response, containsString("Content-Type: text/html;charset=utf-8"));
+        assertThat("Response status code", response.getStatus(), is(404));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html;charset=UTF-8"));
+
+        assertContent(response);
     }
 
     @Test
     public void test404HtmlAcceptNotUtf8Charset() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Host: Localhost\r\n" +
                 "Accept: text/html\r\n" +
                 "Accept-Charset: utf-8;q=0\r\n" +
                 "\r\n");
 
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
-        assertThat(response, not(containsString("Content-Length: 0")));
+
-        assertThat(response, containsString("Content-Type: text/html;charset=iso-8859-1"));
+        assertThat("Response status code", response.getStatus(), is(404));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html;charset=ISO-8859-1"));
+
+        assertContent(response);
     }
 
     @Test
     public void test404HtmlAcceptNotUtf8UnknownCharset() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Host: Localhost\r\n" +
                 "Accept: text/html\r\n" +
                 "Accept-Charset: utf-8;q=0,unknown\r\n" +
                 "\r\n");
 
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
-        assertThat(response, containsString("Content-Length: 0"));
+
-        assertThat(response, not(containsString("Content-Type")));
+        assertThat("Response status code", response.getStatus(), is(404));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), is(0));
+        assertThat("Response Content-Type", response.getField(HttpHeader.CONTENT_TYPE), is(nullValue()));
     }
 
     @Test
     public void test404HtmlAcceptUnknownUtf8Charset() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Host: Localhost\r\n" +
                 "Accept: text/html\r\n" +
                 "Accept-Charset: utf-8;q=0.1,unknown\r\n" +
                 "\r\n");
 
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
-        assertThat(response, not(containsString("Content-Length: 0")));
+
-        assertThat(response, containsString("Content-Type: text/html;charset=utf-8"));
+        assertThat("Response status code", response.getStatus(), is(404));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html;charset=UTF-8"));
+
+        assertContent(response);
     }
 
     @Test
     public void test404PreferHtml() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Host: Localhost\r\n" +
                 "Accept: text/html;q=1.0,text/json;q=0.5,*/*\r\n" +
                 "Accept-Charset: *\r\n" +
                 "\r\n");
 
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
-        assertThat(response, not(containsString("Content-Length: 0")));
+
-        assertThat(response, containsString("Content-Type: text/html;charset=utf-8"));
+        assertThat("Response status code", response.getStatus(), is(404));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html;charset=UTF-8"));
+
+        assertContent(response);
     }
 
     @Test
     public void test404PreferJson() throws Exception
     {
-        String response = connector.getResponse(
+        String rawResponse = connector.getResponse(
             "GET / HTTP/1.1\r\n" +
                 "Host: Localhost\r\n" +
                 "Accept: text/html;q=0.5,text/json;q=1.0,*/*\r\n" +
                 "Accept-Charset: *\r\n" +
                 "\r\n");
 
-        assertThat(response, startsWith("HTTP/1.1 404 "));
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
-        assertThat(response, not(containsString("Content-Length: 0")));
+
-        assertThat(response, containsString("Content-Type: text/json"));
+        assertThat("Response status code", response.getStatus(), is(404));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/json"));
+
+        assertContent(response);
     }
 
     @Test
@@ -286,12 +373,14 @@ public class ErrorHandlerTest
                 "Host: Localhost\r\n" +
                 "Accept: text/plain\r\n" +
                 "\r\n");
+
         HttpTester.Response response = HttpTester.parseResponse(rawResponse);
 
         assertThat("Response status code", response.getStatus(), is(404));
-        HttpField contentType = response.getField(HttpHeader.CONTENT_TYPE);
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
-        assertThat("Response Content-Type", contentType, is(notNullValue()));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/plain"));
-        assertThat("Response Content-Type value", contentType.getValue(), not(containsString("null")));
+
+        assertContent(response);
     }
 
     @Test
@@ -301,29 +390,173 @@ public class ErrorHandlerTest
             "GET /badmessage/444 HTTP/1.1\r\n" +
                 "Host: Localhost\r\n" +
                 "\r\n");
+
         HttpTester.Response response = HttpTester.parseResponse(rawResponse);
 
         assertThat("Response status code", response.getStatus(), is(444));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html;charset=ISO-8859-1"));
+
+        assertContent(response);
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+        "/jsonmessage/",
+        "/xmlmessage/",
+        "/htmlmessage/",
+        "/utf8message/",
+    })
+    public void testComplexCauseMessageNoAcceptHeader(String path) throws Exception
+    {
+        String rawResponse = connector.getResponse(
+            "GET " + path + " HTTP/1.1\r\n" +
+                "Host: Localhost\r\n" +
+                "\r\n");
+
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
+
+        assertThat("Response status code", response.getStatus(), is(500));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html;charset=ISO-8859-1"));
+
+        String content = assertContent(response);
+
+        if (path.startsWith("/utf8"))
+        {
+            // we are Not expecting UTF-8 output, look for mangled ISO-8859-1 version
+            assertThat("content", content, containsString("Euro is &amp;euro; and ? and %E2%82%AC"));
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+        "/jsonmessage/",
+        "/xmlmessage/",
+        "/htmlmessage/",
+        "/utf8message/",
+    })
+    public void testComplexCauseMessageAcceptUtf8Header(String path) throws Exception
+    {
+        String rawResponse = connector.getResponse(
+            "GET " + path + " HTTP/1.1\r\n" +
+                "Host: Localhost\r\n" +
+                "Accept: text/html\r\n" +
+                "Accept-Charset: utf-8\r\n" +
+                "\r\n");
+
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
+
+        assertThat("Response status code", response.getStatus(), is(500));
+        assertThat("Response Content-Length", response.getField(HttpHeader.CONTENT_LENGTH).getIntValue(), greaterThan(0));
+        assertThat("Response Content-Type", response.get(HttpHeader.CONTENT_TYPE), containsString("text/html;charset=UTF-8"));
+
+        String content = assertContent(response);
+
+        if (path.startsWith("/utf8"))
+        {
+            // we are Not expecting UTF-8 output, look for mangled ISO-8859-1 version
+            assertThat("content", content, containsString("Euro is &amp;euro; and \u20AC and %E2%82%AC"));
+        }
+    }
+
+    private String assertContent(HttpTester.Response response)
+    {
+        String contentType = response.get(HttpHeader.CONTENT_TYPE);
+        String content = response.getContent();
+
+        if (contentType.contains("text/html"))
+        {
+            assertThat(content, not(containsString("<script>")));
+            assertThat(content, not(containsString("<glossary>")));
+            assertThat(content, not(containsString("<!DOCTYPE>")));
+            assertThat(content, not(containsString("&euro;")));
+        }
+        else if (contentType.contains("text/json"))
+        {
+            Map jo = (Map)JSON.parse(response.getContent());
+
+            Set<String> acceptableKeyNames = new HashSet<>();
+            acceptableKeyNames.add("url");
+            acceptableKeyNames.add("status");
+            acceptableKeyNames.add("message");
+            acceptableKeyNames.add("servlet");
+            acceptableKeyNames.add("cause0");
+            acceptableKeyNames.add("cause1");
+            acceptableKeyNames.add("cause2");
+
+            for (Object key : jo.keySet())
+            {
+                String keyStr = (String)key;
+                assertTrue(acceptableKeyNames.contains(keyStr), "Unexpected Key [" + keyStr + "]");
+
+                Object value = jo.get(key);
+                assertThat("Unexpected value type (" + value.getClass().getName() + ")",
+                    value, instanceOf(String.class));
+            }
+
+            assertThat("url field", jo.get("url"), is(notNullValue()));
+            String expectedStatus = String.valueOf(response.getStatus());
+            assertThat("status field", jo.get("status"), is(expectedStatus));
+            String message = (String)jo.get("message");
+            assertThat("message field", message, is(notNullValue()));
+            assertThat("message field", message, anyOf(
+                not(containsString("<")),
+                not(containsString(">"))));
+        }
+        else if (contentType.contains("text/plain"))
+        {
+            assertThat(content, containsString("STATUS: " + response.getStatus()));
+        }
+        else
+        {
+            System.out.println("Not checked Content-Type: " + contentType);
+            System.out.println(content);
+        }
+
+        return content;
     }
 
     @Test
     public void testJsonResponse() throws Exception
     {
         String rawResponse = connector.getResponse(
-                "GET /badmessage/444 HTTP/1.1\r\n" +
+            "GET /badmessage/444 HTTP/1.1\r\n" +
-                        "Host: Localhost\r\n" +
+                "Host: Localhost\r\n" +
-                        "Accept: text/json\r\n" +
+                "Accept: text/json\r\n" +
-                        "\r\n");
+                "\r\n");
+
         HttpTester.Response response = HttpTester.parseResponse(rawResponse);
 
         assertThat("Response status code", response.getStatus(), is(444));
 
-        System.out.println("response:" + response.getContent());
+        assertContent(response);
+    }
 
-        Map<Object,Object> jo = (Map) JSON.parse(response.getContent());
+    @ParameterizedTest
+    @ValueSource(strings = {
+        "/jsonmessage/",
+        "/xmlmessage/",
+        "/htmlmessage/",
+        "/utf8message/",
+    })
+    public void testJsonResponseWorse(String path) throws Exception
+    {
+        String rawResponse = connector.getResponse(
+            "GET " + path + " HTTP/1.1\r\n" +
+                "Host: Localhost\r\n" +
+                "Accept: text/json\r\n" +
+                "\r\n");
 
-        assertThat("url field null", jo.get("url"), is(notNullValue()));
+        HttpTester.Response response = HttpTester.parseResponse(rawResponse);
-        assertThat("status field null", jo.get("status"), is(notNullValue()));
+        assertThat("Response status code", response.getStatus(), is(500));
-        assertThat("message field null", jo.get("message"), is(notNullValue()));
+
+        String content = assertContent(response);
+
+        if (path.startsWith("/utf8"))
+        {
+            // we are expecting UTF-8 output, look for it.
+            assertThat("content", content, containsString("Euro is &amp;euro; and \u20AC and %E2%82%AC"));
+        }
     }
 }

jetty-server/src/test/java/org/eclipse/jetty/server/RequestTest.java

@@ -38,7 +38,6 @@ import javax.servlet.DispatcherType;
 import javax.servlet.MultipartConfigElement;
 import javax.servlet.ServletException;
 import javax.servlet.ServletInputStream;
-import javax.servlet.ServletRequestEvent;
 import javax.servlet.http.Cookie;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -341,28 +340,13 @@ public class RequestTest
         testTmpDir.deleteOnExit();
         assertTrue(testTmpDir.list().length == 0);
 
+        // We should have two tmp files after parsing the multipart form.
+        RequestTester tester = (request, response) -> testTmpDir.list().length == 2;
+
         ContextHandler contextHandler = new ContextHandler();
         contextHandler.setContextPath("/foo");
         contextHandler.setResourceBase(".");
-        contextHandler.setHandler(new MultiPartRequestHandler(testTmpDir));
+        contextHandler.setHandler(new MultiPartRequestHandler(testTmpDir, tester));
-        contextHandler.addEventListener(new MultiPartCleanerListener()
-        {
-
-            @Override
-            public void requestDestroyed(ServletRequestEvent sre)
-            {
-                MultiParts m = (MultiParts)sre.getServletRequest().getAttribute(Request.__MULTIPARTS);
-                assertNotNull(m);
-                ContextHandler.Context c = m.getContext();
-                assertNotNull(c);
-                assertTrue(c == sre.getServletContext());
-                assertTrue(!m.isEmpty());
-                assertTrue(testTmpDir.list().length == 2);
-                super.requestDestroyed(sre);
-                String[] files = testTmpDir.list();
-                assertTrue(files.length == 0);
-            }
-        });
         _server.stop();
         _server.setHandler(contextHandler);
         _server.start();
@@ -387,51 +371,16 @@ public class RequestTest
             multipart;
 
         String responses = _connector.getResponse(request);
-        //System.err.println(responses);
         assertTrue(responses.startsWith("HTTP/1.1 200"));
-    }
 
-    @Test
+        // We know the previous request has completed if another request can be processed.
-    public void testHttpMultiPart() throws Exception
+        String cleanupRequest = "GET /foo/cleanup HTTP/1.1\r\n" +
-    {
-        final File testTmpDir = File.createTempFile("reqtest", null);
-        if (testTmpDir.exists())
-            testTmpDir.delete();
-        testTmpDir.mkdir();
-        testTmpDir.deleteOnExit();
-        assertTrue(testTmpDir.list().length == 0);
-
-        ContextHandler contextHandler = new ContextHandler();
-        contextHandler.setContextPath("/foo");
-        contextHandler.setResourceBase(".");
-        contextHandler.setHandler(new MultiPartRequestHandler(testTmpDir));
-
-        _server.stop();
-        _server.setHandler(contextHandler);
-        _server.start();
-
-        String multipart = "      --AaB03x\r" +
-            "content-disposition: form-data; name=\"field1\"\r" +
-            "\r" +
-            "Joe Blow\r" +
-            "--AaB03x\r" +
-            "content-disposition: form-data; name=\"stuff\"; filename=\"foo.upload\"\r" +
-            "Content-Type: text/plain;charset=ISO-8859-1\r" +
-            "\r" +
-            "000000000000000000000000000000000000000000000000000\r" +
-            "--AaB03x--\r";
-
-        String request = "GET /foo/x.html HTTP/1.1\r\n" +
             "Host: whatever\r\n" +
-            "Content-Type: multipart/form-data; boundary=\"AaB03x\"\r\n" +
-            "Content-Length: " + multipart.getBytes().length + "\r\n" +
             "Connection: close\r\n" +
-            "\r\n" +
+            "\r\n";
-            multipart;
+        String cleanupResponse = _connector.getResponse(cleanupRequest);
-
+        assertTrue(cleanupResponse.startsWith("HTTP/1.1 200"));
-        String responses = _connector.getResponse(request);
+        assertThat(testTmpDir.list().length, is(0));
-        //System.err.println(responses);
-        assertThat(responses, Matchers.startsWith("HTTP/1.1 500"));
     }
 
     @Test
@@ -449,22 +398,6 @@ public class RequestTest
         contextHandler.setContextPath("/foo");
         contextHandler.setResourceBase(".");
         contextHandler.setHandler(new BadMultiPartRequestHandler(testTmpDir));
-        contextHandler.addEventListener(new MultiPartCleanerListener()
-        {
-
-            @Override
-            public void requestDestroyed(ServletRequestEvent sre)
-            {
-                MultiParts m = (MultiParts)sre.getServletRequest().getAttribute(Request.__MULTIPARTS);
-                assertNotNull(m);
-                ContextHandler.Context c = m.getContext();
-                assertNotNull(c);
-                assertTrue(c == sre.getServletContext());
-                super.requestDestroyed(sre);
-                String[] files = testTmpDir.list();
-                assertTrue(files.length == 0);
-            }
-        });
         _server.stop();
         _server.setHandler(contextHandler);
         _server.start();
@@ -494,6 +427,15 @@ public class RequestTest
             //System.err.println(responses);
             assertTrue(responses.startsWith("HTTP/1.1 500"));
         }
+
+        // We know the previous request has completed if another request can be processed.
+        String cleanupRequest = "GET /foo/cleanup HTTP/1.1\r\n" +
+            "Host: whatever\r\n" +
+            "Connection: close\r\n" +
+            "\r\n";
+        String cleanupResponse = _connector.getResponse(cleanupRequest);
+        assertTrue(cleanupResponse.startsWith("HTTP/1.1 200"));
+        assertThat(testTmpDir.list().length, is(0));
     }
 
     @Test
@@ -1818,20 +1760,27 @@ public class RequestTest
 
     private class MultiPartRequestHandler extends AbstractHandler
     {
+        RequestTester checker;
         File tmpDir;
 
-        public MultiPartRequestHandler(File tmpDir)
+        public MultiPartRequestHandler(File tmpDir, RequestTester checker)
         {
             this.tmpDir = tmpDir;
+            this.checker = checker;
         }
 
         @Override
         public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
         {
             ((Request)request).setHandled(true);
+            if ("/cleanup".equals(target))
+            {
+                response.setStatus(200);
+                return;
+            }
+
             try
             {
-
                 MultipartConfigElement mpce = new MultipartConfigElement(tmpDir.getAbsolutePath(), -1, -1, 2);
                 request.setAttribute(Request.__MULTIPART_CONFIG_ELEMENT, mpce);
 
@@ -1850,6 +1799,9 @@ public class RequestTest
                         response.addHeader("Violation", v);
                     }
                 }
+
+                if (checker != null && !checker.check(request, response))
+                    response.sendError(500);
             }
             catch (IllegalStateException e)
             {
@@ -1877,6 +1829,12 @@ public class RequestTest
         public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException
         {
             ((Request)request).setHandled(true);
+            if ("/cleanup".equals(target))
+            {
+                response.setStatus(200);
+                return;
+            }
+
             try
             {
 

jetty-server/src/test/java/org/eclipse/jetty/server/ResponseTest.java

@@ -48,13 +48,11 @@ import org.eclipse.jetty.http.HttpCookie;
 import org.eclipse.jetty.http.HttpField;
 import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
-import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.http.HttpURI;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.http.MetaData;
 import org.eclipse.jetty.io.AbstractEndPoint;
 import org.eclipse.jetty.io.ByteArrayEndPoint;
-import org.eclipse.jetty.io.RuntimeIOException;
 import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.eclipse.jetty.server.handler.ContextHandler;
 import org.eclipse.jetty.server.handler.ErrorHandler;
@@ -702,7 +700,7 @@ public class ResponseTest
     }
 
     @Test
-    public void testWriteRuntimeIOException() throws Exception
+    public void testWriteCheckError() throws Exception
     {
         Response response = getResponse();
 
@@ -716,8 +714,8 @@ public class ResponseTest
         writer.println("test");
         assertTrue(writer.checkError());
 
-        RuntimeIOException e = assertThrows(RuntimeIOException.class, () -> writer.println("test"));
+        writer.println("test"); // this should not cause an Exception
-        assertEquals(cause, e.getCause());
+        assertTrue(writer.checkError());
     }
 
     @Test

jetty-servlet/src/main/java/org/eclipse/jetty/servlet/ServletHolder.java

@@ -49,7 +49,6 @@ import javax.servlet.http.HttpServletResponse;
 
 import org.eclipse.jetty.security.IdentityService;
 import org.eclipse.jetty.security.RunAsToken;
-import org.eclipse.jetty.server.MultiPartCleanerListener;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.UserIdentity;
 import org.eclipse.jetty.server.handler.ContextHandler;
@@ -470,21 +469,16 @@ public class ServletHolder extends Holder<Servlet> implements UserIdentity.Scope
     public Servlet getServlet()
         throws ServletException
     {
-        Servlet servlet = _servlet;
+        synchronized (this)
-        if (servlet == null)
         {
-            synchronized (this)
+            if (_servlet == null && isRunning())
             {
-                servlet = _servlet;
+                if (getHeldClass() != null)
-                if (servlet == null && isRunning())
+                    initServlet();
-                {
-                    if (getHeldClass() != null)
-                        initServlet();
-                    servlet = _servlet;
-                }
             }
         }
-        return servlet;
+
+        return _servlet;
     }
 
     /**
@@ -592,8 +586,6 @@ public class ServletHolder extends Holder<Servlet> implements UserIdentity.Scope
             else if (_forcedPath != null)
                 detectJspContainer();
 
-            initMultiPart();
-
             if (LOG.isDebugEnabled())
                 LOG.debug("Servlet.init {} for {}", _servlet, getName());
             _servlet.init(_config);
@@ -626,7 +618,7 @@ public class ServletHolder extends Holder<Servlet> implements UserIdentity.Scope
         ContextHandler ch = ContextHandler.getContextHandler(getServletHandler().getServletContext());
 
         /* Set the webapp's classpath for Jasper */
-        ch.setAttribute("org.apache.catalina.jspgetHeldClass()path", ch.getClassPath());
+        ch.setAttribute("org.apache.catalina.jsp_classpath", ch.getClassPath());
 
         /* Set up other classpath attribute */
         if ("?".equals(getInitParameter("classpath")))
@@ -652,29 +644,6 @@ public class ServletHolder extends Holder<Servlet> implements UserIdentity.Scope
             scratch.mkdir();
     }
 
-    /**
-     * Register a ServletRequestListener that will ensure tmp multipart
-     * files are deleted when the request goes out of scope.
-     *
-     * @throws Exception if unable to init the multipart
-     */
-    protected void initMultiPart() throws Exception
-    {
-        //if this servlet can handle multipart requests, ensure tmp files will be
-        //cleaned up correctly
-        if (((Registration)getRegistration()).getMultipartConfig() != null)
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("multipart cleanup listener added for {}", this);
-
-            //Register a listener to delete tmp files that are created as a result of this
-            //servlet calling Request.getPart() or Request.getParts()
-            ContextHandler ch = ContextHandler.getContextHandler(getServletHandler().getServletContext());
-            if (!ch.getEventListeners().contains(MultiPartCleanerListener.INSTANCE))
-                ch.addEventListener(MultiPartCleanerListener.INSTANCE);
-        }
-    }
-
     @Override
     public ContextHandler getContextHandler()
     {

jetty-servlet/src/test/java/org/eclipse/jetty/servlet/InitServletTest.java

@@ -0,0 +1,132 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.servlet;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.api.Result;
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.server.NetworkConnector;
+import org.eclipse.jetty.server.Server;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class InitServletTest
+{
+    public static class DemoServlet extends HttpServlet
+    {
+        private static final long INIT_SLEEP = 2000;
+        private final AtomicInteger initCount = new AtomicInteger();
+
+        @Override
+        public void init() throws ServletException
+        {
+            super.init();
+            try
+            {
+                // Make the initialization last a little while.
+                // Other requests must wait.
+                Thread.sleep(INIT_SLEEP);
+            }
+            catch (InterruptedException e)
+            {
+                throw new ServletException(e);
+            }
+            initCount.incrementAndGet();
+        }
+
+        @Override
+        public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException
+        {
+            // Check that the init() method has been totally finished (by another request)
+            // before the servlet service() method is called.
+            if (initCount.get() != 1)
+                resp.sendError(500, "Servlet not initialized!");
+        }
+    }
+
+    private static class AsyncResponseListener implements Response.CompleteListener
+    {
+        private final AtomicInteger index = new AtomicInteger();
+        private final CountDownLatch resultsLatch;
+        private final int[] results;
+
+        public AsyncResponseListener(CountDownLatch resultsLatch, int[] results)
+        {
+            this.resultsLatch = resultsLatch;
+            this.results = results;
+        }
+
+        public void onComplete(Result result)
+        {
+            results[index.getAndIncrement()] = result.getResponse().getStatus();
+            resultsLatch.countDown();
+        }
+    }
+
+    @Test
+    public void testServletInitialization() throws Exception
+    {
+        Server server = new Server(0);
+        ServletContextHandler context = new ServletContextHandler(server, "/");
+        server.setHandler(context);
+        // Add a lazily instantiated servlet.
+        context.addServlet(new ServletHolder(DemoServlet.class), "/*");
+        HttpClient client = new HttpClient();
+        server.addBean(client);
+        server.start();
+        try
+        {
+            int port = ((NetworkConnector)server.getConnectors()[0]).getLocalPort();
+
+            // Expect 2 responses
+            CountDownLatch resultsLatch = new CountDownLatch(2);
+            int[] results = new int[2];
+            AsyncResponseListener l = new AsyncResponseListener(resultsLatch, results);
+
+            // Req1: should initialize servlet.
+            client.newRequest("http://localhost:" + port + "/r1").send(l);
+
+            // Need to give 1st request a head start before request2.
+            Thread.sleep(DemoServlet.INIT_SLEEP / 4);
+
+            // Req2: should see servlet fully initialized by request1.
+            client.newRequest("http://localhost:" + port + "/r2").send(l);
+
+            assertTrue(resultsLatch.await(DemoServlet.INIT_SLEEP * 2, TimeUnit.MILLISECONDS));
+            assertEquals(HttpStatus.OK_200, results[0]);
+            assertEquals(HttpStatus.OK_200, results[1]);
+        }
+        finally
+        {
+            server.stop();
+        }
+    }
+}

jetty-servlets/src/main/java/org/eclipse/jetty/servlets/QoSFilter.java

@@ -216,6 +216,8 @@ public class QoSFilter implements Filter
         {
             if (accepted)
             {
+                _passes.release();
+
                 for (int p = _queues.length - 1; p >= 0; --p)
                 {
                     AsyncContext asyncContext = _queues[p].poll();
@@ -225,13 +227,20 @@ public class QoSFilter implements Filter
                         Boolean suspended = (Boolean)candidate.getAttribute(_suspended);
                         if (Boolean.TRUE.equals(suspended))
                         {
-                            candidate.setAttribute(_resumed, Boolean.TRUE);
+                            try
-                            asyncContext.dispatch();
+                            {  
-                            break;
+                                candidate.setAttribute(_resumed, Boolean.TRUE);
+                                asyncContext.dispatch();
+                                break;
+                            }
+                            catch (IllegalStateException x)
+                            {
+                                LOG.warn(x);
+                                continue;
+                            }
                         }
                     }
                 }
-                _passes.release();
             }
         }
     }
@@ -368,7 +377,8 @@ public class QoSFilter implements Filter
             // redispatched again at the end of the filtering.
             AsyncContext asyncContext = event.getAsyncContext();
             _queues[priority].remove(asyncContext);
-            asyncContext.dispatch();
+            ((HttpServletResponse)event.getSuppliedResponse()).sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+            asyncContext.complete();
         }
 
         @Override

jetty-start/src/main/java/org/eclipse/jetty/start/StartArgs.java

@@ -706,7 +706,7 @@ public class StartArgs
                 }
                 else
                 {
-                    cmd.addRawArg(x);
+                    cmd.addRawArg(getProperties().expand(x));
                 }
             }
 

jetty-start/src/test/java/org/eclipse/jetty/start/MainTest.java

@@ -31,6 +31,7 @@ import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.startsWith;
@@ -169,6 +170,37 @@ public class MainTest
         ConfigurationAssert.assertConfiguration(baseHome, args, "assert-home-with-jvm.txt");
     }
 
+    @Test
+    public void testJvmArgExpansion() throws Exception
+    {
+        List<String> cmdLineArgs = new ArrayList<>();
+
+        Path homePath = MavenTestingUtils.getTestResourceDir("dist-home").toPath().toRealPath();
+        cmdLineArgs.add("jetty.home=" + homePath.toString());
+        cmdLineArgs.add("user.dir=" + homePath.toString());
+
+        // JVM args
+        cmdLineArgs.add("--exec");
+        cmdLineArgs.add("-Xms1g");
+        cmdLineArgs.add("-Xmx4g");
+        cmdLineArgs.add("-Xloggc:${jetty.base}/logs/gc-${java.version}.log");
+
+        Main main = new Main();
+
+        StartArgs args = main.processCommandLine(cmdLineArgs.toArray(new String[cmdLineArgs.size()]));
+        BaseHome baseHome = main.getBaseHome();
+
+        assertThat("jetty.home", baseHome.getHome(), is(homePath.toString()));
+        assertThat("jetty.base", baseHome.getBase(), is(homePath.toString()));
+
+        CommandLineBuilder commandLineBuilder = args.getMainArgs(true);
+        String commandLine = commandLineBuilder.toString("\n");
+        String expectedExpansion = String.format("-Xloggc:%s/logs/gc-%s.log",
+            baseHome.getBase(), System.getProperty("java.version")
+        );
+        assertThat(commandLine, containsString(expectedExpansion));
+    }
+
     @Test
     public void testWithModules() throws Exception
     {

jetty-util/src/main/config/modules/console-capture.mod

@@ -18,7 +18,7 @@ resources/
 
 [ini-template]
 ## Logging directory (relative to $jetty.base)
-# jetty.console-capture.dir=logs
+# jetty.console-capture.dir=./logs
 
 ## Whether to append to existing file
 # jetty.console-capture.append=true

jetty-util/src/main/java/org/eclipse/jetty/util/UrlEncoded.java

@@ -945,7 +945,8 @@ public class UrlEncoded extends MultiMap<String> implements Cloneable
             }
             else if (b >= 'a' && b <= 'z' ||
                 b >= 'A' && b <= 'Z' ||
-                b >= '0' && b <= '9')
+                b >= '0' && b <= '9' ||
+                b == '-' || b == '.' || b == '_' || b == '~')
             {
                 encoded[n++] = b;
             }

jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java

@@ -1140,8 +1140,6 @@ public abstract class SslContextFactory extends AbstractLifeCycle implements Dum
                             managers[idx] = new AliasedX509ExtendedKeyManager((X509ExtendedKeyManager)managers[idx], alias);
                     }
                 }
-
-
             }
         }
 

jetty-util/src/test/java/org/eclipse/jetty/util/ssl/X509Test.java

@@ -18,16 +18,21 @@
 
 package org.eclipse.jetty.util.ssl;
 
+import java.nio.file.Path;
 import java.security.cert.X509Certificate;
 import javax.net.ssl.KeyManager;
 import javax.net.ssl.X509ExtendedKeyManager;
 
+import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
+import org.eclipse.jetty.util.resource.PathResource;
 import org.eclipse.jetty.util.resource.Resource;
 import org.junit.jupiter.api.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 public class X509Test
 {
@@ -128,57 +133,47 @@ public class X509Test
         assertThat("Normal X509", X509.isCertSign(bogusX509), is(false));
     }
 
-    private X509ExtendedKeyManager getX509ExtendedKeyManager(SslContextFactory sslContextFactory) throws Exception
+    @Test
+    public void testServerClass_WithSni() throws Exception
     {
-        Resource keystoreResource = Resource.newSystemResource("keystore");
+        SslContextFactory serverSsl = new SslContextFactory.Server();
-        Resource truststoreResource = Resource.newSystemResource("keystore");
+        Path keystorePath = MavenTestingUtils.getTestResourcePathFile("keystore_sni.p12");
-        sslContextFactory.setKeyStoreResource(keystoreResource);
+        serverSsl.setKeyStoreResource(new PathResource(keystorePath));
-        sslContextFactory.setTrustStoreResource(truststoreResource);
+        serverSsl.setKeyStorePassword("OBF:1vny1zlo1x8e1vnw1vn61x8g1zlu1vn4");
-        sslContextFactory.setKeyStorePassword("storepwd");
+        serverSsl.setKeyManagerPassword("OBF:1u2u1wml1z7s1z7a1wnl1u2g");
-        sslContextFactory.setKeyManagerPassword("keypwd");
+        serverSsl.start();
-        sslContextFactory.setTrustStorePassword("storepwd");
-        sslContextFactory.start();
-
-        KeyManager[] keyManagers = sslContextFactory.getKeyManagers(sslContextFactory.getKeyStore());
-        X509ExtendedKeyManager x509ExtendedKeyManager = null;
-
-        for (KeyManager keyManager : keyManagers)
-        {
-            if (keyManager instanceof X509ExtendedKeyManager)
-            {
-                x509ExtendedKeyManager = (X509ExtendedKeyManager)keyManager;
-                break;
-            }
-        }
-        assertThat("Found X509ExtendedKeyManager", x509ExtendedKeyManager, is(notNullValue()));
-        return x509ExtendedKeyManager;
     }
 
     @Test
-    public void testSniX509ExtendedKeyManager_ServerClass() throws Exception
+    public void testClientClass_WithSni() throws Exception
     {
-        SslContextFactory.Server serverSsl = new SslContextFactory.Server();
+        SslContextFactory clientSsl = new SslContextFactory.Client();
+        Path keystorePath = MavenTestingUtils.getTestResourcePathFile("keystore_sni.p12");
+        clientSsl.setKeyStoreResource(new PathResource(keystorePath));
+        clientSsl.setKeyStorePassword("OBF:1vny1zlo1x8e1vnw1vn61x8g1zlu1vn4");
+        clientSsl.setKeyManagerPassword("OBF:1u2u1wml1z7s1z7a1wnl1u2g");
+        clientSsl.start();
+    }
+
+    @Test
+    public void testServerClass_WithoutSni() throws Exception
+    {
+        SslContextFactory serverSsl = new SslContextFactory.Server();
         Resource keystoreResource = Resource.newSystemResource("keystore");
-        Resource truststoreResource = Resource.newSystemResource("keystore");
         serverSsl.setKeyStoreResource(keystoreResource);
-        serverSsl.setTrustStoreResource(truststoreResource);
         serverSsl.setKeyStorePassword("storepwd");
         serverSsl.setKeyManagerPassword("keypwd");
-        serverSsl.setTrustStorePassword("storepwd");
         serverSsl.start();
+    }
 
-        KeyManager[] keyManagers = serverSsl.getKeyManagers(serverSsl.getKeyStore());
+    @Test
-        X509ExtendedKeyManager x509ExtendedKeyManager = null;
+    public void testClientClass_WithoutSni() throws Exception
-
+    {
-        for (KeyManager keyManager : keyManagers)
+        SslContextFactory clientSsl = new SslContextFactory.Client();
-        {
+        Resource keystoreResource = Resource.newSystemResource("keystore");
-            if (keyManager instanceof X509ExtendedKeyManager)
+        clientSsl.setKeyStoreResource(keystoreResource);
-            {
+        clientSsl.setKeyStorePassword("storepwd");
-                x509ExtendedKeyManager = (X509ExtendedKeyManager)keyManager;
+        clientSsl.setKeyManagerPassword("keypwd");
-                break;
+        clientSsl.start();
-            }
-        }
-        assertThat("Found X509ExtendedKeyManager", x509ExtendedKeyManager, is(notNullValue()));
-        serverSsl.newSniX509ExtendedKeyManager(x509ExtendedKeyManager);
     }
 }

jetty-webapp/src/main/config/etc/webdefault.xml

@@ -194,107 +194,25 @@
   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
   <!-- The JSP page compiler and execution servlet, which is the mechanism  -->
   <!-- used by the jsp container to support JSP pages.  Traditionally,      -->
-  <!-- this servlet is mapped to URL pattern "*.jsp".  This servlet         -->
+  <!-- this servlet is mapped to URL pattern "*.jsp".                       -->
-  <!-- supports the following initialization parameters (default values     -->
+  <!-- See http://https://www.eclipse.org/jetty/documentation/              -->
-  <!-- are in square brackets):                                             -->
+  <!-- for applicable configuration params.                                 -->
-  <!--                                                                      -->
-  <!--   checkInterval       If development is false and reloading is true, -->
-  <!--                       background compiles are enabled. checkInterval -->
-  <!--                       is the time in seconds between checks to see   -->
-  <!--                       if a JSP page needs to be recompiled. [300]    -->
-  <!--                                                                      -->
-  <!--   compiler            Which compiler Ant should use to compile JSP   -->
-  <!--                       pages.  See the Ant documentation for more     -->
-  <!--                       information. [javac]                           -->
-  <!--                                                                      -->
-  <!--   classdebuginfo      Should the class file be compiled with         -->
-  <!--                       debugging information?  [true]                 -->
-  <!--                                                                      -->
-  <!--   classpath           What class path should I use while compiling   -->
-  <!--                       generated servlets?  [Created dynamically      -->
-  <!--                       based on the current web application]          -->
-  <!--                       Set to ? to make the container explicitly set  -->
-  <!--                       this parameter.                                -->
-  <!--                                                                      -->
-  <!--   development         Is Jasper used in development mode (will check -->
-  <!--                       for JSP modification on every access)?  [true] -->
-  <!--                                                                      -->
-  <!--   enablePooling       Determines whether tag handler pooling is      -->
-  <!--                       enabled  [true]                                -->
-  <!--                                                                      -->
-  <!--   fork                Tell Ant to fork compiles of JSP pages so that -->
-  <!--                       a separate JVM is used for JSP page compiles   -->
-  <!--                       from the one Tomcat is running in. [true]      -->
-  <!--                                                                      -->
-  <!--   ieClassId           The class-id value to be sent to Internet      -->
-  <!--                       Explorer when using <jsp:plugin> tags.         -->
-  <!--                       [clsid:8AD9C840-044E-11D1-B3E9-00805F499D93]   -->
-  <!--                                                                      -->
-  <!--   javaEncoding        Java file encoding to use for generating java  -->
-  <!--                       source files. [UTF-8]                          -->
-  <!--                                                                      -->
-  <!--   keepgenerated       Should we keep the generated Java source code  -->
-  <!--                       for each page instead of deleting it? [true]   -->
-  <!--                                                                      -->
-  <!--   logVerbosityLevel   The level of detailed messages to be produced  -->
-  <!--                       by this servlet.  Increasing levels cause the  -->
-  <!--                       generation of more messages.  Valid values are -->
-  <!--                       FATAL, ERROR, WARNING, INFORMATION, and DEBUG. -->
-  <!--                       [WARNING]                                      -->
-  <!--                                                                      -->
-  <!--   mappedfile          Should we generate static content with one     -->
-  <!--                       print statement per input line, to ease        -->
-  <!--                       debugging?  [false]                            -->
-  <!--                                                                      -->
-  <!--                                                                      -->
-  <!--   reloading           Should Jasper check for modified JSPs?  [true] -->
-  <!--                                                                      -->
-  <!--   suppressSmap        Should the generation of SMAP info for JSR45   -->
-  <!--                       debugging be suppressed?  [false]              -->
-  <!--                                                                      -->
-  <!--   dumpSmap            Should the SMAP info for JSR45 debugging be    -->
-  <!--                       dumped to a file? [false]                      -->
-  <!--                       False if suppressSmap is true                  -->
-  <!--                                                                      -->
-  <!--   scratchdir          What scratch directory should we use when      -->
-  <!--                       compiling JSP pages?  [default work directory  -->
-  <!--                       for the current web application]               -->
-  <!--                                                                      -->
-  <!--   tagpoolMaxSize      The maximum tag handler pool size  [5]         -->
-  <!--                                                                      -->
-  <!--   xpoweredBy          Determines whether X-Powered-By response       -->
-  <!--                       header is added by generated servlet  [false]  -->
-  <!--                                                                      -->
   <!-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -  -->
   <servlet id="jsp">
     <servlet-name>jsp</servlet-name>
     <servlet-class>org.eclipse.jetty.jsp.JettyJspServlet</servlet-class>
-    <init-param>
-      <param-name>logVerbosityLevel</param-name>
-      <param-value>DEBUG</param-value>
-    </init-param>
-    <init-param>
-      <param-name>fork</param-name>
-      <param-value>false</param-value>
-    </init-param>
     <init-param>
       <param-name>xpoweredBy</param-name>
       <param-value>false</param-value>
     </init-param>
     <init-param>
       <param-name>compilerTargetVM</param-name>
-      <param-value>1.7</param-value>
+      <param-value>1.8</param-value>
     </init-param>
     <init-param>
       <param-name>compilerSourceVM</param-name>
-      <param-value>1.7</param-value>
+      <param-value>1.8</param-value>
     </init-param>
-    <!--  
-    <init-param>
-        <param-name>classpath</param-name>
-        <param-value>?</param-value>
-    </init-param>
-    -->
     <load-on-startup>0</load-on-startup>
   </servlet>
 

jetty-websocket/javax-websocket-client/src/main/java/org/eclipse/jetty/websocket/javax/client/JavaxClientUpgradeRequest.java

@@ -21,7 +21,7 @@ package org.eclipse.jetty.websocket.javax.client;
 import java.net.URI;
 
 import org.eclipse.jetty.client.HttpResponse;
-import org.eclipse.jetty.client.http.HttpConnectionOverHTTP;
+import org.eclipse.jetty.io.EndPoint;
 import org.eclipse.jetty.websocket.core.FrameHandler;
 import org.eclipse.jetty.websocket.core.client.ClientUpgradeRequest;
 import org.eclipse.jetty.websocket.core.client.WebSocketCoreClient;
@@ -43,11 +43,11 @@ public class JavaxClientUpgradeRequest extends ClientUpgradeRequest
     }
 
     @Override
-    public void upgrade(HttpResponse response, HttpConnectionOverHTTP httpConnection)
+    public void upgrade(HttpResponse response, EndPoint endPoint)
     {
         frameHandler.setUpgradeRequest(new DelegatedJavaxClientUpgradeRequest(this));
         frameHandler.setUpgradeResponse(new DelegatedJavaxClientUpgradeResponse(response));
-        super.upgrade(response, httpConnection);
+        super.upgrade(response, endPoint);
     }
 
     @Override

jetty-websocket/javax-websocket-common/src/main/java/org/eclipse/jetty/websocket/javax/common/JavaxWebSocketSession.java

@@ -38,12 +38,9 @@ import javax.websocket.RemoteEndpoint.Basic;
 import javax.websocket.Session;
 import javax.websocket.WebSocketContainer;
 
-import org.eclipse.jetty.util.Callback;
 import org.eclipse.jetty.util.SharedBlockingCallback;
-import org.eclipse.jetty.util.component.AbstractLifeCycle;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
-import org.eclipse.jetty.websocket.core.CloseStatus;
 import org.eclipse.jetty.websocket.core.ExtensionConfig;
 import org.eclipse.jetty.websocket.core.FrameHandler;
 import org.eclipse.jetty.websocket.javax.common.decoders.AvailableDecoders;
@@ -53,7 +50,7 @@ import org.eclipse.jetty.websocket.javax.common.util.ReflectUtils;
 /**
  * Client Session for the JSR.
  */
-public class JavaxWebSocketSession extends AbstractLifeCycle implements javax.websocket.Session
+public class JavaxWebSocketSession implements javax.websocket.Session
 {
     private static final Logger LOG = Log.getLogger(JavaxWebSocketSession.class);
 
@@ -547,25 +544,6 @@ public class JavaxWebSocketSession extends AbstractLifeCycle implements javax.we
         return coreSession.isSecure();
     }
 
-    @Override
-    protected void doStop()
-    {
-        coreSession.close(CloseStatus.SHUTDOWN, "Container being shut down", new Callback()
-        {
-            @Override
-            public void succeeded()
-            {
-                coreSession.abort();
-            }
-
-            @Override
-            public void failed(Throwable x)
-            {
-                coreSession.abort();
-            }
-        });
-    }
-
     @Override
     public synchronized void removeMessageHandler(MessageHandler handler)
     {

jetty-websocket/javax-websocket-common/src/main/java/org/eclipse/jetty/websocket/javax/common/SessionTracker.java

@@ -21,10 +21,10 @@ package org.eclipse.jetty.websocket.javax.common;
 import java.util.Collections;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArraySet;
+import javax.websocket.CloseReason;
 import javax.websocket.Session;
 
 import org.eclipse.jetty.util.component.AbstractLifeCycle;
-import org.eclipse.jetty.util.component.LifeCycle;
 
 public class SessionTracker extends AbstractLifeCycle implements JavaxWebSocketSessionListener
 {
@@ -50,10 +50,12 @@ public class SessionTracker extends AbstractLifeCycle implements JavaxWebSocketS
     @Override
     protected void doStop() throws Exception
     {
-        for (JavaxWebSocketSession session : sessions)
+        for (Session session : sessions)
         {
-            LifeCycle.stop(session);
+            // GOING_AWAY is abnormal close status so it will hard close connection after sent.
+            session.close(new CloseReason(CloseReason.CloseCodes.GOING_AWAY, "Container being shut down"));
         }
+
         super.doStop();
     }
 }

jetty-websocket/javax-websocket-common/src/main/java/org/eclipse/jetty/websocket/javax/common/messages/MessageOutputStream.java

@@ -117,6 +117,7 @@ public class MessageOutputStream extends OutputStream
             frame.setPayload(buffer);
             frame.setFin(fin);
 
+            int initialBufferSize = buffer.remaining();
             try (SharedBlockingCallback.Blocker b = blocker.acquire())
             {
                 coreSession.sendFrame(frame, b, false);
@@ -127,8 +128,8 @@ public class MessageOutputStream extends OutputStream
             // Any flush after the first will be a CONTINUATION frame.
             frame = new Frame(OpCode.CONTINUATION);
 
-            // Buffer has been sent, buffer should have been consumed
+            // Buffer has been sent, but buffer should not have been consumed.
-            assert buffer.remaining() == 0;
+            assert buffer.remaining() == initialBufferSize;
 
             BufferUtil.clearToFill(buffer);
         }

jetty-websocket/javax-websocket-common/src/test/java/org/eclipse/jetty/websocket/javax/common/AbstractSessionTest.java

@@ -41,7 +41,6 @@ public abstract class AbstractSessionTest
         JavaxWebSocketFrameHandler frameHandler = container.newFrameHandler(websocketPojo, upgradeRequest);
         FrameHandler.CoreSession coreSession = new FrameHandler.CoreSession.Empty();
         session = new JavaxWebSocketSession(container, coreSession, frameHandler, null);
-        container.addManaged(session);
     }
 
     @AfterAll

jetty-websocket/javax-websocket-tests/src/main/java/org/eclipse/jetty/websocket/javax/tests/LocalFuzzer.java

@@ -20,6 +20,7 @@ package org.eclipse.jetty.websocket.javax.tests;
 
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
@@ -53,7 +54,7 @@ public class LocalFuzzer extends Fuzzer.Adapter implements Fuzzer, AutoCloseable
 
     public LocalFuzzer(Provider provider, CharSequence requestPath) throws Exception
     {
-        this(provider, requestPath, UpgradeUtils.newDefaultUpgradeRequestHeaders());
+        this(provider, requestPath, new HashMap<>());
     }
 
     public LocalFuzzer(Provider provider, CharSequence requestPath, Map<String, String> headers) throws Exception

jetty-websocket/javax-websocket-tests/src/main/java/org/eclipse/jetty/websocket/javax/tests/NetworkFuzzer.java

@@ -91,12 +91,11 @@ public class NetworkFuzzer extends Fuzzer.Adapter implements Fuzzer, AutoCloseab
     public ByteBuffer asNetworkBuffer(List<Frame> frames)
     {
         int bufferLength = frames.stream().mapToInt((f) -> f.getPayloadLength() + Generator.MAX_HEADER_LENGTH).sum();
-        ByteBuffer buffer = ByteBuffer.allocate(bufferLength);
+        ByteBuffer buffer = BufferUtil.allocate(bufferLength);
         for (Frame f : frames)
         {
             generator.generate(buffer, f);
         }
-        BufferUtil.flipToFlush(buffer, 0);
         return buffer;
     }
 
@@ -208,9 +207,9 @@ public class NetworkFuzzer extends Fuzzer.Adapter implements Fuzzer, AutoCloseab
         }
 
         @Override
-        protected void customize(EndPoint endp)
+        protected void customize(EndPoint endPoint)
         {
-            frameCapture.setEndPoint(endp);
+            frameCapture.setEndPoint(endPoint);
             futureCapture.complete(frameCapture);
         }
 

jetty-websocket/javax-websocket-tests/src/main/java/org/eclipse/jetty/websocket/javax/tests/UnitGenerator.java

@@ -23,7 +23,6 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
-import org.eclipse.jetty.io.MappedByteBufferPool;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.websocket.core.Behavior;
 import org.eclipse.jetty.websocket.core.Frame;
@@ -40,7 +39,6 @@ public class UnitGenerator extends Generator
 
     public UnitGenerator(Behavior behavior)
     {
-        super(new MappedByteBufferPool());
         applyMask = (behavior == Behavior.CLIENT);
     }
 

jetty-websocket/javax-websocket-tests/src/main/java/org/eclipse/jetty/websocket/javax/tests/UpgradeUtils.java

@@ -19,9 +19,6 @@
 package org.eclipse.jetty.websocket.javax.tests;
 
 import java.util.Map;
-import java.util.TreeMap;
-
-import org.eclipse.jetty.http.HttpHeader;
 
 public class UpgradeUtils
 {
@@ -31,32 +28,9 @@ public class UpgradeUtils
         upgradeRequest.append("GET ");
         upgradeRequest.append(requestPath == null ? "/" : requestPath);
         upgradeRequest.append(" HTTP/1.1\r\n");
-        headers.entrySet().stream().forEach(e ->
+        headers.entrySet().forEach(e ->
             upgradeRequest.append(e.getKey()).append(": ").append(e.getValue()).append("\r\n"));
         upgradeRequest.append("\r\n");
         return upgradeRequest.toString();
     }
-
-    public static String generateUpgradeRequest()
-    {
-        return generateUpgradeRequest("/", newDefaultUpgradeRequestHeaders());
-    }
-
-    public static String generateUpgradeRequest(CharSequence requestPath)
-    {
-        return generateUpgradeRequest(requestPath, newDefaultUpgradeRequestHeaders());
-    }
-
-    public static Map<String, String> newDefaultUpgradeRequestHeaders()
-    {
-        Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-        headers.put("Host", "local");
-        headers.put("Connection", "Upgrade");
-        headers.put("Upgrade", "WebSocket");
-        headers.put(HttpHeader.SEC_WEBSOCKET_KEY.asString(), "dGhlIHNhbXBsZSBub25jZQ==");
-        headers.put(HttpHeader.ORIGIN.asString(), "ws://local/");
-        // headers.put(WSConstants.SEC_WEBSOCKET_PROTOCOL, "echo");
-        headers.put(HttpHeader.SEC_WEBSOCKET_VERSION.asString(), "13");
-        return headers;
-    }
 }

jetty-websocket/javax-websocket-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/client/AbstractClientSessionTest.java

@@ -44,7 +44,6 @@ public abstract class AbstractClientSessionTest
         JavaxWebSocketFrameHandler frameHandler = container.newFrameHandler(websocketPojo, upgradeRequest);
         FrameHandler.CoreSession coreSession = new FrameHandler.CoreSession.Empty();
         session = new JavaxWebSocketSession(container, coreSession, frameHandler, null);
-        container.addManaged(session);
     }
 
     @AfterAll

jetty-websocket/javax-websocket-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/client/SessionAddMessageHandlerTest.java

@@ -82,13 +82,11 @@ public class SessionAddMessageHandlerTest
 
         // Session
         session = frameHandler.getSession();
-        session.start();
     }
 
     @AfterEach
     public void stopSession() throws Exception
     {
-        session.stop();
         container.stop();
     }
 

jetty-websocket/javax-websocket-tests/src/test/java/org/eclipse/jetty/websocket/javax/tests/server/AnnotatedServerEndpointTest.java

@@ -20,6 +20,7 @@ package org.eclipse.jetty.websocket.javax.tests.server;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import javax.websocket.server.ServerEndpointConfig;
@@ -30,7 +31,6 @@ import org.eclipse.jetty.websocket.core.Frame;
 import org.eclipse.jetty.websocket.core.OpCode;
 import org.eclipse.jetty.websocket.javax.tests.Fuzzer;
 import org.eclipse.jetty.websocket.javax.tests.LocalServer;
-import org.eclipse.jetty.websocket.javax.tests.UpgradeUtils;
 import org.eclipse.jetty.websocket.javax.tests.coders.DateDecoder;
 import org.eclipse.jetty.websocket.javax.tests.coders.TimeEncoder;
 import org.eclipse.jetty.websocket.javax.tests.server.configs.EchoSocketConfigurator;
@@ -67,8 +67,8 @@ public class AnnotatedServerEndpointTest
 
     private void assertResponse(String message, String expectedText) throws Exception
     {
-        Map<String, String> upgradeRequest = UpgradeUtils.newDefaultUpgradeRequestHeaders();
+        Map<String, String> headers = new HashMap<>();
-        upgradeRequest.put(HttpHeader.SEC_WEBSOCKET_SUBPROTOCOL.asString(), subprotocol);
+        headers.put(HttpHeader.SEC_WEBSOCKET_SUBPROTOCOL.asString(), subprotocol);
 
         List<Frame> send = new ArrayList<>();
         send.add(new Frame(OpCode.TEXT).setPayload(message));
@@ -78,7 +78,7 @@ public class AnnotatedServerEndpointTest
         expect.add(new Frame(OpCode.TEXT).setPayload(expectedText));
         expect.add(CloseStatus.toFrame(CloseStatus.NORMAL));
 
-        try (Fuzzer session = server.newNetworkFuzzer(path, upgradeRequest))
+        try (Fuzzer session = server.newNetworkFuzzer(path, headers))
         {
             session.sendFrames(send);
             session.expect(expect);

jetty-websocket/javax-websocket-tests/src/test/resources/jetty-logging.properties

@@ -12,4 +12,4 @@ org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog
 # org.eclipse.jetty.websocket.LEVEL=INFO
 # org.eclipse.jetty.websocket.tests.LEVEL=DEBUG
 # org.eclipse.jetty.websocket.tests.client.LEVEL=DEBUG
-# org.eclipse.jetty.websocket.tests.server.LEVEL=DEBUG
+# org.eclipse.jetty.websocket.tests.server.LEVEL=DEBUG

jetty-websocket/jetty-websocket-api/src/main/java/org/eclipse/jetty/websocket/api/UpgradeRequest.java

@@ -255,34 +255,6 @@ public interface UpgradeRequest
      */
     void setHeaders(Map<String, List<String>> headers);
 
-    /**
-     * Set the HTTP Version to use.
-     * <p>
-     * As of <a href="http://tools.ietf.org/html/rfc6455">RFC6455 (December 2011)</a> this should always be
-     * {@code HTTP/1.1}
-     *
-     * @param httpVersion the HTTP version to use.
-     */
-    void setHttpVersion(String httpVersion);
-
-    /**
-     * Set the HTTP method to use.
-     * <p>
-     * As of <a href="http://tools.ietf.org/html/rfc6455">RFC6455 (December 2011)</a> this is always {@code GET}
-     *
-     * @param method the HTTP method to use.
-     */
-    void setMethod(String method);
-
-    /**
-     * Set the Request URI to use for this request.
-     * <p>
-     * Must be an absolute URI with scheme {@code 'ws'} or {@code 'wss'}
-     *
-     * @param uri the Request URI
-     */
-    void setRequestURI(URI uri);
-
     /**
      * Set the Session associated with this request.
      * <p>

jetty-websocket/jetty-websocket-client/src/main/java/org/eclipse/jetty/websocket/client/ClientUpgradeRequest.java

@@ -26,9 +26,11 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.TreeMap;
 
 import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpScheme;
 import org.eclipse.jetty.websocket.api.UpgradeRequest;
 import org.eclipse.jetty.websocket.api.extensions.ExtensionConfig;
 
@@ -46,7 +48,6 @@ public final class ClientUpgradeRequest implements UpgradeRequest
     private String httpVersion;
     private String method;
     private String host;
-    private boolean secure;
 
     public ClientUpgradeRequest()
     {
@@ -57,12 +58,9 @@ public final class ClientUpgradeRequest implements UpgradeRequest
     {
         this.requestURI = uri;
         String scheme = uri.getScheme();
-        if (!"ws".equalsIgnoreCase(scheme) && !"wss".equalsIgnoreCase(scheme))
+        if (!HttpScheme.WS.is(scheme) || !HttpScheme.WSS.is(scheme))
-        {
             throw new IllegalArgumentException("URI scheme must be 'ws' or 'wss'");
-        }
         this.host = this.requestURI.getHost();
-        this.parameters.clear();
     }
 
     @Override
@@ -193,11 +191,7 @@ public final class ClientUpgradeRequest implements UpgradeRequest
     public String getProtocolVersion()
     {
         String version = getHeader("Sec-WebSocket-Version");
-        if (version == null)
+        return Objects.requireNonNullElse(version, "13");
-        {
-            return "13"; // Default
-        }
-        return version;
     }
 
     @Override
@@ -288,8 +282,7 @@ public final class ClientUpgradeRequest implements UpgradeRequest
     @Override
     public void setHeaders(Map<String, List<String>> headers)
     {
-        headers.clear();
+        this.headers.clear();
-
         for (Map.Entry<String, List<String>> entry : headers.entrySet())
         {
             String name = entry.getKey();
@@ -298,24 +291,6 @@ public final class ClientUpgradeRequest implements UpgradeRequest
         }
     }
 
-    @Override
-    public void setHttpVersion(String httpVersion)
-    {
-        this.httpVersion = httpVersion;
-    }
-
-    @Override
-    public void setMethod(String method)
-    {
-        this.method = method;
-    }
-
-    @Override
-    public void setRequestURI(URI uri)
-    {
-        this.requestURI = uri;
-    }
-
     @Override
     public void setSession(Object session)
     {

jetty-websocket/jetty-websocket-client/src/main/java/org/eclipse/jetty/websocket/client/impl/DelegatedJettyClientUpgradeRequest.java

@@ -231,24 +231,6 @@ public class DelegatedJettyClientUpgradeRequest implements UpgradeRequest
         // TODO
     }
 
-    @Override
-    public void setHttpVersion(String httpVersion)
-    {
-        // TODO
-    }
-
-    @Override
-    public void setMethod(String method)
-    {
-
-    }
-
-    @Override
-    public void setRequestURI(URI uri)
-    {
-        // TODO
-    }
-
     @Override
     public void setSession(Object session)
     {

jetty-websocket/jetty-websocket-client/src/main/java/org/eclipse/jetty/websocket/client/impl/JettyClientUpgradeRequest.java

@@ -24,7 +24,6 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import org.eclipse.jetty.client.HttpResponse;
-import org.eclipse.jetty.client.http.HttpConnectionOverHTTP;
 import org.eclipse.jetty.http.HttpFields;
 import org.eclipse.jetty.http.HttpHeader;
 import org.eclipse.jetty.http.HttpVersion;
@@ -89,18 +88,18 @@ public class JettyClientUpgradeRequest extends ClientUpgradeRequest
     }
 
     @Override
-    protected void customize(EndPoint endp)
+    protected void customize(EndPoint endPoint)
     {
-        super.customize(endp);
+        super.customize(endPoint);
-        handshakeRequest.configure(endp);
+        handshakeRequest.configure(endPoint);
     }
 
     @Override
-    public void upgrade(HttpResponse response, HttpConnectionOverHTTP httpConnection)
+    public void upgrade(HttpResponse response, EndPoint endPoint)
     {
         frameHandler.setUpgradeRequest(new DelegatedJettyClientUpgradeRequest(this));
         frameHandler.setUpgradeResponse(new DelegatedJettyClientUpgradeResponse(response));
-        super.upgrade(response, httpConnection);
+        super.upgrade(response, endPoint);
     }
 
     @Override

jetty-websocket/jetty-websocket-common/src/main/java/org/eclipse/jetty/websocket/common/SessionTracker.java

@@ -23,8 +23,8 @@ import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 
 import org.eclipse.jetty.util.component.AbstractLifeCycle;
-import org.eclipse.jetty.util.component.LifeCycle;
 import org.eclipse.jetty.websocket.api.Session;
+import org.eclipse.jetty.websocket.api.StatusCode;
 import org.eclipse.jetty.websocket.api.WebSocketSessionListener;
 
 public class SessionTracker extends AbstractLifeCycle implements WebSocketSessionListener
@@ -39,7 +39,6 @@ public class SessionTracker extends AbstractLifeCycle implements WebSocketSessio
     @Override
     public void onWebSocketSessionOpened(Session session)
     {
-        LifeCycle.start(session);
         sessions.add(session);
     }
 
@@ -47,7 +46,6 @@ public class SessionTracker extends AbstractLifeCycle implements WebSocketSessio
     public void onWebSocketSessionClosed(Session session)
     {
         sessions.remove(session);
-        LifeCycle.stop(session);
     }
 
     @Override
@@ -55,8 +53,10 @@ public class SessionTracker extends AbstractLifeCycle implements WebSocketSessio
     {
         for (Session session : sessions)
         {
-            LifeCycle.stop(session);
+            // SHUTDOWN is abnormal close status so it will hard close connection after sent.
+            session.close(StatusCode.SHUTDOWN, "Container being shut down");
         }
+
         super.doStop();
     }
 }

jetty-websocket/jetty-websocket-common/src/main/java/org/eclipse/jetty/websocket/common/WebSocketSession.java

@@ -23,21 +23,18 @@ import java.net.SocketAddress;
 import java.time.Duration;
 import java.util.Objects;
 
-import org.eclipse.jetty.util.Callback;
-import org.eclipse.jetty.util.component.AbstractLifeCycle;
 import org.eclipse.jetty.util.component.Dumpable;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 import org.eclipse.jetty.websocket.api.CloseStatus;
 import org.eclipse.jetty.websocket.api.Session;
-import org.eclipse.jetty.websocket.api.StatusCode;
 import org.eclipse.jetty.websocket.api.SuspendToken;
 import org.eclipse.jetty.websocket.api.UpgradeRequest;
 import org.eclipse.jetty.websocket.api.UpgradeResponse;
 import org.eclipse.jetty.websocket.api.WebSocketBehavior;
 import org.eclipse.jetty.websocket.core.FrameHandler;
 
-public class WebSocketSession extends AbstractLifeCycle implements Session, SuspendToken, Dumpable
+public class WebSocketSession implements Session, SuspendToken, Dumpable
 {
     private static final Logger LOG = Log.getLogger(WebSocketSession.class);
     private final FrameHandler.CoreSession coreSession;
@@ -243,25 +240,6 @@ public class WebSocketSession extends AbstractLifeCycle implements Session, Susp
         return coreSession;
     }
 
-    @Override
-    protected void doStop() throws Exception
-    {
-        coreSession.close(StatusCode.SHUTDOWN, "Container being shut down", new Callback()
-        {
-            @Override
-            public void succeeded()
-            {
-                coreSession.abort();
-            }
-
-            @Override
-            public void failed(Throwable x)
-            {
-                coreSession.abort();
-            }
-        });
-    }
-
     @Override
     public void dump(Appendable out, String indent) throws IOException
     {

jetty-websocket/jetty-websocket-common/src/test/java/org/eclipse/jetty/websocket/common/handshake/DummyUpgradeRequest.java

@@ -185,24 +185,6 @@ public class DummyUpgradeRequest implements UpgradeRequest
 
     }
 
-    @Override
-    public void setHttpVersion(String httpVersion)
-    {
-
-    }
-
-    @Override
-    public void setMethod(String method)
-    {
-
-    }
-
-    @Override
-    public void setRequestURI(URI uri)
-    {
-
-    }
-
     @Override
     public void setSession(Object session)
     {

jetty-websocket/jetty-websocket-server/src/main/java/org/eclipse/jetty/websocket/server/internal/UpgradeRequestAdapter.java

@@ -197,24 +197,6 @@ public class UpgradeRequestAdapter implements UpgradeRequest
         throw new UnsupportedOperationException("Not supported from Servlet API");
     }
 
-    @Override
-    public void setHttpVersion(String httpVersion)
-    {
-        throw new UnsupportedOperationException("Not supported from Servlet API");
-    }
-
-    @Override
-    public void setMethod(String method)
-    {
-        throw new UnsupportedOperationException("Not supported from Servlet API");
-    }
-
-    @Override
-    public void setRequestURI(URI uri)
-    {
-        throw new UnsupportedOperationException("Not supported from Servlet API");
-    }
-
     @Override
     public void setSession(Object session)
     {

jetty-websocket/jetty-websocket-tests/pom.xml

@@ -19,26 +19,49 @@
       <groupId>org.eclipse.jetty.websocket</groupId>
       <artifactId>jetty-websocket-api</artifactId>
       <version>${project.version}</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.eclipse.jetty.websocket</groupId>
       <artifactId>jetty-websocket-client</artifactId>
       <version>${project.version}</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.eclipse.jetty.websocket</groupId>
       <artifactId>jetty-websocket-server</artifactId>
       <version>${project.version}</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.eclipse.jetty.tests</groupId>
       <artifactId>jetty-http-tools</artifactId>
       <version>${project.version}</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.eclipse.jetty.toolchain</groupId>
+      <groupId>org.eclipse.jetty.http2</groupId>
-      <artifactId>jetty-test-helper</artifactId>
+      <artifactId>http2-server</artifactId>
-      <scope>compile</scope>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-alpn-server</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-alpn-java-server</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty.http2</groupId>
+      <artifactId>http2-http-client-transport</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
     </dependency>
   </dependencies>
 

jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/ConcurrentConnectTest.java

@@ -134,11 +134,6 @@ public class ConcurrentConnectTest
         }
 
         closeListener.closeLatch.await(5, TimeUnit.SECONDS);
-        for (EventSocket l : listeners)
-        {
-            assertTrue(((WebSocketSession)l.session).isStopped());
-        }
-
         assertTrue(client.getOpenSessions().isEmpty());
         assertTrue(client.getContainedBeans(WebSocketSession.class).isEmpty());
     }

jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/WebSocketOverHTTP2Test.java

@@ -0,0 +1,348 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.websocket.tests;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.ConnectException;
+import java.net.URI;
+import java.nio.channels.ClosedChannelException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.dynamic.HttpClientTransportDynamic;
+import org.eclipse.jetty.client.http.HttpClientConnectionFactory;
+import org.eclipse.jetty.http2.ErrorCode;
+import org.eclipse.jetty.http2.HTTP2Cipher;
+import org.eclipse.jetty.http2.client.HTTP2Client;
+import org.eclipse.jetty.http2.client.http.ClientConnectionFactoryOverHTTP2;
+import org.eclipse.jetty.http2.server.AbstractHTTP2ServerConnectionFactory;
+import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory;
+import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory;
+import org.eclipse.jetty.io.ClientConnectionFactory;
+import org.eclipse.jetty.io.ClientConnector;
+import org.eclipse.jetty.server.HttpChannel;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.log.StacklessLogging;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.eclipse.jetty.websocket.api.Session;
+import org.eclipse.jetty.websocket.api.StatusCode;
+import org.eclipse.jetty.websocket.api.UpgradeException;
+import org.eclipse.jetty.websocket.client.WebSocketClient;
+import org.eclipse.jetty.websocket.server.JettyWebSocketServlet;
+import org.eclipse.jetty.websocket.server.JettyWebSocketServletFactory;
+import org.eclipse.jetty.websocket.server.config.JettyWebSocketServletContainerInitializer;
+import org.eclipse.jetty.websocket.servlet.internal.UpgradeHttpServletRequest;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsStringIgnoringCase;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class WebSocketOverHTTP2Test
+{
+    private Server server;
+    private ServerConnector connector;
+    private ServerConnector tlsConnector;
+    private WebSocketClient wsClient;
+
+    private void startServer() throws Exception
+    {
+        startServer(new TestJettyWebSocketServlet());
+    }
+
+    private void startServer(TestJettyWebSocketServlet servlet) throws Exception
+    {
+        QueuedThreadPool serverThreads = new QueuedThreadPool();
+        serverThreads.setName("server");
+        server = new Server(serverThreads);
+        HttpConfiguration httpConfig = new HttpConfiguration();
+        HttpConnectionFactory h1c = new HttpConnectionFactory(httpConfig);
+        HTTP2CServerConnectionFactory h2c = new HTTP2CServerConnectionFactory(httpConfig);
+        connector = new ServerConnector(server, 1, 1, h1c, h2c);
+        server.addConnector(connector);
+
+        SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
+        sslContextFactory.setKeyStorePath("src/test/resources/keystore.p12");
+        sslContextFactory.setKeyStorePassword("storepwd");
+        sslContextFactory.setCipherComparator(HTTP2Cipher.COMPARATOR);
+
+        HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig);
+        httpsConfig.addCustomizer(new SecureRequestCustomizer());
+        HttpConnectionFactory h1s = new HttpConnectionFactory(httpsConfig);
+        HTTP2ServerConnectionFactory h2s = new HTTP2ServerConnectionFactory(httpsConfig);
+        ALPNServerConnectionFactory alpn = new ALPNServerConnectionFactory();
+        alpn.setDefaultProtocol(h1s.getProtocol());
+        SslConnectionFactory ssl = new SslConnectionFactory(sslContextFactory, alpn.getProtocol());
+        tlsConnector = new ServerConnector(server, 1, 1, ssl, alpn, h1s, h2s);
+        server.addConnector(tlsConnector);
+
+        ServletContextHandler context = new ServletContextHandler(server, "/");
+        context.addServlet(new ServletHolder(servlet), "/ws/*");
+        JettyWebSocketServletContainerInitializer.configure(context, null);
+
+        server.start();
+    }
+
+    private void startClient(Function<ClientConnector, ClientConnectionFactory.Info> protocolFn) throws Exception
+    {
+        ClientConnector clientConnector = new ClientConnector();
+        clientConnector.setSslContextFactory(new SslContextFactory.Client(true));
+        QueuedThreadPool clientThreads = new QueuedThreadPool();
+        clientThreads.setName("client");
+        clientConnector.setExecutor(clientThreads);
+        HttpClient httpClient = new HttpClient(new HttpClientTransportDynamic(clientConnector, protocolFn.apply(clientConnector)));
+        wsClient = new WebSocketClient(httpClient);
+        wsClient.start();
+    }
+
+    @AfterEach
+    public void stopServer() throws Exception
+    {
+        if (server != null)
+            server.stop();
+        if (wsClient != null)
+            wsClient.stop();
+    }
+
+    @Test
+    public void testWebSocketOverDynamicHTTP1() throws Exception
+    {
+        testWebSocketOverDynamicTransport(clientConnector -> HttpClientConnectionFactory.HTTP11);
+    }
+
+    @Test
+    public void testWebSocketOverDynamicHTTP2() throws Exception
+    {
+        testWebSocketOverDynamicTransport(clientConnector -> new ClientConnectionFactoryOverHTTP2.H2C(new HTTP2Client(clientConnector)));
+    }
+
+    private void testWebSocketOverDynamicTransport(Function<ClientConnector, ClientConnectionFactory.Info> protocolFn) throws Exception
+    {
+        startServer();
+        startClient(protocolFn);
+
+        EventSocket wsEndPoint = new EventSocket();
+        URI uri = URI.create("ws://localhost:" + connector.getLocalPort() + "/ws/echo");
+        Session session = wsClient.connect(wsEndPoint, uri).get(5, TimeUnit.SECONDS);
+
+        String text = "websocket";
+        session.getRemote().sendString(text);
+
+        String message = wsEndPoint.messageQueue.poll(5, TimeUnit.SECONDS);
+        assertNotNull(message);
+        assertEquals(text, message);
+
+        session.close(StatusCode.NORMAL, null);
+        assertTrue(wsEndPoint.closeLatch.await(5, TimeUnit.SECONDS));
+        assertEquals(StatusCode.NORMAL, wsEndPoint.statusCode);
+        assertNull(wsEndPoint.error);
+    }
+
+    @Test
+    public void testConnectProtocolDisabled() throws Exception
+    {
+        startServer();
+        AbstractHTTP2ServerConnectionFactory h2c = connector.getBean(AbstractHTTP2ServerConnectionFactory.class);
+        h2c.setConnectProtocolEnabled(false);
+
+        startClient(clientConnector -> new ClientConnectionFactoryOverHTTP2.H2C(new HTTP2Client(clientConnector)));
+
+        EventSocket wsEndPoint = new EventSocket();
+        URI uri = URI.create("ws://localhost:" + connector.getLocalPort() + "/ws/echo");
+
+        ExecutionException failure = Assertions.assertThrows(ExecutionException.class, () ->
+            wsClient.connect(wsEndPoint, uri).get(5, TimeUnit.SECONDS));
+
+        Throwable cause = failure.getCause();
+        assertThat(cause.getMessage(), containsStringIgnoringCase(ErrorCode.PROTOCOL_ERROR.name()));
+    }
+
+    @Test
+    public void testSlowWebSocketUpgradeWithHTTP2DataFramesQueued() throws Exception
+    {
+        startServer(new TestJettyWebSocketServlet()
+        {
+            @Override
+            protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
+            {
+                try
+                {
+                    super.service(request, response);
+                    // Flush the response to the client then wait before exiting
+                    // this method so that the client can send HTTP/2 DATA frames
+                    // that will be processed by the server while this method sleeps.
+                    response.flushBuffer();
+                    Thread.sleep(1000);
+                }
+                catch (InterruptedException x)
+                {
+                    throw new InterruptedIOException();
+                }
+            }
+        });
+
+        startClient(clientConnector -> new ClientConnectionFactoryOverHTTP2.H2(new HTTP2Client(clientConnector)));
+
+        // Connect and send immediately a message, so the message
+        // arrives to the server while the server is still upgrading.
+        EventSocket wsEndPoint = new EventSocket();
+        URI uri = URI.create("wss://localhost:" + tlsConnector.getLocalPort() + "/ws/echo");
+        Session session = wsClient.connect(wsEndPoint, uri).get(5, TimeUnit.SECONDS);
+        String text = "websocket";
+        session.getRemote().sendString(text);
+
+        String message = wsEndPoint.messageQueue.poll(5, TimeUnit.SECONDS);
+        assertNotNull(message);
+        assertEquals(text, message);
+
+        session.close(StatusCode.NORMAL, null);
+        assertTrue(wsEndPoint.closeLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testWebSocketConnectPortDoesNotExist() throws Exception
+    {
+        startServer();
+        startClient(clientConnector -> new ClientConnectionFactoryOverHTTP2.H2(new HTTP2Client(clientConnector)));
+
+        EventSocket wsEndPoint = new EventSocket();
+        URI uri = URI.create("ws://localhost:" + (connector.getLocalPort()+1) + "/ws/echo");
+
+        ExecutionException failure = Assertions.assertThrows(ExecutionException.class, () ->
+            wsClient.connect(wsEndPoint, uri).get(5, TimeUnit.SECONDS));
+
+        Throwable cause = failure.getCause();
+        assertThat(cause, instanceOf(ConnectException.class));
+        assertThat(cause.getMessage(), containsStringIgnoringCase("Connection refused"));
+    }
+
+    @Test
+    public void testWebSocketNotFound() throws Exception
+    {
+        startServer();
+        startClient(clientConnector -> new ClientConnectionFactoryOverHTTP2.H2(new HTTP2Client(clientConnector)));
+
+        EventSocket wsEndPoint = new EventSocket();
+        URI uri = URI.create("ws://localhost:" + connector.getLocalPort() + "/nothing");
+
+        ExecutionException failure = Assertions.assertThrows(ExecutionException.class, () ->
+            wsClient.connect(wsEndPoint, uri).get(5, TimeUnit.SECONDS));
+
+        Throwable cause = failure.getCause();
+        assertThat(cause, instanceOf(UpgradeException.class));
+        assertThat(cause.getMessage(), containsStringIgnoringCase("Unexpected HTTP Response Status Code: 501"));
+    }
+
+    @Test
+    public void testNotNegotiated() throws Exception
+    {
+        startServer();
+        startClient(clientConnector -> new ClientConnectionFactoryOverHTTP2.H2(new HTTP2Client(clientConnector)));
+
+        EventSocket wsEndPoint = new EventSocket();
+        URI uri = URI.create("ws://localhost:" + connector.getLocalPort() + "/ws/null");
+
+        ExecutionException failure = Assertions.assertThrows(ExecutionException.class, () ->
+            wsClient.connect(wsEndPoint, uri).get(5, TimeUnit.SECONDS));
+
+        Throwable cause = failure.getCause();
+        assertThat(cause, instanceOf(UpgradeException.class));
+        assertThat(cause.getMessage(), containsStringIgnoringCase("Unexpected HTTP Response Status Code: 503"));
+    }
+
+    @Test
+    public void testThrowFromCreator() throws Exception
+    {
+        startServer();
+        startClient(clientConnector -> new ClientConnectionFactoryOverHTTP2.H2(new HTTP2Client(clientConnector)));
+
+        EventSocket wsEndPoint = new EventSocket();
+        URI uri = URI.create("ws://localhost:" + connector.getLocalPort() + "/ws/throw");
+
+        ExecutionException failure;
+        try (StacklessLogging stacklessLogging = new StacklessLogging(HttpChannel.class))
+        {
+            failure = Assertions.assertThrows(ExecutionException.class, () ->
+                wsClient.connect(wsEndPoint, uri).get(5, TimeUnit.SECONDS));
+        }
+
+        Throwable cause = failure.getCause();
+        assertThat(cause, instanceOf(UpgradeException.class));
+        assertThat(cause.getMessage(), containsStringIgnoringCase("Unexpected HTTP Response Status Code: 500"));
+    }
+
+    @Test
+    public void testServerConnectionClose() throws Exception
+    {
+        startServer();
+        startClient(clientConnector -> new ClientConnectionFactoryOverHTTP2.H2(new HTTP2Client(clientConnector)));
+
+        EventSocket wsEndPoint = new EventSocket();
+        URI uri = URI.create("ws://localhost:" + connector.getLocalPort() + "/ws/connectionClose");
+
+        ExecutionException failure = Assertions.assertThrows(ExecutionException.class, () ->
+            wsClient.connect(wsEndPoint, uri).get(5, TimeUnit.SECONDS));
+
+        Throwable cause = failure.getCause();
+        assertThat(cause, instanceOf(ClosedChannelException.class));
+    }
+
+    private static class TestJettyWebSocketServlet extends JettyWebSocketServlet
+    {
+        @Override
+        protected void configure(JettyWebSocketServletFactory factory)
+        {
+            factory.addMapping("/ws/echo", (request, response) -> new EchoSocket());
+            factory.addMapping("/ws/null", (request, response) -> null);
+            factory.addMapping("/ws/throw", (request, response) ->
+            {
+                throw new RuntimeException("throwing from creator");
+            });
+            factory.addMapping("/ws/connectionClose", (request, response) ->
+            {
+                UpgradeHttpServletRequest servletRequest = (UpgradeHttpServletRequest)request.getHttpServletRequest();
+                Request baseRequest = servletRequest.getBaseRequest();
+                baseRequest.getHttpChannel().getEndPoint().close();
+                return new EchoSocket();
+            });
+        }
+    }
+}

jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/WebSocketStatsTest.java

@@ -24,10 +24,8 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import org.eclipse.jetty.io.ByteBufferPool;
 import org.eclipse.jetty.io.Connection;
 import org.eclipse.jetty.io.ConnectionStatistics;
-import org.eclipse.jetty.io.MappedByteBufferPool;
 import org.eclipse.jetty.server.HttpConnection;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
@@ -112,12 +110,10 @@ public class WebSocketStatsTest
 
     long getFrameByteSize(Frame frame)
     {
-        ByteBufferPool bufferPool = new MappedByteBufferPool();
+        Generator generator = new Generator();
-        Generator generator = new Generator(bufferPool);
+        ByteBuffer headerBuffer = BufferUtil.allocate(Generator.MAX_HEADER_LENGTH);
-        ByteBuffer buffer = bufferPool.acquire(frame.getPayloadLength() + 10, true);
+        generator.generateHeader(frame, headerBuffer);
-        int pos = BufferUtil.flipToFill(buffer);
+        return headerBuffer.remaining() + frame.getPayloadLength();
-        generator.generateWholeFrame(frame, buffer);
-        return buffer.position() - pos;
     }
 
     @Test

jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/WebSocketStopTest.java

@@ -38,12 +38,12 @@ import org.junit.jupiter.api.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-public class WriteAfterStopTest
+public class WebSocketStopTest
 {
-
     public class UpgradeServlet extends JettyWebSocketServlet
     {
         @Override
@@ -83,7 +83,28 @@ public class WriteAfterStopTest
     }
 
     @Test
-    public void test() throws Exception
+    public void stopWithOpenSessions() throws Exception
+    {
+        final URI uri = new URI("ws://localhost:" + connector.getLocalPort() + "/");
+
+        // Connect to two sessions to the server.
+        EventSocket clientSocket1 = new EventSocket();
+        EventSocket clientSocket2 = new EventSocket();
+        assertNotNull(client.connect(clientSocket1, uri).get(5, TimeUnit.SECONDS));
+        assertNotNull(client.connect(clientSocket2, uri).get(5, TimeUnit.SECONDS));
+        assertTrue(clientSocket1.openLatch.await(5, TimeUnit.SECONDS));
+        assertTrue(clientSocket2.openLatch.await(5, TimeUnit.SECONDS));
+
+        // WS client is stopped and closes sessions with SHUTDOWN code.
+        client.stop();
+        assertTrue(clientSocket1.closeLatch.await(5, TimeUnit.SECONDS));
+        assertTrue(clientSocket2.closeLatch.await(5, TimeUnit.SECONDS));
+        assertThat(clientSocket1.statusCode, is(StatusCode.SHUTDOWN));
+        assertThat(clientSocket2.statusCode, is(StatusCode.SHUTDOWN));
+    }
+
+    @Test
+    public void testWriteAfterStop() throws Exception
     {
         URI uri = new URI("ws://localhost:" + connector.getLocalPort() + "/");
         EventSocket clientSocket = new EventSocket();

jetty-websocket/jetty-websocket-tests/src/test/resources/jetty-logging.properties

@@ -7,4 +7,4 @@ org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog
 # org.eclipse.jetty.io.FillInterest.LEVEL=DEBUG
 # org.eclipse.jetty.client.LEVEL=DEBUG
 # org.eclipse.jetty.io.LEVEL=DEBUG
-# org.eclipse.jetty.io.ManagedSelector.LEVEL=INFO
+# org.eclipse.jetty.io.ManagedSelector.LEVEL=INFO