Improved connection/destination close, so that a connection can

always notify its destination that it has been closed.

jetty-client/src/main/java/org/eclipse/jetty/client/HttpDestination.java

@@ -218,6 +218,10 @@ public abstract class HttpDestination implements Destination, Closeable, Dumpabl
         LOG.debug("Closed {}", this);
     }
 
+    public void close(Connection connection)
+    {
+    }
+
     /**
      * Aborts all the {@link HttpExchange}s queued in this destination.
      *

jetty-client/src/main/java/org/eclipse/jetty/client/MultiplexHttpDestination.java

@@ -126,6 +126,19 @@ public abstract class MultiplexHttpDestination<C extends Connection> extends Htt
         return true;
     }
 
+    @Override
+    public void close(Connection connection)
+    {
+        super.close(connection);
+        assert this.connection == connection;
+        while (true)
+        {
+            ConnectState current = connect.get();
+            if (connect.compareAndSet(current, ConnectState.DISCONNECTED))
+                break;
+        }
+    }
+
     protected abstract void send(C connection, HttpExchange exchange);
 
     private enum ConnectState

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpConnectionOverHTTP.java

@@ -96,7 +96,7 @@ public class HttpConnectionOverHTTP extends AbstractConnection implements Connec
         if (exchange != null)
             return exchange.getRequest().abort(new TimeoutException());
 
-        getHttpDestination().remove(this);
+        getHttpDestination().close(this);
         return true;
     }
 
@@ -126,7 +126,7 @@ public class HttpConnectionOverHTTP extends AbstractConnection implements Connec
     @Override
     public void close()
     {
-        getHttpDestination().remove(this);
+        getHttpDestination().close(this);
         getEndPoint().shutdownOutput();
         LOG.debug("{} oshut", this);
         getEndPoint().close();

jetty-client/src/main/java/org/eclipse/jetty/client/http/HttpDestinationOverHTTP.java

@@ -155,23 +155,25 @@ public class HttpDestinationOverHTTP extends HttpDestination implements Promise<
         else
         {
             LOG.debug("{} is stopped", client);
-            remove(connection);
+            close(connection);
             connection.close();
         }
     }
 
-    protected void remove(HttpConnectionOverHTTP connection)
+    @Override
+    public void close(Connection oldConnection)
     {
-        connectionPool.remove(connection);
+        super.close(oldConnection);
+        connectionPool.remove(oldConnection);
 
         // We need to execute queued requests even if this connection failed.
         // We may create a connection that is not needed, but it will eventually
         // idle timeout, so no worries
         if (!getHttpExchanges().isEmpty())
         {
-            connection = acquire();
-            if (connection != null)
-                process(connection, false);
+            HttpConnectionOverHTTP newConnection = acquire();
+            if (newConnection != null)
+                process(newConnection, false);
         }
     }