Issue #3481 TLS Close

Updates from review:
 - shutdown after flush
 - close if write fails

Signed-off-by: Greg Wilkins <gregw@webtide.com>

jetty-io/src/main/java/org/eclipse/jetty/io/ssl/SslConnection.java

@@ -1077,14 +1077,15 @@ public class SslConnection extends AbstractConnection implements Connection.Upgr
         @Override
         public void doShutdownOutput()
         {
+            final EndPoint endp = getEndPoint();
             try
             {
                 boolean close;
                 boolean flush = false;
                 synchronized (_decryptedEndPoint)
                 {
-                    boolean ishut = getEndPoint().isInputShutdown();
-                    boolean oshut = getEndPoint().isOutputShutdown();
+                    boolean ishut = endp.isInputShutdown();
+                    boolean oshut = endp.isOutputShutdown();
                     if (LOG.isDebugEnabled())
                         LOG.debug("shutdownOutput: {} oshut={}, ishut={} {}", SslConnection.this, oshut, ishut);
 
@@ -1100,30 +1101,31 @@ public class SslConnection extends AbstractConnection implements Connection.Upgr
                     close = ishut;
                 }
 
-                if (flush && !flush(BufferUtil.EMPTY_BUFFER))
+                if (flush)
                 {
-                    // We failed to flush. Retry after short delay just in case progress can be made
-                    Thread.yield();
-                    if (!flush(BufferUtil.EMPTY_BUFFER) && !close)
+                    if (flush(BufferUtil.EMPTY_BUFFER))
+                        endp.shutdownOutput();
+                    else if (!close)
                     {
+                        Thread.yield();
                         // if we still can't flush, but we are not closing the endpoint,
                         // let's just flush the encrypted output in the background.
                         // and continue as if we are closed. The assumption here is that
                         // the encrypted buffer will contain the entire close handshake
                         // and that a call to flush(EMPTY_BUFFER) is not needed.
-                        getEndPoint().write(Callback.NOOP, _encryptedOutput);
+                        endp.write(Callback.from(endp::shutdownOutput, t-> endp.close()), _encryptedOutput);
                     }
                 }
 
                 if (close)
-                    getEndPoint().close();
+                    endp.close();
                 else
                     ensureFillInterested();
             }
             catch (Throwable x)
             {
                 LOG.ignore(x);
-                getEndPoint().close();
+                endp.close();
             }
         }
 

jetty-util/src/main/java/org/eclipse/jetty/util/Callback.java

@@ -19,6 +19,7 @@
 package org.eclipse.jetty.util;
 
 import java.util.concurrent.CompletableFuture;
+import java.util.function.Consumer;
 
 import org.eclipse.jetty.util.thread.Invocable;
 
@@ -110,7 +111,130 @@ public interface Callback extends Invocable
         };
     }
 
-    class Nested implements Callback
+    /**
+     * Create a callback from the passed success and failure
+     * @param success Called when the callback succeeds
+     * @param failure Called when the callback fails
+     * @return a new Callback
+     */
+    static Callback from(Runnable success, Consumer<Throwable> failure)
+    {
+        return new Callback()
+        {
+            @Override
+            public void succeeded()
+            {
+                success.run();
+            }
+
+            @Override
+            public void failed(Throwable x)
+            {
+                failure.accept(x);
+            }
+        };
+    }
+
+    /** Creaste a callback that runs completed when it succeeds or fails
+     * @param completed The completion to run on success or failure
+     * @return a new callback
+     */
+    static Callback from(Runnable completed)
+    {
+        return new Completing()
+        {
+            public void completed()
+            {
+                completed.run();
+            }
+        };
+    }
+
+    /**
+     * Create a nested callback that runs completed after
+     * completing the nested callback.
+     * @param callback The nested callback
+     * @param completed The completion to run after the nested callback is completed
+     * @return a new callback.
+     */
+    static Callback from(Callback callback, Runnable completed)
+    {
+        return new Nested(callback)
+        {
+            public void completed()
+            {
+                completed.run();
+            }
+        };
+    }
+
+    /**
+     * Create a nested callback that runs completed before
+     * completing the nested callback.
+     * @param callback The nested callback
+     * @param completed The completion to run before the nested callback is completed. Any exceptions thrown
+     *                  from completed will result in a callback failure.
+     * @return a new callback.
+     */
+    static Callback from(Runnable completed, Callback callback)
+    {
+        return new Callback()
+        {
+            @Override
+            public void succeeded()
+            {
+                try
+                {
+                    completed.run();
+                    callback.succeeded();
+                }
+                catch(Throwable t)
+                {
+                    callback.failed(t);
+                }
+            }
+
+            @Override
+            public void failed(Throwable x)
+            {
+                try
+                {
+                    completed.run();
+                }
+                catch(Throwable t)
+                {
+                    x.addSuppressed(t);
+                }
+                callback.failed(x);
+            }
+        };
+    }
+
+
+    class Completing implements Callback
+    {
+        @Override
+        public void succeeded()
+        {
+            completed();
+        }
+
+        @Override
+        public void failed(Throwable x)
+        {
+            completed();
+        }
+
+        public void completed()
+        {
+        }
+    }
+
+    /**
+     * Nested Completing Callback that completes after
+     * completing the nested callback
+     */
+    class Nested extends Completing
     {
         private final Callback callback;
 
@@ -132,13 +256,27 @@ public interface Callback extends Invocable
         @Override
         public void succeeded()
         {
-            callback.succeeded();
+            try
+            {
+                callback.succeeded();
+            }
+            finally
+            {
+                completed();
+            }
         }
 
         @Override
         public void failed(Throwable x)
         {
-            callback.failed(x);
+            try
+            {
+                callback.failed(x);
+            }
+            finally
+            {
+                completed();
+            }
         }
 
         @Override
@@ -147,6 +285,7 @@ public interface Callback extends Invocable
             return callback.getInvocationType();
         }
     }
+
     /**
      * <p>A CompletableFuture that is also a Callback.</p>
      */