Improved failsafe close handling for half closed endpoints

jetty-io/src/main/java/org/eclipse/jetty/io/AbstractEndPoint.java

@@ -44,6 +44,7 @@ public abstract class AbstractEndPoint extends IdleTimeout implements EndPoint
             return AbstractEndPoint.this.needsFill();
         }
     };
+    
     private final WriteFlusher _writeFlusher = new WriteFlusher(this)
     {
         @Override
@@ -142,9 +143,22 @@ public abstract class AbstractEndPoint extends IdleTimeout implements EndPoint
     @Override
     protected void onIdleExpired(TimeoutException timeout)
     {
-        // Note: Rely on fillInterest to notify onReadTimeout to close connection.
+        boolean output_shutdown=isOutputShutdown();
-        _fillInterest.onFail(timeout);
+        boolean input_shutdown=isInputShutdown();
-        _writeFlusher.onFail(timeout);
+        boolean fillFailed = _fillInterest.onFail(timeout);
+        boolean writeFailed = _writeFlusher.onFail(timeout);
+        
+        // If the endpoint is half closed and there was no onFail handling, the close here
+        // This handles the situation where the connection has completed its close handling 
+        // and the endpoint is half closed, but the other party does not complete the close.
+        // This perhaps should not check for half closed, however the servlet spec case allows
+        // for a dispatched servlet or suspended request to extend beyond the connections idle 
+        // time.  So if this test would always close an idle endpoint that is not handled, then 
+        // we would need a mode to ignore timeouts for some HTTP states
+        if (isOpen() && (output_shutdown || input_shutdown) && !(fillFailed || writeFailed))
+            close();
+        else 
+            LOG.debug("Ignored idle endpoint {}",this);
     }
 
     @Override

jetty-io/src/main/java/org/eclipse/jetty/io/FillInterest.java

@@ -93,12 +93,17 @@ public abstract class FillInterest
     
     /* ------------------------------------------------------------ */
     /** Call to signal a failure to a registered interest
+     * @return true if the cause was passed to a {@link Callback} instance
      */
-    public void onFail(Throwable cause)
+    public boolean onFail(Throwable cause)
     {
         Callback callback=_interested.get();
         if (callback!=null && _interested.compareAndSet(callback,null))
+        {
             callback.failed(cause);
+            return true;
+        }
+        return false;
     }
     
     /* ------------------------------------------------------------ */

jetty-io/src/main/java/org/eclipse/jetty/io/WriteFlusher.java

@@ -253,10 +253,14 @@ abstract public class WriteFlusher
             return _buffers;
         }
 
-        protected void fail(Throwable cause)
+        protected boolean fail(Throwable cause)
         {
             if (_callback!=null)
+            {
                 _callback.failed(cause);
+                return true;
+            }
+            return false;
         }
 
         protected void complete()
@@ -430,7 +434,12 @@ abstract public class WriteFlusher
         }
     }
 
-    public void onFail(Throwable cause)
+    /* ------------------------------------------------------------ */
+    /** Notify the flusher of a failure
+     * @param cause The cause of the failure
+     * @return true if the flusher passed the failure to a {@link Callback} instance
+     */
+    public boolean onFail(Throwable cause)
     {
         // Keep trying to handle the failure until we get to IDLE or FAILED state
         while(true)
@@ -442,7 +451,7 @@ abstract public class WriteFlusher
                 case FAILED:
                     if (DEBUG)
                         LOG.debug("ignored: {} {}", this, cause);
-                    return;
+                    return false;
 
                 case PENDING:
                     if (DEBUG)
@@ -450,10 +459,7 @@ abstract public class WriteFlusher
 
                     PendingState pending = (PendingState)current;
                     if (updateState(pending,__IDLE))
-                    {
+                        return pending.fail(cause);
-                        pending.fail(cause);
-                        return;
-                    }
                     break;
 
                 default:
@@ -461,7 +467,7 @@ abstract public class WriteFlusher
                         LOG.debug("failed: {} {}", this, cause);
 
                     if (updateState(current,new FailedState(cause)))
-                        return;
+                        return false;
                     break;
             }
         }

jetty-io/src/test/java/org/eclipse/jetty/io/ByteArrayEndPointTest.java

@@ -19,8 +19,6 @@
 package org.eclipse.jetty.io;
 
 import static org.hamcrest.Matchers.*;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -37,8 +35,6 @@ import org.eclipse.jetty.toolchain.test.AdvancedRunner;
 import org.eclipse.jetty.toolchain.test.annotation.Slow;
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.FutureCallback;
-import org.eclipse.jetty.util.log.Log;
-import org.eclipse.jetty.util.log.Logger;
 import org.eclipse.jetty.util.thread.Scheduler;
 import org.eclipse.jetty.util.thread.TimerScheduler;
 import org.junit.After;
@@ -132,6 +128,7 @@ public class ByteArrayEndPointTest
 
         assertEquals(true,endp.flush(BufferUtil.EMPTY_BUFFER,BufferUtil.toBuffer(" and"),BufferUtil.toBuffer(" more")));
         assertEquals("some output some more and more",endp.getOutputString());
+        endp.close();
     }
 
     @Test
@@ -150,6 +147,7 @@ public class ByteArrayEndPointTest
 
         assertEquals(true,endp.flush(data));
         assertEquals("data.",BufferUtil.toString(endp.takeOutput()));
+        endp.close();
     }
 
 
@@ -237,6 +235,7 @@ public class ByteArrayEndPointTest
         assertTrue(fcb.isDone());
         assertEquals(null, fcb.get());
         assertEquals(" more.", endp.getOutputString());
+        endp.close();
     }
     
     /**