Merge pull request #4823 from eclipse/jetty-9.4.x-4798-recover_selector_failures

Issue #4798 - Recover from Selector Failures

jetty-io/src/main/java/org/eclipse/jetty/io/AbstractEndPoint.java

@@ -458,11 +458,11 @@ public abstract class AbstractEndPoint extends IdleTimeout implements EndPoint
             name = c.getSimpleName();
         }
 
-        return String.format("%s@%h{%s<->%s,%s,fill=%s,flush=%s,to=%d/%d}",
+        return String.format("%s@%h{l=%s,r=%s,%s,fill=%s,flush=%s,to=%d/%d}",
             name,
             this,
-            getRemoteAddress(),
             getLocalAddress(),
+            getRemoteAddress(),
             _state.get(),
             _fillInterest.toStateString(),
             _writeFlusher.toStateString(),

jetty-io/src/main/java/org/eclipse/jetty/io/ChannelEndPoint.java

@@ -20,12 +20,13 @@ package org.eclipse.jetty.io;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
 import java.nio.ByteBuffer;
-import java.nio.channels.ByteChannel;
 import java.nio.channels.CancelledKeyException;
-import java.nio.channels.GatheringByteChannel;
 import java.nio.channels.SelectionKey;
 import java.nio.channels.Selector;
+import java.nio.channels.SocketChannel;
 
 import org.eclipse.jetty.util.BufferUtil;
 import org.eclipse.jetty.util.log.Log;
@@ -41,21 +42,14 @@ public abstract class ChannelEndPoint extends AbstractEndPoint implements Manage
 {
     private static final Logger LOG = Log.getLogger(ChannelEndPoint.class);
 
-    private final ByteChannel _channel;
+    private final SocketChannel _channel;
-    private final GatheringByteChannel _gather;
+    private final ManagedSelector _selector;
-    protected final ManagedSelector _selector;
+    private SelectionKey _key;
-    protected final SelectionKey _key;
     private boolean _updatePending;
-
+    // The current value for interestOps.
-    /**
+    private int _currentInterestOps;
-     * The current value for {@link SelectionKey#interestOps()}.
+    // The desired value for interestOps.
-     */
+    private int _desiredInterestOps;
-    protected int _currentInterestOps;
-
-    /**
-     * The desired value for {@link SelectionKey#interestOps()}.
-     */
-    protected int _desiredInterestOps;
 
     private abstract class RunnableTask implements Runnable, Invocable
     {
@@ -94,14 +88,7 @@ public abstract class ChannelEndPoint extends AbstractEndPoint implements Manage
         }
     }
 
-    private final ManagedSelector.SelectorUpdate _updateKeyAction = new ManagedSelector.SelectorUpdate()
+    private final ManagedSelector.SelectorUpdate _updateKeyAction = this::updateKeyAction;
-    {
-        @Override
-        public void update(Selector selector)
-        {
-            updateKey();
-        }
-    };
 
     private final Runnable _runFillable = new RunnableCloseable("runFillable")
     {
@@ -166,13 +153,24 @@ public abstract class ChannelEndPoint extends AbstractEndPoint implements Manage
         }
     };
 
-    public ChannelEndPoint(ByteChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler)
+    public ChannelEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler)
     {
         super(scheduler);
         _channel = channel;
         _selector = selector;
         _key = key;
-        _gather = (channel instanceof GatheringByteChannel) ? (GatheringByteChannel)channel : null;
+    }
+
+    @Override
+    public InetSocketAddress getLocalAddress()
+    {
+        return (InetSocketAddress)_channel.socket().getLocalSocketAddress();
+    }
+
+    @Override
+    public InetSocketAddress getRemoteAddress()
+    {
+        return (InetSocketAddress)_channel.socket().getRemoteSocketAddress();
     }
 
     @Override
@@ -187,6 +185,21 @@ public abstract class ChannelEndPoint extends AbstractEndPoint implements Manage
         return _channel.isOpen();
     }
 
+    @Override
+    protected void doShutdownOutput()
+    {
+        try
+        {
+            Socket socket = _channel.socket();
+            if (!socket.isOutputShutdown())
+                socket.shutdownOutput();
+        }
+        catch (IOException e)
+        {
+            LOG.debug(e);
+        }
+    }
+
     @Override
     public void doClose()
     {
@@ -254,27 +267,10 @@ public abstract class ChannelEndPoint extends AbstractEndPoint implements Manage
     @Override
     public boolean flush(ByteBuffer... buffers) throws IOException
     {
-        long flushed = 0;
+        long flushed;
         try
         {
-            if (buffers.length == 1)
+            flushed = _channel.write(buffers);
-                flushed = _channel.write(buffers[0]);
-            else if (_gather != null && buffers.length > 1)
-                flushed = _gather.write(buffers, 0, buffers.length);
-            else
-            {
-                for (ByteBuffer b : buffers)
-                {
-                    if (b.hasRemaining())
-                    {
-                        int l = _channel.write(b);
-                        if (l > 0)
-                            flushed += l;
-                        if (b.hasRemaining())
-                            break;
-                    }
-                }
-            }
             if (LOG.isDebugEnabled())
                 LOG.debug("flushed {} {}", flushed, this);
         }
@@ -295,7 +291,7 @@ public abstract class ChannelEndPoint extends AbstractEndPoint implements Manage
         return true;
     }
 
-    public ByteChannel getChannel()
+    public SocketChannel getChannel()
     {
         return _channel;
     }
@@ -321,9 +317,8 @@ public abstract class ChannelEndPoint extends AbstractEndPoint implements Manage
     @Override
     public Runnable onSelected()
     {
-        /**
+        // This method runs from the selector thread,
-         * This method may run concurrently with {@link #changeInterests(int)}.
+        // possibly concurrently with changeInterests(int).
-         */
 
         int readyOps = _key.readyOps();
         int oldInterestOps;
@@ -357,12 +352,16 @@ public abstract class ChannelEndPoint extends AbstractEndPoint implements Manage
         return task;
     }
 
+    private void updateKeyAction(Selector selector)
+    {
+        updateKey();
+    }
+
     @Override
     public void updateKey()
     {
-        /**
+        // This method runs from the selector thread,
-         * This method may run concurrently with {@link #changeInterests(int)}.
+        // possibly concurrently with changeInterests(int).
-         */
 
         try
         {
@@ -385,22 +384,27 @@ public abstract class ChannelEndPoint extends AbstractEndPoint implements Manage
         }
         catch (CancelledKeyException x)
         {
-            LOG.debug("Ignoring key update for concurrently closed channel {}", this);
+            if (LOG.isDebugEnabled())
+                LOG.debug("Ignoring key update for cancelled key {}", this, x);
             close();
         }
         catch (Throwable x)
         {
-            LOG.warn("Ignoring key update for " + this, x);
+            LOG.warn("Ignoring key update for {}", this, x);
             close();
         }
     }
 
+    @Override
+    public void replaceKey(SelectionKey newKey)
+    {
+        _key = newKey;
+    }
+
     private void changeInterests(int operation)
     {
-        /**
+        // This method runs from any thread, possibly
-         * This method may run concurrently with
+        // concurrently with updateKey() and onSelected().
-         * {@link #updateKey()} and {@link #onSelected()}.
-         */
 
         int oldInterestOps;
         int newInterestOps;

jetty-io/src/main/java/org/eclipse/jetty/io/ManagedSelector.java

@@ -79,7 +79,7 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
     }
 
     private final AtomicBoolean _started = new AtomicBoolean(false);
-    private boolean _selecting = false;
+    private boolean _selecting;
     private final SelectorManager _selectorManager;
     private final int _id;
     private final ExecutionStrategy _strategy;
@@ -123,22 +123,6 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         start._started.await();
     }
 
-    protected void onSelectFailed(Throwable cause)
-    {
-        // override to change behavior
-    }
-
-    public int size()
-    {
-        Selector s = _selector;
-        if (s == null)
-            return 0;
-        Set<SelectionKey> keys = s.keys();
-        if (keys == null)
-            return 0;
-        return keys.size();
-    }
-
     @Override
     protected void doStop() throws Exception
     {
@@ -160,22 +144,119 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         super.doStop();
     }
 
+    protected int nioSelect(Selector selector, boolean now) throws IOException
+    {
+        return now ? selector.selectNow() : selector.select();
+    }
+
+    protected int select(Selector selector) throws IOException
+    {
+        try
+        {
+            int selected = nioSelect(selector, false);
+            if (selected == 0)
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Selector {} woken with none selected", selector);
+
+                if (Thread.interrupted() && !isRunning())
+                    throw new ClosedSelectorException();
+
+                if (FORCE_SELECT_NOW)
+                    selected = nioSelect(selector, true);
+            }
+            return selected;
+        }
+        catch (ClosedSelectorException x)
+        {
+            throw x;
+        }
+        catch (Throwable x)
+        {
+            handleSelectFailure(selector, x);
+            return 0;
+        }
+    }
+
+    protected void handleSelectFailure(Selector selector, Throwable failure) throws IOException
+    {
+        LOG.info("Caught select() failure, trying to recover: {}", failure.toString());
+        if (LOG.isDebugEnabled())
+            LOG.debug(failure);
+
+        Selector newSelector = _selectorManager.newSelector();
+        for (SelectionKey oldKey : selector.keys())
+        {
+            SelectableChannel channel = oldKey.channel();
+            int interestOps = safeInterestOps(oldKey);
+            if (interestOps >= 0)
+            {
+                try
+                {
+                    Object attachment = oldKey.attachment();
+                    SelectionKey newKey = channel.register(newSelector, interestOps, attachment);
+                    if (attachment instanceof Selectable)
+                        ((Selectable)attachment).replaceKey(newKey);
+                    oldKey.cancel();
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("Transferred {} iOps={} att={}", channel, interestOps, attachment);
+                }
+                catch (Throwable t)
+                {
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("Could not transfer {}", channel, t);
+                    IO.close(channel);
+                }
+            }
+            else
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Invalid interestOps for {}", channel);
+                IO.close(channel);
+            }
+        }
+
+        IO.close(selector);
+        _selector = newSelector;
+    }
+
+    protected void onSelectFailed(Throwable cause)
+    {
+        // override to change behavior
+    }
+
+    public int size()
+    {
+        Selector s = _selector;
+        if (s == null)
+            return 0;
+        Set<SelectionKey> keys = s.keys();
+        if (keys == null)
+            return 0;
+        return keys.size();
+    }
+
     /**
      * Submit an {@link SelectorUpdate} to be acted on between calls to {@link Selector#select()}
      *
      * @param update The selector update to apply at next wakeup
      */
     public void submit(SelectorUpdate update)
+    {
+        submit(update, false);
+    }
+
+    private void submit(SelectorUpdate update, boolean lazy)
     {
         if (LOG.isDebugEnabled())
-            LOG.debug("Queued change {} on {}", update, this);
+            LOG.debug("Queued change lazy={} {} on {}", lazy, update, this);
 
         Selector selector = null;
         synchronized (ManagedSelector.this)
         {
             _updates.offer(update);
 
-            if (_selecting)
+            if (_selecting && !lazy)
             {
                 selector = _selector;
                 // To avoid the extra select wakeup.
@@ -223,7 +304,7 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         }
     }
 
-    private void processConnect(SelectionKey key, final Connect connect)
+    private void processConnect(SelectionKey key, Connect connect)
     {
         SelectableChannel channel = key.channel();
         try
@@ -270,7 +351,18 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         EndPoint endPoint = _selectorManager.newEndPoint(channel, this, selectionKey);
         Connection connection = _selectorManager.newConnection(channel, endPoint, selectionKey.attachment());
         endPoint.setConnection(connection);
-        selectionKey.attach(endPoint);
+        submit(selector ->
+        {
+            SelectionKey key = selectionKey;
+            if (key.selector() != selector)
+            {
+                key = channel.keyFor(selector);
+                if (key != null && endPoint instanceof Selectable)
+                    ((Selectable)endPoint).replaceKey(key);
+            }
+            if (key != null)
+                key.attach(endPoint);
+        }, true);
         endPoint.onOpen();
         endPointOpened(endPoint);
         _selectorManager.connectionOpened(connection);
@@ -278,7 +370,7 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             LOG.debug("Created {}", endPoint);
     }
 
-    public void destroyEndPoint(final EndPoint endPoint)
+    void destroyEndPoint(EndPoint endPoint)
     {
         // Waking up the selector is necessary to clean the
         // cancelled-key set and tell the TCP stack that the
@@ -386,6 +478,14 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
          * {@link ManagedSelector} for this endpoint have been processed.
          */
         void updateKey();
+
+        /**
+         * Callback method invoked when the SelectionKey is replaced
+         * because the channel has been moved to a new selector.
+         *
+         * @param newKey the new SelectionKey
+         */
+        void replaceKey(SelectionKey newKey);
     }
 
     private class SelectorProducer implements ExecutionStrategy.Producer
@@ -465,39 +565,33 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             try
             {
                 Selector selector = _selector;
-                if (selector != null && selector.isOpen())
+                if (selector != null)
                 {
                     if (LOG.isDebugEnabled())
                         LOG.debug("Selector {} waiting with {} keys", selector, selector.keys().size());
-                    int selected = selector.select();
+                    int selected = ManagedSelector.this.select(selector);
-                    if (selected == 0)
+                    // The selector may have been recreated.
+                    selector = _selector;
+                    if (selector != null)
                     {
                         if (LOG.isDebugEnabled())
-                            LOG.debug("Selector {} woken with none selected", selector);
+                            LOG.debug("Selector {} woken up from select, {}/{}/{} selected", selector, selected, selector.selectedKeys().size(), selector.keys().size());
 
-                        if (Thread.interrupted() && !isRunning())
+                        int updates;
-                            throw new ClosedSelectorException();
+                        synchronized (ManagedSelector.this)
+                        {
+                            // finished selecting
+                            _selecting = false;
+                            updates = _updates.size();
+                        }
 
-                        if (FORCE_SELECT_NOW)
+                        _keys = selector.selectedKeys();
-                            selected = selector.selectNow();
+                        _cursor = _keys.isEmpty() ? Collections.emptyIterator() : _keys.iterator();
+                        if (LOG.isDebugEnabled())
+                            LOG.debug("Selector {} processing {} keys, {} updates", selector, _keys.size(), updates);
+
+                        return true;
                     }
-                    if (LOG.isDebugEnabled())
-                        LOG.debug("Selector {} woken up from select, {}/{}/{} selected", selector, selected, selector.selectedKeys().size(), selector.keys().size());
-
-                    int updates;
-                    synchronized (ManagedSelector.this)
-                    {
-                        // finished selecting
-                        _selecting = false;
-                        updates = _updates.size();
-                    }
-
-                    _keys = selector.selectedKeys();
-                    _cursor = _keys.isEmpty() ? Collections.emptyIterator() : _keys.iterator();
-                    if (LOG.isDebugEnabled())
-                        LOG.debug("Selector {} processing {} keys, {} updates", selector, _keys.size(), updates);
-
-                    return true;
                 }
             }
             catch (Throwable x)
@@ -513,7 +607,8 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
                 else
                 {
                     LOG.warn(x.toString());
-                    LOG.debug(x);
+                    if (LOG.isDebugEnabled())
+                        LOG.debug(x);
                 }
             }
             return false;
@@ -524,9 +619,10 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             while (_cursor.hasNext())
             {
                 SelectionKey key = _cursor.next();
+                Object attachment = key.attachment();
+                SelectableChannel channel = key.channel();
                 if (key.isValid())
                 {
-                    Object attachment = key.attachment();
                     if (LOG.isDebugEnabled())
                         LOG.debug("selected {} {} {} ", safeReadyOps(key), key, attachment);
                     try
@@ -549,24 +645,21 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
                     }
                     catch (CancelledKeyException x)
                     {
-                        LOG.debug("Ignoring cancelled key for channel {}", key.channel());
+                        if (LOG.isDebugEnabled())
-                        if (attachment instanceof EndPoint)
+                            LOG.debug("Ignoring cancelled key for channel {}", channel);
-                            IO.close((EndPoint)attachment);
+                        IO.close(attachment instanceof EndPoint ? (EndPoint)attachment : channel);
                     }
                     catch (Throwable x)
                     {
-                        LOG.warn("Could not process key for channel " + key.channel(), x);
+                        LOG.warn("Could not process key for channel {}", channel, x);
-                        if (attachment instanceof EndPoint)
+                        IO.close(attachment instanceof EndPoint ? (EndPoint)attachment : channel);
-                            IO.close((EndPoint)attachment);
                     }
                 }
                 else
                 {
                     if (LOG.isDebugEnabled())
-                        LOG.debug("Selector loop ignoring invalid key for channel {}", key.channel());
+                        LOG.debug("Selector loop ignoring invalid key for channel {}", channel);
-                    Object attachment = key.attachment();
+                    IO.close(attachment instanceof EndPoint ? (EndPoint)attachment : channel);
-                    if (attachment instanceof EndPoint)
-                        IO.close((EndPoint)attachment);
                 }
             }
             return null;
@@ -615,7 +708,7 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
 
     private static class DumpKeys implements SelectorUpdate
     {
-        private CountDownLatch latch = new CountDownLatch(1);
+        private final CountDownLatch latch = new CountDownLatch(1);
         private List<String> keys;
 
         @Override
@@ -651,9 +744,9 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         private final SelectableChannel _channel;
         private SelectionKey _key;
 
-        public Acceptor(SelectableChannel channel)
+        Acceptor(SelectableChannel channel)
         {
-            this._channel = channel;
+            _channel = channel;
         }
 
         @Override
@@ -661,13 +754,9 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         {
             try
             {
-                if (_key == null)
+                _key = _channel.register(selector, SelectionKey.OP_ACCEPT, this);
-                {
-                    _key = _channel.register(selector, SelectionKey.OP_ACCEPT, this);
-                }
-
                 if (LOG.isDebugEnabled())
-                    LOG.debug("{} acceptor={}", this, _key);
+                    LOG.debug("{} acceptor={}", this, _channel);
             }
             catch (Throwable x)
             {
@@ -679,13 +768,12 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         @Override
         public Runnable onSelected()
         {
-            SelectableChannel server = _key.channel();
             SelectableChannel channel = null;
             try
             {
                 while (true)
                 {
-                    channel = _selectorManager.doAccept(server);
+                    channel = _selectorManager.doAccept(_channel);
                     if (channel == null)
                         break;
                     _selectorManager.accepted(channel);
@@ -693,10 +781,9 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             }
             catch (Throwable x)
             {
+                LOG.warn("Accept failed for channel {}", channel, x);
                 IO.close(channel);
-                LOG.warn("Accept failed for channel " + channel, x);
             }
-
             return null;
         }
 
@@ -705,13 +792,18 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         {
         }
 
+        @Override
+        public void replaceKey(SelectionKey newKey)
+        {
+            _key = newKey;
+        }
+
         @Override
         public void close() throws IOException
         {
-            SelectionKey key = _key;
+            // May be called from any thread.
-            _key = null;
+            // Implements AbstractConnector.setAccepting(boolean).
-            if (key != null && key.isValid())
+            submit(selector -> _key.cancel());
-                key.cancel();
         }
     }
 
@@ -731,7 +823,8 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         @Override
         public void close()
         {
-            LOG.debug("closed accept of {}", channel);
+            if (LOG.isDebugEnabled())
+                LOG.debug("closed accept of {}", channel);
             IO.close(channel);
         }
 
@@ -747,7 +840,8 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             {
                 IO.close(channel);
                 _selectorManager.onAcceptFailed(channel, x);
-                LOG.debug(x);
+                if (LOG.isDebugEnabled())
+                    LOG.debug(x);
             }
         }
 
@@ -761,7 +855,8 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             }
             catch (Throwable x)
             {
-                LOG.debug(x);
+                if (LOG.isDebugEnabled())
+                    LOG.debug(x);
                 failed(x);
             }
         }
@@ -770,9 +865,16 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         {
             IO.close(channel);
             LOG.warn(String.valueOf(failure));
-            LOG.debug(failure);
+            if (LOG.isDebugEnabled())
+                LOG.debug(failure);
             _selectorManager.onAcceptFailed(channel, failure);
         }
+
+        @Override
+        public String toString()
+        {
+            return String.format("%s@%x[%s]", getClass().getSimpleName(), hashCode(), channel);
+        }
     }
 
     class Connect implements SelectorUpdate, Runnable
@@ -832,16 +934,15 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
 
     private class CloseConnections implements SelectorUpdate
     {
-        final Set<Closeable> _closed;
+        private final Set<Closeable> _closed;
-        final CountDownLatch _noEndPoints = new CountDownLatch(1);
+        private final CountDownLatch _complete = new CountDownLatch(1);
-        final CountDownLatch _complete = new CountDownLatch(1);
 
-        public CloseConnections()
+        private CloseConnections()
         {
             this(null);
         }
 
-        public CloseConnections(Set<Closeable> closed)
+        private CloseConnections(Set<Closeable> closed)
         {
             _closed = closed;
         }
@@ -851,7 +952,6 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         {
             if (LOG.isDebugEnabled())
                 LOG.debug("Closing {} connections on {}", selector.keys().size(), ManagedSelector.this);
-            boolean zero = true;
             for (SelectionKey key : selector.keys())
             {
                 if (key != null && key.isValid())
@@ -860,14 +960,12 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
                     Object attachment = key.attachment();
                     if (attachment instanceof EndPoint)
                     {
-                        EndPoint endp = (EndPoint)attachment;
+                        EndPoint endPoint = (EndPoint)attachment;
-                        if (!endp.isOutputShutdown())
+                        Connection connection = endPoint.getConnection();
-                            zero = false;
-                        Connection connection = endp.getConnection();
                         if (connection != null)
                             closeable = connection;
                         else
-                            closeable = endp;
+                            closeable = endPoint;
                     }
 
                     if (closeable != null)
@@ -884,30 +982,26 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
                     }
                 }
             }
-
-            if (zero)
-                _noEndPoints.countDown();
             _complete.countDown();
         }
     }
 
     private class StopSelector implements SelectorUpdate
     {
-        CountDownLatch _stopped = new CountDownLatch(1);
+        private final CountDownLatch _stopped = new CountDownLatch(1);
 
         @Override
         public void update(Selector selector)
         {
             for (SelectionKey key : selector.keys())
             {
-                if (key != null && key.isValid())
+                // Key may be null when using the UnixSocket selector.
-                {
+                if (key == null)
-                    Object attachment = key.attachment();
+                    continue;
-                    if (attachment instanceof EndPoint)
+                Object attachment = key.attachment();
-                        IO.close((EndPoint)attachment);
+                if (attachment instanceof Closeable)
-                }
+                    IO.close((Closeable)attachment);
             }
-
             _selector = null;
             IO.close(selector);
             _stopped.countDown();
@@ -936,7 +1030,8 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
             {
                 IO.close(_connect.channel);
                 LOG.warn(String.valueOf(failure));
-                LOG.debug(failure);
+                if (LOG.isDebugEnabled())
+                    LOG.debug(failure);
                 _connect.failed(failure);
             }
         }
@@ -944,7 +1039,7 @@ public class ManagedSelector extends ContainerLifeCycle implements Dumpable
         @Override
         public String toString()
         {
-            return String.format("CreateEndPoint@%x{%s,%s}", hashCode(), _connect, _key);
+            return String.format("CreateEndPoint@%x{%s}", hashCode(), _connect);
         }
     }
 

jetty-io/src/main/java/org/eclipse/jetty/io/SelectorManager.java

@@ -214,7 +214,7 @@ public abstract class SelectorManager extends ContainerLifeCycle implements Dump
      */
     public void accept(SelectableChannel channel, Object attachment)
     {
-        final ManagedSelector selector = chooseSelector();
+        ManagedSelector selector = chooseSelector();
         selector.submit(selector.new Accept(channel, attachment));
     }
 
@@ -229,7 +229,7 @@ public abstract class SelectorManager extends ContainerLifeCycle implements Dump
      */
     public Closeable acceptor(SelectableChannel server)
     {
-        final ManagedSelector selector = chooseSelector();
+        ManagedSelector selector = chooseSelector();
         ManagedSelector.Acceptor acceptor = selector.new Acceptor(server);
         selector.submit(acceptor);
         return acceptor;

jetty-io/src/main/java/org/eclipse/jetty/io/SocketChannelEndPoint.java

@@ -18,24 +18,15 @@
 
 package org.eclipse.jetty.io;
 
-import java.io.IOException;
-import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.nio.channels.SelectableChannel;
 import java.nio.channels.SelectionKey;
 import java.nio.channels.SocketChannel;
 
-import org.eclipse.jetty.util.log.Log;
-import org.eclipse.jetty.util.log.Logger;
 import org.eclipse.jetty.util.thread.Scheduler;
 
 public class SocketChannelEndPoint extends ChannelEndPoint
 {
-    private static final Logger LOG = Log.getLogger(SocketChannelEndPoint.class);
-    private final Socket _socket;
-    private final InetSocketAddress _local;
-    private final InetSocketAddress _remote;
-
     public SocketChannelEndPoint(SelectableChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler)
     {
         this((SocketChannel)channel, selector, key, scheduler);
@@ -44,40 +35,10 @@ public class SocketChannelEndPoint extends ChannelEndPoint
     public SocketChannelEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler)
     {
         super(channel, selector, key, scheduler);
-
-        _socket = channel.socket();
-        _local = (InetSocketAddress)_socket.getLocalSocketAddress();
-        _remote = (InetSocketAddress)_socket.getRemoteSocketAddress();
     }
 
     public Socket getSocket()
     {
-        return _socket;
+        return getChannel().socket();
-    }
-
-    @Override
-    public InetSocketAddress getLocalAddress()
-    {
-        return _local;
-    }
-
-    @Override
-    public InetSocketAddress getRemoteAddress()
-    {
-        return _remote;
-    }
-
-    @Override
-    protected void doShutdownOutput()
-    {
-        try
-        {
-            if (!_socket.isOutputShutdown())
-                _socket.shutdownOutput();
-        }
-        catch (IOException e)
-        {
-            LOG.debug(e);
-        }
     }
 }

jetty-unixsocket/src/main/java/org/eclipse/jetty/unixsocket/UnixSocketEndPoint.java

@@ -33,12 +33,15 @@ public class UnixSocketEndPoint extends ChannelEndPoint
 {
     private static final Logger LOG = Log.getLogger(UnixSocketEndPoint.class);
 
-    private final UnixSocketChannel _channel;
-
     public UnixSocketEndPoint(UnixSocketChannel channel, ManagedSelector selector, SelectionKey key, Scheduler scheduler)
     {
         super(channel, selector, key, scheduler);
-        _channel = channel;
+    }
+
+    @Override
+    public UnixSocketChannel getChannel()
+    {
+        return (UnixSocketChannel)super.getChannel();
     }
 
     @Override
@@ -56,11 +59,9 @@ public class UnixSocketEndPoint extends ChannelEndPoint
     @Override
     protected void doShutdownOutput()
     {
-        if (LOG.isDebugEnabled())
-            LOG.debug("oshut {}", this);
         try
         {
-            _channel.shutdownOutput();
+            getChannel().shutdownOutput();
             super.doShutdownOutput();
         }
         catch (IOException e)

jetty-unixsocket/src/test/java/org/eclipse/jetty/unixsocket/UnixSocketTest.java

@@ -41,6 +41,7 @@ import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.log.Logger;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.EnabledOnOs;
 
@@ -145,6 +146,7 @@ public class UnixSocketTest
         assertThat(contentResponse.getContentAsString(), containsString("Hello World"));
     }
 
+    @Tag("external")
     @Test
     public void testNotLocal() throws Exception
     {

tests/test-http-client-transport/src/test/java/org/eclipse/jetty/http/client/AsyncIOServletTest.java

@@ -75,6 +75,7 @@ import org.eclipse.jetty.util.FuturePromise;
 import org.eclipse.jetty.util.log.StacklessLogging;
 import org.hamcrest.Matchers;
 import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ArgumentsSource;
@@ -398,6 +399,7 @@ public class AsyncIOServletTest extends AbstractTest<AsyncIOServletTest.AsyncTra
     @ParameterizedTest
     @ArgumentsSource(TransportProvider.class)
     @Tag("Unstable")
+    @Disabled
     public void testAsyncWriteClosed(Transport transport) throws Exception
     {
         init(transport);

tests/test-integration/src/test/java/org/eclipse/jetty/test/RecoverFailedSelectorTest.java

@@ -0,0 +1,379 @@
+//
+//  ========================================================================
+//  Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//  ------------------------------------------------------------------------
+//  All rights reserved. This program and the accompanying materials
+//  are made available under the terms of the Eclipse Public License v1.0
+//  and Apache License v2.0 which accompanies this distribution.
+//
+//      The Eclipse Public License is available at
+//      http://www.eclipse.org/legal/epl-v10.html
+//
+//      The Apache License v2.0 is available at
+//      http://www.opensource.org/licenses/apache2.0.php
+//
+//  You may elect to redistribute this code under either of these licenses.
+//  ========================================================================
+//
+
+package org.eclipse.jetty.test;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectableChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.SocketChannel;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.http.HttpTester;
+import org.eclipse.jetty.io.ChannelEndPoint;
+import org.eclipse.jetty.io.Connection;
+import org.eclipse.jetty.io.EndPoint;
+import org.eclipse.jetty.io.ManagedSelector;
+import org.eclipse.jetty.io.SelectorManager;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.util.thread.Scheduler;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class RecoverFailedSelectorTest
+{
+    private Server server;
+    private ServerConnector connector;
+
+    private void start(Function<Server, ServerConnector> consumer) throws Exception
+    {
+        server = new Server();
+        connector = consumer.apply(server);
+        server.addConnector(connector);
+        server.start();
+    }
+
+    @AfterEach
+    public void dispose() throws Exception
+    {
+        server.stop();
+    }
+
+    @Test
+    public void testSelectFailureBetweenReads() throws Exception
+    {
+        // There will be 3 calls to select(): one at start(),
+        // one to accept, and one to set read interest.
+        CountDownLatch selectLatch = new CountDownLatch(3);
+        CountDownLatch failureLatch = new CountDownLatch(1);
+        AtomicBoolean fail = new AtomicBoolean();
+        start(server -> new ServerConnector(server, 1, 1)
+        {
+            @Override
+            protected SelectorManager newSelectorManager(Executor executor, Scheduler scheduler, int selectors)
+            {
+                return new ServerConnectorManager(executor, scheduler, selectors)
+                {
+                    @Override
+                    protected ManagedSelector newSelector(int id)
+                    {
+                        return new ManagedSelector(this, id)
+                        {
+                            @Override
+                            protected int nioSelect(Selector selector, boolean now) throws IOException
+                            {
+                                selectLatch.countDown();
+                                if (fail.getAndSet(false))
+                                    throw new IOException("explicit select() failure");
+                                return super.nioSelect(selector, now);
+                            }
+
+                            @Override
+                            protected void handleSelectFailure(Selector selector, Throwable failure) throws IOException
+                            {
+                                super.handleSelectFailure(selector, failure);
+                                failureLatch.countDown();
+                            }
+                        };
+                    }
+                };
+            }
+        });
+
+        try (SocketChannel client = SocketChannel.open(new InetSocketAddress("localhost", connector.getLocalPort())))
+        {
+            assertTrue(selectLatch.await(5, TimeUnit.SECONDS));
+
+            String request = "GET / HTTP/1.0\r\n\r\n";
+            int split = request.length() / 2;
+            ByteBuffer chunk1 = StandardCharsets.UTF_8.encode(request.substring(0, split));
+            ByteBuffer chunk2 = StandardCharsets.UTF_8.encode(request.substring(split));
+
+            // Wake up the selector and fail it.
+            fail.set(true);
+            client.write(chunk1);
+
+            // Wait for the failure handling to be completed.
+            assertTrue(failureLatch.await(5, TimeUnit.SECONDS));
+
+            // Write the rest of the request, the
+            // server should be able to continue.
+            client.write(chunk2);
+
+            HttpTester.Response response = HttpTester.parseResponse(HttpTester.from(client));
+            assertNotNull(response);
+            assertEquals(HttpStatus.NOT_FOUND_404, response.getStatus());
+        }
+    }
+
+    @Test
+    public void testAcceptDuringSelectFailure() throws Exception
+    {
+        // There will be 3 calls to select(): one at start(),
+        // one to accept, and one to set read interest.
+        CountDownLatch selectLatch = new CountDownLatch(3);
+        CountDownLatch failureLatch = new CountDownLatch(1);
+        AtomicBoolean fail = new AtomicBoolean();
+        AtomicReference<SocketChannel> socketRef = new AtomicReference<>();
+        start(server -> new ServerConnector(server, 1, 1)
+        {
+            @Override
+            protected SelectorManager newSelectorManager(Executor executor, Scheduler scheduler, int selectors)
+            {
+                return new ServerConnectorManager(executor, scheduler, selectors)
+                {
+                    @Override
+                    protected ManagedSelector newSelector(int id)
+                    {
+                        return new ManagedSelector(this, id)
+                        {
+                            @Override
+                            protected int nioSelect(Selector selector, boolean now) throws IOException
+                            {
+                                selectLatch.countDown();
+                                if (fail.getAndSet(false))
+                                    throw new IOException("explicit select() failure");
+                                return super.nioSelect(selector, now);
+                            }
+
+                            @Override
+                            protected void handleSelectFailure(Selector selector, Throwable failure) throws IOException
+                            {
+                                // Before handling the failure, connect with another socket.
+                                SocketChannel socket = SocketChannel.open(new InetSocketAddress("localhost", connector.getLocalPort()));
+                                socketRef.set(socket);
+                                super.handleSelectFailure(selector, failure);
+                                failureLatch.countDown();
+                            }
+                        };
+                    }
+                };
+            }
+        });
+
+        try (SocketChannel client = SocketChannel.open(new InetSocketAddress("localhost", connector.getLocalPort())))
+        {
+            assertTrue(selectLatch.await(5, TimeUnit.SECONDS));
+
+            String request = "GET / HTTP/1.0\r\n\r\n";
+            ByteBuffer buffer = StandardCharsets.UTF_8.encode(request);
+
+            // Wake up the selector and fail it.
+            fail.set(true);
+            client.write(buffer);
+
+            // Wait for the failure handling to be completed.
+            assertTrue(failureLatch.await(5, TimeUnit.SECONDS));
+
+            HttpTester.Response response = HttpTester.parseResponse(HttpTester.from(client));
+            assertNotNull(response);
+            assertEquals(HttpStatus.NOT_FOUND_404, response.getStatus());
+
+            // Verify that the newly created socket works well.
+            SocketChannel socket = socketRef.get();
+            buffer.flip();
+            socket.write(buffer);
+            response = HttpTester.parseResponse(HttpTester.from(socket));
+            assertNotNull(response);
+            assertEquals(HttpStatus.NOT_FOUND_404, response.getStatus());
+        }
+    }
+
+    @Test
+    public void testSelectFailureDuringEndPointCreation() throws Exception
+    {
+        // There will be 2 calls to select(): one at start(), one to accept.
+        CountDownLatch selectLatch = new CountDownLatch(2);
+        CountDownLatch failureLatch = new CountDownLatch(1);
+        AtomicBoolean fail = new AtomicBoolean();
+        CountDownLatch endPointLatch1 = new CountDownLatch(1);
+        CountDownLatch endPointLatch2 = new CountDownLatch(1);
+        start(server -> new ServerConnector(server, 1, 1)
+        {
+            @Override
+            protected SelectorManager newSelectorManager(Executor executor, Scheduler scheduler, int selectors)
+            {
+                return new ServerConnectorManager(executor, scheduler, selectors)
+                {
+                    @Override
+                    protected ManagedSelector newSelector(int id)
+                    {
+                        return new ManagedSelector(this, id)
+                        {
+                            @Override
+                            protected int nioSelect(Selector selector, boolean now) throws IOException
+                            {
+                                selectLatch.countDown();
+                                if (fail.getAndSet(false))
+                                    throw new IOException("explicit select() failure");
+                                return super.nioSelect(selector, now);
+                            }
+
+                            @Override
+                            protected void handleSelectFailure(Selector selector, Throwable failure) throws IOException
+                            {
+                                super.handleSelectFailure(selector, failure);
+                                failureLatch.countDown();
+                            }
+                        };
+                    }
+
+                    @Override
+                    protected ChannelEndPoint newEndPoint(SelectableChannel channel, ManagedSelector selectSet, SelectionKey selectionKey) throws IOException
+                    {
+                        try
+                        {
+                            ChannelEndPoint endPoint = super.newEndPoint(channel, selectSet, selectionKey);
+                            endPointLatch1.countDown();
+                            assertTrue(endPointLatch2.await(5, TimeUnit.SECONDS));
+                            return endPoint;
+                        }
+                        catch (InterruptedException x)
+                        {
+                            throw new InterruptedIOException();
+                        }
+                    }
+                };
+            }
+        });
+
+        try (SocketChannel client = SocketChannel.open(new InetSocketAddress("localhost", connector.getLocalPort())))
+        {
+            assertTrue(selectLatch.await(5, TimeUnit.SECONDS));
+
+            // Wait until the server EndPoint instance is created.
+            assertTrue(endPointLatch1.await(5, TimeUnit.SECONDS));
+
+            // Wake up the selector and fail it.
+            fail.set(true);
+            SocketChannel.open(new InetSocketAddress("localhost", connector.getLocalPort())).close();
+
+            // Wait until the selector is replaced.
+            assertTrue(failureLatch.await(5, TimeUnit.SECONDS));
+
+            // Continue the EndPoint creation.
+            endPointLatch2.countDown();
+
+            String request = "GET / HTTP/1.0\r\n\r\n";
+            ByteBuffer buffer = StandardCharsets.UTF_8.encode(request);
+            client.write(buffer);
+
+            HttpTester.Response response = HttpTester.parseResponse(HttpTester.from(client));
+            assertNotNull(response);
+            assertEquals(HttpStatus.NOT_FOUND_404, response.getStatus());
+        }
+    }
+
+    @Test
+    public void testSelectFailureDuringEndPointCreatedThenClosed() throws Exception
+    {
+        // There will be 2 calls to select(): one at start(), one to accept.
+        CountDownLatch selectLatch = new CountDownLatch(2);
+        CountDownLatch failureLatch = new CountDownLatch(1);
+        AtomicBoolean fail = new AtomicBoolean();
+        CountDownLatch connectionLatch1 = new CountDownLatch(1);
+        CountDownLatch connectionLatch2 = new CountDownLatch(1);
+        start(server -> new ServerConnector(server, 1, 1)
+        {
+            @Override
+            protected SelectorManager newSelectorManager(Executor executor, Scheduler scheduler, int selectors)
+            {
+                return new ServerConnectorManager(executor, scheduler, selectors)
+                {
+                    @Override
+                    protected ManagedSelector newSelector(int id)
+                    {
+                        return new ManagedSelector(this, id)
+                        {
+                            @Override
+                            protected int nioSelect(Selector selector, boolean now) throws IOException
+                            {
+                                selectLatch.countDown();
+                                if (fail.getAndSet(false))
+                                    throw new IOException("explicit select() failure");
+                                return super.nioSelect(selector, now);
+                            }
+
+                            @Override
+                            protected void handleSelectFailure(Selector selector, Throwable failure) throws IOException
+                            {
+                                super.handleSelectFailure(selector, failure);
+                                failureLatch.countDown();
+                            }
+                        };
+                    }
+
+                    @Override
+                    public Connection newConnection(SelectableChannel channel, EndPoint endPoint, Object attachment) throws IOException
+                    {
+                        try
+                        {
+                            Connection connection = super.newConnection(channel, endPoint, attachment);
+                            endPoint.close();
+                            connectionLatch1.countDown();
+                            assertTrue(connectionLatch2.await(5, TimeUnit.SECONDS));
+                            return connection;
+                        }
+                        catch (InterruptedException e)
+                        {
+                            throw new InterruptedIOException();
+                        }
+                    }
+                };
+            }
+        });
+
+        try (SocketChannel client = SocketChannel.open(new InetSocketAddress("localhost", connector.getLocalPort())))
+        {
+            assertTrue(selectLatch.await(5, TimeUnit.SECONDS));
+
+            // Wait until the server EndPoint is closed.
+            assertTrue(connectionLatch1.await(5, TimeUnit.SECONDS));
+
+            // Wake up the selector and fail it.
+            fail.set(true);
+            SocketChannel.open(new InetSocketAddress("localhost", connector.getLocalPort())).close();
+
+            // Wait until the selector is replaced.
+            assertTrue(failureLatch.await(5, TimeUnit.SECONDS));
+
+            // Continue the server processing.
+            connectionLatch2.countDown();
+
+            // The channel has been closed on the server.
+            int read = client.read(ByteBuffer.allocate(1));
+            assertTrue(read < 0);
+        }
+    }
+}

tests/test-integration/src/test/resources/jetty-logging.properties

@@ -1,5 +1,4 @@
 org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog
 #org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.Slf4jLog
-org.eclipse.jetty.LEVEL=WARN
 #org.eclipse.jetty.LEVEL=DEBUG
 #org.eclipse.jetty.websocket.LEVEL=DEBUG