Fixes #4612 - ReservedThreadExecutor hangs when the last reserved thread idles out.

Explicitly removing the idled out thread from the stack,
rather than calling tryExecute().
Side benefit is that we are now removing idled out threads
that are least recently used.

Signed-off-by: Simone Bordet <simone.bordet@gmail.com>

jetty-util/src/main/java/org/eclipse/jetty/util/thread/ReservedThreadExecutor.java

@@ -282,7 +282,7 @@ public class ReservedThreadExecutor extends AbstractLifeCycle implements TryExec
             {
                 LOG.ignore(e);
                 _size.getAndIncrement();
-                _stack.addFirst(this);
+                _stack.offerFirst(this);
                 return false;
             }
         }
@@ -307,14 +307,13 @@ public class ReservedThreadExecutor extends AbstractLifeCycle implements TryExec
                     if (task != null)
                         return task;
 
-                    // Because threads are held in a stack, excess threads will be
-                    // idle.  However, we cannot remove threads from the bottom of
-                    // the stack, so we submit a poison pill job to stop the thread
-                    // on top of the stack (which unfortunately will be the most
-                    // recently used)
+                    if (_stack.remove(this))
+                    {
                         if (LOG.isDebugEnabled())
                             LOG.debug("{} IDLE", this);
-                    tryExecute(STOP);
+                        _size.decrementAndGet();
+                        return STOP;
+                    }
                 }
                 catch (InterruptedException e)
                 {

jetty-util/src/test/java/org/eclipse/jetty/util/thread/ReservedThreadExecutorTest.java

@@ -32,6 +32,8 @@ import org.junit.jupiter.api.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
@@ -183,6 +185,26 @@ public class ReservedThreadExecutorTest
         assertThat(_reservedExecutor.getAvailable(), is(0));
     }
 
+    @Test
+    public void testReservedIdleTimeoutWithOneReservedThread() throws Exception
+    {
+        long idleTimeout = 500;
+        _reservedExecutor.stop();
+        _reservedExecutor.setIdleTimeout(idleTimeout, TimeUnit.MILLISECONDS);
+        _reservedExecutor.start();
+
+        assertThat(_reservedExecutor.tryExecute(NOOP), is(false));
+        Thread thread = _executor.startThread();
+        assertNotNull(thread);
+        waitForAvailable(1);
+
+        Thread.sleep(2 * idleTimeout);
+
+        waitForAvailable(0);
+        thread.join(2 * idleTimeout);
+        assertFalse(thread.isAlive());
+    }
+
     protected void waitForAvailable(int size) throws InterruptedException
     {
         long started = System.nanoTime();
@@ -211,11 +233,16 @@ public class ReservedThreadExecutorTest
             _queue.addLast(task);
         }
 
-        public void startThread()
+        public Thread startThread()
         {
             Runnable task = _queue.pollFirst();
             if (task != null)
-                new Thread(task).start();
+            {
+                Thread thread = new Thread(task);
+                thread.start();
+                return thread;
+            }
+            return null;
         }
     }