Merge remote-tracking branch 'origin/jetty-9.2.x'

jetty-util/src/main/java/org/eclipse/jetty/util/thread/QueuedThreadPool.java

@@ -409,7 +409,7 @@ public class QueuedThreadPool extends AbstractLifeCycle implements SizedThreadPo
 
     private boolean startThreads(int threadsToStart)
     {
-        while (threadsToStart > 0)
+        while (threadsToStart > 0 && isRunning())
         {
             int threads = _threadsStarted.get();
             if (threads >= _maxThreads)
@@ -561,11 +561,10 @@ public class QueuedThreadPool extends AbstractLifeCycle implements SizedThreadPo
                                     long now = System.nanoTime();
                                     if (last == 0 || (now - last) > TimeUnit.MILLISECONDS.toNanos(_idleTimeout))
                                     {
-                                        shrink = _lastShrink.compareAndSet(last, now) &&
+                                        if (_lastShrink.compareAndSet(last, now) && _threadsStarted.compareAndSet(size, size - 1))
-                                                _threadsStarted.compareAndSet(size, size - 1);
-                                        if (shrink)
                                         {
-                                            return;
+                                            shrink=true;
+                                            break loop;
                                         }
                                     }
                                 }
@@ -576,9 +575,7 @@ public class QueuedThreadPool extends AbstractLifeCycle implements SizedThreadPo
                     finally
                     {
                         if (_threadsIdle.decrementAndGet() == 0)
-                        {
                             startThreads(1);
-                        }
                     }
                 }
             }
@@ -592,8 +589,13 @@ public class QueuedThreadPool extends AbstractLifeCycle implements SizedThreadPo
             }
             finally
             {
-                if (!shrink)
+                if (!shrink && isRunning())
-                    _threadsStarted.decrementAndGet();
+                {
+                    LOG.warn("Unexpected thread death: {} in {}",this,QueuedThreadPool.this);
+                    // This is an unexpected thread death!
+                    if (_threadsStarted.decrementAndGet()<getMaxThreads())
+                        startThreads(1);
+                }
                 _threads.remove(Thread.currentThread());
             }
         }

jetty-util/src/test/java/org/eclipse/jetty/util/thread/QueuedThreadPoolTest.java

@@ -18,6 +18,9 @@
 
 package org.eclipse.jetty.util.thread;
 
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 import java.util.concurrent.CountDownLatch;
@@ -26,6 +29,9 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 import org.eclipse.jetty.toolchain.test.AdvancedRunner;
 import org.eclipse.jetty.toolchain.test.annotation.Slow;
+import org.eclipse.jetty.util.log.Log;
+import org.eclipse.jetty.util.log.StdErrLog;
+import org.hamcrest.Matchers;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -255,7 +261,31 @@ public class QueuedThreadPoolTest
             {}
             now=System.currentTimeMillis();
         }
-        Assert.assertEquals(threads,tp.getThreads());
+        assertEquals(threads,tp.getThreads());
     }
 
+    @Test
+    public void testException() throws Exception
+    {
+        QueuedThreadPool tp= new QueuedThreadPool();
+        tp.setMinThreads(5);
+        tp.setMaxThreads(10);
+        tp.setIdleTimeout(1000);
+        tp.start();
+        try
+        {
+            ((StdErrLog)Log.getLogger(QueuedThreadPool.class)).setHideStacks(true);
+            tp.execute(new Runnable(){ public void run () { throw new IllegalStateException(); } });
+            tp.execute(new Runnable(){ public void run () { throw new Error(); } });
+            tp.execute(new Runnable(){ public void run () { throw new RuntimeException(); } });
+            tp.execute(new Runnable(){ public void run () { throw new ThreadDeath(); } });
+            
+            Thread.sleep(100);
+            assertThat(tp.getThreads(),greaterThanOrEqualTo(5));
+        }
+        finally
+        {
+            ((StdErrLog)Log.getLogger(QueuedThreadPool.class)).setHideStacks(false);
+        }
+    }
 }