Improve EngineSearcher tracking

Currently we fail tests is any searcher reference is pending. Yet,
on a slow machine the freeContext calls that are async could still be
in flight so if there are pending searchers we wait for a bit to make
sure we don't fail if a freeContext call is in flight.
The MockEngine now also contains the stack trace of the first close call
if a searcher is closed twice.

src/test/java/org/elasticsearch/test/ElasticSearchTestCase.java

@@ -49,11 +49,11 @@ public abstract class ElasticSearchTestCase extends AbstractRandomizedTest {
 
     public static final String CHILD_VM_ID = System.getProperty("junit4.childvm.id", "" + System.currentTimeMillis());
     
-    public boolean awaitBusy(Predicate<?> breakPredicate) throws InterruptedException {
+    public static boolean awaitBusy(Predicate<?> breakPredicate) throws InterruptedException {
         return awaitBusy(breakPredicate, 10, TimeUnit.SECONDS);
     }
     
-    public boolean awaitBusy(Predicate<?> breakPredicate, long maxWaitTime, TimeUnit unit) throws InterruptedException {
+    public static boolean awaitBusy(Predicate<?> breakPredicate, long maxWaitTime, TimeUnit unit) throws InterruptedException {
         long maxTimeInMillis = TimeUnit.MILLISECONDS.convert(maxWaitTime, unit);
         long iterations = Math.max(Math.round(Math.log10(maxTimeInMillis) / Math.log10(2)), 1);
         long timeInMillis = 1;
@@ -102,8 +102,21 @@ public abstract class ElasticSearchTestCase extends AbstractRandomizedTest {
     }
     
     public static void ensureAllSearchersClosed() {
-        if (MockRobinEngine.INFLIGHT_ENGINE_SEARCHERS.isEmpty()) {
+        /* in some cases we finish a test faster than the freeContext calls make it to the
-            return;
+         * shards. Let's wait for some time if there are still searchers. If the are really 
+         * pending we will fail anyway.*/
+        try {
+            if (awaitBusy(new Predicate<Object>() {
+                public boolean apply(Object o) {
+                    return MockRobinEngine.INFLIGHT_ENGINE_SEARCHERS.isEmpty();
+                }
+            }, 5, TimeUnit.SECONDS)) {
+                return;
+            }
+        } catch (InterruptedException ex) {
+            if (MockRobinEngine.INFLIGHT_ENGINE_SEARCHERS.isEmpty()) {
+                return;
+            }
         }
         try {
             RuntimeException ex = null;

src/test/java/org/elasticsearch/test/engine/MockRobinEngine.java

@@ -95,6 +95,8 @@ public final class MockRobinEngine extends RobinEngine implements Engine {
     public static final class AssertingSearcher implements Searcher {
         private final Searcher searcher;
         private final ShardId shardId;
+        private RuntimeException firstReleaseStack;
+        private final Object lock = new Object();
 
         public AssertingSearcher(Searcher searcher, ShardId shardId) {
             this.searcher = searcher;
@@ -110,7 +112,16 @@ public final class MockRobinEngine extends RobinEngine implements Engine {
         @Override
         public boolean release() throws ElasticSearchException {
             RuntimeException remove = INFLIGHT_ENGINE_SEARCHERS.remove(this);
-            assert remove != null : "Released Searcher more than once, source [" + searcher.source() + "]";
+            synchronized (lock) { 
+                // make sure we only get this once and store the stack of the first caller!
+                if (remove == null) {
+                    assert firstReleaseStack != null;
+                    throw new AssertionError("Released Searcher more than once, source [" + searcher.source() + "]", firstReleaseStack);
+                } else {
+                    assert firstReleaseStack == null;
+                    firstReleaseStack = new RuntimeException("Searcher Released first here, source [" + searcher.source() + "]");
+                }
+            }
             return searcher.release();
         }