From 8d1e526616826fffee5dff49f04f6e1704ba2679 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Thu, 21 Nov 2013 11:51:54 +0100
Subject: [PATCH] By default don't fail on close sicne pending open files are
 common in ES

We have pending open files on a regular basis since we search while
relocating etc. and keep search contexts around that are cleaned up
later. We should rather let the close call pass on even if files are
open and only force failures on teardown in ElasticsearchIntegrationTest
---
 .../test/ElasticsearchTestCase.java           |  4 +-
 .../test/store/MockDirectoryHelper.java       | 43 ++++++++++++++++++-
 2 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java b/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java
index b05d1cb53ad..71dcac12813 100644
--- a/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java
+++ b/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java
@@ -99,9 +99,9 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest {
     
     public static void ensureAllFilesClosed() throws IOException {
         try {
-            for (MockDirectoryWrapper w : MockDirectoryHelper.wrappers) {
+            for (MockDirectoryHelper.ElasticsearchMockDirectoryWrapper w : MockDirectoryHelper.wrappers) {
                 if (w.isOpen()) {
-                    w.close();
+                    w.closeWithRuntimeException();
                 }
             }
         } finally {
diff --git a/src/test/java/org/elasticsearch/test/store/MockDirectoryHelper.java b/src/test/java/org/elasticsearch/test/store/MockDirectoryHelper.java
index 16e126d87f6..862c0f99bc8 100644
--- a/src/test/java/org/elasticsearch/test/store/MockDirectoryHelper.java
+++ b/src/test/java/org/elasticsearch/test/store/MockDirectoryHelper.java
@@ -38,6 +38,7 @@ import org.elasticsearch.index.store.memory.ByteBufferDirectoryService;
 import org.elasticsearch.index.store.ram.RamDirectoryService;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
 
+import java.io.IOException;
 import java.util.Random;
 import java.util.Set;
 
@@ -48,7 +49,9 @@ public class MockDirectoryHelper {
     public static final String CHECK_INDEX_ON_CLOSE = "index.store.mock.check_index_on_close";
     public static final String RANDOM_PREVENT_DOUBLE_WRITE = "index.store.mock.random.prevent_double_write";
     public static final String RANDOM_NO_DELETE_OPEN_FILE = "index.store.mock.random.no_delete_open_file";
-    public static final Set<MockDirectoryWrapper> wrappers = ConcurrentCollections.newConcurrentSet();
+    public static final String RANDOM_FAIL_ON_CLOSE= "index.store.mock.random.fail_on_close";
+
+    public static final Set<ElasticsearchMockDirectoryWrapper> wrappers = ConcurrentCollections.newConcurrentSet();
     
     private final Random random;
     private final double randomIOExceptionRate;
@@ -59,6 +62,8 @@ public class MockDirectoryHelper {
     private final ShardId shardId;
     private final boolean preventDoubleWrite;
     private final boolean noDeleteOpenFile;
+    private final ESLogger logger;
+    private final boolean failOnClose;
 
     public MockDirectoryHelper(ShardId shardId, Settings indexSettings, ESLogger logger) {
         final long seed = indexSettings.getAsLong(ElasticsearchIntegrationTest.INDEX_SEED_SETTING, 0l);
@@ -70,16 +75,19 @@ public class MockDirectoryHelper {
         random.nextInt(shardId.getId() + 1); // some randomness per shard
         throttle = Throttling.valueOf(indexSettings.get(RANDOM_THROTTLE, random.nextDouble() < 0.1 ? "SOMETIMES" : "NEVER"));
         checkIndexOnClose = indexSettings.getAsBoolean(CHECK_INDEX_ON_CLOSE, random.nextDouble() < 0.1);
+        failOnClose = indexSettings.getAsBoolean(RANDOM_FAIL_ON_CLOSE, false);
+
         if (logger.isDebugEnabled()) {
             logger.debug("Using MockDirWrapper with seed [{}] throttle: [{}] checkIndexOnClose: [{}]", SeedUtils.formatSeed(seed),
                     throttle, checkIndexOnClose);
         }
         this.indexSettings = indexSettings;
         this.shardId = shardId;
+        this.logger = logger;
     }
 
     public Directory wrap(Directory dir) {
-        final MockDirectoryWrapper w = new MockDirectoryWrapper(random, dir);
+        final ElasticsearchMockDirectoryWrapper w = new ElasticsearchMockDirectoryWrapper(random, dir, logger, failOnClose);
         w.setRandomIOExceptionRate(randomIOExceptionRate);
         w.setRandomIOExceptionRateOnOpen(randomIOExceptionRateOnOpen);
         w.setThrottling(throttle);
@@ -129,4 +137,35 @@ public class MockDirectoryHelper {
 
     }
 
+    public static final class ElasticsearchMockDirectoryWrapper extends MockDirectoryWrapper {
+
+        private final ESLogger logger;
+        private final boolean failOnClose;
+
+        public ElasticsearchMockDirectoryWrapper(Random random, Directory delegate, ESLogger logger, boolean failOnClose) {
+            super(random, delegate);
+            this.logger = logger;
+            this.failOnClose = failOnClose;
+        }
+
+        @Override
+        public  void close() throws IOException {
+            try {
+                super.close();
+            } catch (RuntimeException ex) {
+                if (failOnClose) {
+                    throw ex;
+                }
+                // we catch the exception on close to properly close shards even if there are open files
+                // the test framework will call closeWithRuntimeException after the test exits to fail
+                // on unclosed files.
+                logger.debug("MockDirectoryWrapper#close() threw exception", ex);
+            }
+        }
+
+        public void closeWithRuntimeException() throws IOException {
+            super.close(); // force fail if open files etc. called in tear down of ElasticsearchIntegrationTest
+        }
+    }
+
 }