From f5f679f92120a21eab27014c5f2fe63521719202 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 30 Jan 2018 10:02:26 +0100 Subject: [PATCH] testTranslogReplayWithFailure: do not assume engine is recoverable when disk errors stop The test currently makes the assumption that if underlying directory stops throwing exceptions, we can always open the engine. This is not the case as some errors can cause a corruption marker to be placed in the store. This commit refactors the test to only check that everything is OK if the engine was successfully opened. On top of that, there is no point in checking replay with no errors as we have another test for that. Closes #28426 --- .../index/engine/InternalEngineTests.java | 57 +++++++++---------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index d375790a1cc..dcb5146df03 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -80,8 +80,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.lucene.uid.Versions; @@ -115,7 +115,6 @@ import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexSearcherWrapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; -import org.elasticsearch.index.store.DirectoryUtils; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.SnapshotMatchers; import org.elasticsearch.index.translog.Translog; @@ -2382,46 +2381,46 @@ public class InternalEngineTests extends EngineTestCase { } public void testTranslogReplayWithFailure() throws IOException { - final int numDocs = randomIntBetween(1, 10); - for (int i = 0; i < numDocs; i++) { - ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null); - Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbers.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); - Engine.IndexResult indexResult = engine.index(firstIndexRequest); - assertThat(indexResult.getVersion(), equalTo(1L)); - } - assertVisibleCount(engine, numDocs); - engine.close(); - final MockDirectoryWrapper directory = DirectoryUtils.getLeaf(store.directory(), MockDirectoryWrapper.class); - if (directory != null) { + final MockDirectoryWrapper directory = newMockDirectory(); + final Path translogPath = createTempDir("testTranslogReplayWithFailure"); + try (Store store = createStore(directory)) { + final int numDocs = randomIntBetween(1, 10); + try (final InternalEngine engine = createEngine(store, translogPath)) { + for (int i = 0; i < numDocs; i++) { + ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null); + Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbers.UNASSIGNED_SEQ_NO, 0, + Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false); + Engine.IndexResult indexResult = engine.index(firstIndexRequest); + assertThat(indexResult.getVersion(), equalTo(1L)); + } + assertVisibleCount(engine, numDocs); + } // since we rollback the IW we are writing the same segment files again after starting IW but MDW prevents // this so we have to disable the check explicitly - boolean started = false; - final int numIters = randomIntBetween(10, 20); + final int numIters = randomIntBetween(3, 5); for (int i = 0; i < numIters; i++) { directory.setRandomIOExceptionRateOnOpen(randomDouble()); directory.setRandomIOExceptionRate(randomDouble()); directory.setFailOnOpenInput(randomBoolean()); directory.setAllowRandomFileNotFoundException(randomBoolean()); + boolean started = false; + InternalEngine engine = null; try { - engine = createEngine(store, primaryTranslogDir); + engine = createEngine(store, translogPath); started = true; - break; } catch (EngineException | IOException e) { + logger.trace("exception on open", e); + } + directory.setRandomIOExceptionRateOnOpen(0.0); + directory.setRandomIOExceptionRate(0.0); + directory.setFailOnOpenInput(false); + directory.setAllowRandomFileNotFoundException(false); + if (started) { + assertVisibleCount(engine, numDocs, false); + engine.close(); } } - - directory.setRandomIOExceptionRateOnOpen(0.0); - directory.setRandomIOExceptionRate(0.0); - directory.setFailOnOpenInput(false); - directory.setAllowRandomFileNotFoundException(false); - if (started == false) { - engine = createEngine(store, primaryTranslogDir); - } - } else { - // no mock directory, no fun. - engine = createEngine(store, primaryTranslogDir); } - assertVisibleCount(engine, numDocs, false); } private static void assertVisibleCount(InternalEngine engine, int numDocs) throws IOException {