diff --git a/src/main/java/org/elasticsearch/index/engine/Engine.java b/src/main/java/org/elasticsearch/index/engine/Engine.java index 69bba98bc42..0ab816e9a97 100644 --- a/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -489,13 +489,8 @@ public abstract class Engine implements Closeable { /** Check whether the engine should be failed */ protected boolean maybeFailEngine(String source, Throwable t) { if (Lucene.isCorruptionException(t)) { - if (engineConfig.isFailEngineOnCorruption()) { - failEngine("corrupt file detected source: [" + source + "]", t); - return true; - } else { - logger.warn("corrupt file detected source: [{}] but [{}] is set to [{}]", t, source, - EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, engineConfig.isFailEngineOnCorruption()); - } + failEngine("corrupt file detected source: [" + source + "]", t); + return true; } else if (ExceptionsHelper.isOOM(t)) { failEngine("out of memory", t); return true; diff --git a/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/src/main/java/org/elasticsearch/index/engine/EngineConfig.java index 371bdfe5a77..29801b4031d 100644 --- a/src/main/java/org/elasticsearch/index/engine/EngineConfig.java +++ b/src/main/java/org/elasticsearch/index/engine/EngineConfig.java @@ -50,7 +50,6 @@ import java.util.concurrent.TimeUnit; */ public final class EngineConfig { private final ShardId shardId; - private volatile boolean failEngineOnCorruption = true; private volatile ByteSizeValue indexingBufferSize; private final int indexConcurrency; private volatile boolean compoundOnFlush = true; @@ -98,12 +97,6 @@ public final class EngineConfig { */ public static final String INDEX_GC_DELETES_SETTING = "index.gc_deletes"; - /** - * Index setting to enable / disable engine failures on detected index corruptions. Default is true / enabled. - * This setting is realtime updateable. - */ - public static final String INDEX_FAIL_ON_CORRUPTION_SETTING = "index.fail_on_corruption"; - /** * Index setting to control the initial index buffer size. * This setting is not realtime updateable. @@ -148,7 +141,6 @@ public final class EngineConfig { this.indexConcurrency = indexSettings.getAsInt(EngineConfig.INDEX_CONCURRENCY_SETTING, Math.max(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, (int) (EsExecutors.boundedNumberOfProcessors(indexSettings) * 0.65))); codecName = indexSettings.get(EngineConfig.INDEX_CODEC_SETTING, EngineConfig.DEFAULT_CODEC_NAME); indexingBufferSize = indexSettings.getAsBytesSize(INDEX_BUFFER_SIZE_SETTING, DEFAUTL_INDEX_BUFFER_SIZE); - failEngineOnCorruption = indexSettings.getAsBoolean(INDEX_FAIL_ON_CORRUPTION_SETTING, true); gcDeletesInMillis = indexSettings.getAsTime(INDEX_GC_DELETES_SETTING, EngineConfig.DEFAULT_GC_DELETES).millis(); } @@ -168,13 +160,6 @@ public final class EngineConfig { this.enableGcDeletes = enableGcDeletes; } - /** - * Returns true if the engine should be failed in the case of a corrupted index. Defaults to true - */ - public boolean isFailEngineOnCorruption() { - return failEngineOnCorruption; - } - /** * Returns the initial index buffer size. This setting is only read on startup and otherwise controlled by {@link org.elasticsearch.indices.memory.IndexingMemoryController} */ @@ -355,11 +340,4 @@ public final class EngineConfig { public void setCompoundOnFlush(boolean compoundOnFlush) { this.compoundOnFlush = compoundOnFlush; } - - /** - * Sets if the engine should be failed in the case of a corrupted index. Defaults to true - */ - public void setFailEngineOnCorruption(boolean failEngineOnCorruption) { - this.failEngineOnCorruption = failEngineOnCorruption; - } } diff --git a/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index c56d3309952..4d3041aee19 100644 --- a/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -1093,11 +1093,7 @@ public class InternalEngine extends Engine { @Override public void onFailedMerge(MergePolicy.MergeException e) { if (Lucene.isCorruptionException(e)) { - if (engineConfig.isFailEngineOnCorruption()) { - failEngine("corrupt file detected source: [merge]", e); - } else { - logger.warn("corrupt file detected source: [merge] but [{}] is set to [{}]", e, EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, engineConfig.isFailEngineOnCorruption()); - } + failEngine("corrupt file detected source: [merge]", e); } else { failEngine("merge exception", e); } diff --git a/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java b/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java index b70598f1ed2..e6fb7fbcf88 100644 --- a/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java +++ b/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java @@ -85,7 +85,6 @@ public class IndexDynamicSettingsModule extends AbstractModule { indexDynamicSettings.addDynamicSetting(LogDocMergePolicyProvider.INDEX_COMPOUND_FORMAT); indexDynamicSettings.addDynamicSetting(EngineConfig.INDEX_COMPOUND_ON_FLUSH, Validator.BOOLEAN); indexDynamicSettings.addDynamicSetting(EngineConfig.INDEX_GC_DELETES_SETTING, Validator.TIME); - indexDynamicSettings.addDynamicSetting(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, Validator.BOOLEAN); indexDynamicSettings.addDynamicSetting(IndexShard.INDEX_FLUSH_ON_CLOSE, Validator.BOOLEAN); indexDynamicSettings.addDynamicSetting(ShardSlowLogIndexingService.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN, Validator.TIME); indexDynamicSettings.addDynamicSetting(ShardSlowLogIndexingService.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO, Validator.TIME); diff --git a/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 76fff2ff4e9..4b7ef520bc7 100644 --- a/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -1029,13 +1029,6 @@ public class IndexShard extends AbstractIndexShardComponent { config.setCompoundOnFlush(compoundOnFlush); change = true; } - - final boolean failEngineOnCorruption = settings.getAsBoolean(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, config.isFailEngineOnCorruption()); - if (failEngineOnCorruption != config.isFailEngineOnCorruption()) { - logger.info("updating {} from [{}] to [{}]", EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, config.isFailEngineOnCorruption(), failEngineOnCorruption); - config.setFailEngineOnCorruption(failEngineOnCorruption); - change = true; - } } if (change) { refresh("apply settings"); diff --git a/src/test/java/org/elasticsearch/index/engine/InternalEngineSettingsTest.java b/src/test/java/org/elasticsearch/index/engine/InternalEngineSettingsTest.java index 22cbee7bfee..b244ecb9cdf 100644 --- a/src/test/java/org/elasticsearch/index/engine/InternalEngineSettingsTest.java +++ b/src/test/java/org/elasticsearch/index/engine/InternalEngineSettingsTest.java @@ -44,11 +44,9 @@ public class InternalEngineSettingsTest extends ElasticsearchSingleNodeTest { final int iters = between(1, 20); for (int i = 0; i < iters; i++) { boolean compoundOnFlush = randomBoolean(); - boolean failOnCorruption = randomBoolean(); long gcDeletes = Math.max(0, randomLong()); Settings build = ImmutableSettings.builder() - .put(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, failOnCorruption) .put(EngineConfig.INDEX_COMPOUND_ON_FLUSH, compoundOnFlush) .put(EngineConfig.INDEX_GC_DELETES_SETTING, gcDeletes) .build(); @@ -61,7 +59,6 @@ public class InternalEngineSettingsTest extends ElasticsearchSingleNodeTest { assertEquals(engine.config().getGcDeletesInMillis(), gcDeletes); assertEquals(engine.getGcDeletesInMillis(), gcDeletes); - assertEquals(engine.config().isFailEngineOnCorruption(), failOnCorruption); } Settings settings = ImmutableSettings.builder() diff --git a/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 4257b612311..7233ad7a18e 100644 --- a/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -147,7 +147,6 @@ public class InternalEngineTests extends ElasticsearchLuceneTestCase { defaultSettings = ImmutableSettings.builder() .put(EngineConfig.INDEX_COMPOUND_ON_FLUSH, randomBoolean()) .put(EngineConfig.INDEX_GC_DELETES_SETTING, "1h") // make sure this doesn't kick in on us - .put(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, randomBoolean()) .put(EngineConfig.INDEX_CODEC_SETTING, codecName) .put(EngineConfig.INDEX_CONCURRENCY_SETTING, indexConcurrency) .build(); // TODO randomize more settings @@ -681,7 +680,6 @@ public class InternalEngineTests extends ElasticsearchLuceneTestCase { ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, false); engine.create(new Engine.Create(null, newUid("1"), doc)); engine.flush(); - final boolean failEngine = defaultSettings.getAsBoolean(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, false); final int failInPhase = randomIntBetween(1, 3); try { engine.recover(new Engine.RecoveryHandler() { @@ -724,9 +722,9 @@ public class InternalEngineTests extends ElasticsearchLuceneTestCase { MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 2)); MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(2)); searchResult.close(); - assertThat(failEngine, is(false)); + fail("engine should have failed"); } catch (EngineClosedException ex) { - assertThat(failEngine, is(true)); + // expected } } @@ -1466,7 +1464,6 @@ public class InternalEngineTests extends ElasticsearchLuceneTestCase { assertEquals(store.refCount(), refCount); continue; } - holder.config().setFailEngineOnCorruption(true); assertEquals(store.refCount(), refCount + 1); final int numStarts = scaledRandomIntBetween(1, 5); for (int j = 0; j < numStarts; j++) { @@ -1474,7 +1471,6 @@ public class InternalEngineTests extends ElasticsearchLuceneTestCase { assertEquals(store.refCount(), refCount + 1); holder.close(); holder = createEngine(store, translog); - holder.config().setFailEngineOnCorruption(true); assertEquals(store.refCount(), refCount + 1); } catch (EngineCreationFailureException ex) { // all is fine @@ -1502,7 +1498,6 @@ public class InternalEngineTests extends ElasticsearchLuceneTestCase { IndexDynamicSettingsModule settings = new IndexDynamicSettingsModule(); - assertTrue(settings.containsSetting(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING)); assertTrue(settings.containsSetting(EngineConfig.INDEX_COMPOUND_ON_FLUSH)); assertTrue(settings.containsSetting(EngineConfig.INDEX_GC_DELETES_SETTING)); } diff --git a/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java index 08178714fcb..ab09e517645 100644 --- a/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java +++ b/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java @@ -121,7 +121,6 @@ public class ShadowEngineTests extends ElasticsearchLuceneTestCase { defaultSettings = ImmutableSettings.builder() .put(EngineConfig.INDEX_COMPOUND_ON_FLUSH, randomBoolean()) .put(EngineConfig.INDEX_GC_DELETES_SETTING, "1h") // make sure this doesn't kick in on us - .put(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, randomBoolean()) .put(EngineConfig.INDEX_CODEC_SETTING, codecName) .put(EngineConfig.INDEX_CONCURRENCY_SETTING, indexConcurrency) .build(); // TODO randomize more settings @@ -893,7 +892,6 @@ public class ShadowEngineTests extends ElasticsearchLuceneTestCase { assertEquals(store.refCount(), refCount); continue; } - holder.config().setFailEngineOnCorruption(true); assertEquals(store.refCount(), refCount+1); final int numStarts = scaledRandomIntBetween(1, 5); for (int j = 0; j < numStarts; j++) { @@ -901,7 +899,6 @@ public class ShadowEngineTests extends ElasticsearchLuceneTestCase { assertEquals(store.refCount(), refCount + 1); holder.close(); holder = createShadowEngine(store, translog); - holder.config().setFailEngineOnCorruption(true); assertEquals(store.refCount(), refCount + 1); } catch (EngineCreationFailureException ex) { // all is fine diff --git a/src/test/java/org/elasticsearch/index/store/CorruptedFileTest.java b/src/test/java/org/elasticsearch/index/store/CorruptedFileTest.java index dbb6ede06d4..ed2463d69de 100644 --- a/src/test/java/org/elasticsearch/index/store/CorruptedFileTest.java +++ b/src/test/java/org/elasticsearch/index/store/CorruptedFileTest.java @@ -123,19 +123,13 @@ public class CorruptedFileTest extends ElasticsearchIntegrationTest { } assertThat(cluster().numDataNodes(), greaterThanOrEqualTo(3)); - final boolean failOnCorruption = randomBoolean(); assertAcked(prepareCreate("test").setSettings(ImmutableSettings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1") .put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, NoMergePolicyProvider.class) .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false) // no checkindex - we corrupt shards on purpose - .put(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, failOnCorruption) .put(TranslogService.INDEX_TRANSLOG_DISABLE_FLUSH, true) // no translog based flush - it might change the .liv / segments.N files .put("indices.recovery.concurrent_streams", 10) )); - if (failOnCorruption == false) { // test the dynamic setting - client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder() - .put(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, true)).get(); - } ensureGreen(); disableAllocation("test"); IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs]; @@ -238,7 +232,6 @@ public class CorruptedFileTest extends ElasticsearchIntegrationTest { .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") .put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, NoMergePolicyProvider.class) .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false) // no checkindex - we corrupt shards on purpose - .put(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, true) .put(TranslogService.INDEX_TRANSLOG_DISABLE_FLUSH, true) // no translog based flush - it might change the .liv / segments.N files .put("indices.recovery.concurrent_streams", 10) )); @@ -323,7 +316,6 @@ public class CorruptedFileTest extends ElasticsearchIntegrationTest { assertAcked(prepareCreate("test").setSettings(ImmutableSettings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, true) .put("index.routing.allocation.include._name", primariesNode.getNode().name()) .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE, EnableAllocationDecider.Rebalance.NONE) @@ -387,7 +379,6 @@ public class CorruptedFileTest extends ElasticsearchIntegrationTest { assertAcked(prepareCreate("test").setSettings(ImmutableSettings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(1, 4)) // don't go crazy here it must recovery fast - .put(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, true) // This does corrupt files on the replica, so we can't check: .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false) .put("index.routing.allocation.include._name", primariesNode.getNode().name()) @@ -472,7 +463,6 @@ public class CorruptedFileTest extends ElasticsearchIntegrationTest { .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0") // no replicas for this test .put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, NoMergePolicyProvider.class) .put(MockFSDirectoryService.CHECK_INDEX_ON_CLOSE, false) // no checkindex - we corrupt shards on purpose - .put(EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, true) .put(TranslogService.INDEX_TRANSLOG_DISABLE_FLUSH, true) // no translog based flush - it might change the .liv / segments.N files .put("indices.recovery.concurrent_streams", 10) ));