honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove index.flush_on_close entirely 

This undocumented setting was mainly used for testing and a safety net for
a new flushOnClose feature back in 1.x. We can now safely remove this setting.
The test usage now uses a mock plugin setting to achive the same.
This commit is contained in:
Simon Willnauer 2016-01-14 10:31:54 +01:00
parent dc51dd0056
commit aaf7e55c5f
8 changed files with 39 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
core/src/main/java/org/elasticsearch/index/IndexSettings.java
View File

@ -63,11 +63,6 @@ public final class IndexSettings {
public static final String INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE = "index.translog.flush_threshold_size";
public static final TimeValue DEFAULT_GC_DELETES = TimeValue.timeValueSeconds(60);
/**
* Index setting to control if a flush is executed before engine is closed. The default is <code>true</code>
*/
public static final String INDEX_FLUSH_ON_CLOSE = "index.flush_on_close";
/**
* Index setting to enable / disable deletes garbage collection.
* This setting is realtime updateable
@ -97,7 +92,6 @@ public final class IndexSettings {
private final TimeValue syncInterval;
private volatile TimeValue refreshInterval;
private volatile ByteSizeValue flushThresholdSize;
private final boolean flushOnClose;
private final MergeSchedulerConfig mergeSchedulerConfig;
private final MergePolicyConfig mergePolicyConfig;
@ -185,7 +179,6 @@ public final class IndexSettings {
syncInterval = settings.getAsTime(INDEX_TRANSLOG_SYNC_INTERVAL, TimeValue.timeValueSeconds(5));
refreshInterval = settings.getAsTime(INDEX_REFRESH_INTERVAL, DEFAULT_REFRESH_INTERVAL);
flushThresholdSize = settings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(512, ByteSizeUnit.MB));
flushOnClose = settings.getAsBoolean(IndexSettings.INDEX_FLUSH_ON_CLOSE, true);
mergeSchedulerConfig = new MergeSchedulerConfig(settings);
gcDeletesInMillis = settings.getAsTime(IndexSettings.INDEX_GC_DELETES_SETTING, DEFAULT_GC_DELETES).getMillis();
this.mergePolicyConfig = new MergePolicyConfig(logger, settings);
@ -438,11 +431,6 @@ public final class IndexSettings {
*/
public ByteSizeValue getFlushThresholdSize() { return flushThresholdSize; }
/**
* Returns <code>true</code> iff this index should be flushed on close. Default is <code>true</code>
*/
public boolean isFlushOnClose() { return flushOnClose; }
/**
* Returns the {@link MergeSchedulerConfig}
*/

2
core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
View File

@ -802,7 +802,7 @@ public class IndexShard extends AbstractIndexShardComponent {
} finally {
final Engine engine = this.currentEngineReference.getAndSet(null);
try {
if (engine != null && flushEngine && indexSettings.isFlushOnClose()) {
if (engine != null && flushEngine) {
engine.flushAndClose();
}
} finally { // playing safe here and close the engine even if the above succeeds - close can be called multiple times

10
core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
View File

@ -128,16 +128,6 @@ import static org.hamcrest.Matchers.equalTo;
*/
public class IndexShardTests extends ESSingleNodeTestCase {
public void testFlushOnDeleteSetting() throws Exception {
final boolean booleanValue = randomBoolean();
createIndex("test", settingsBuilder().put(IndexSettings.INDEX_FLUSH_ON_CLOSE, booleanValue).build());
ensureGreen();
IndicesService indicesService = getInstanceFromNode(IndicesService.class);
IndexService test = indicesService.indexService("test");
IndexShard shard = test.getShardOrNull(0);
assertEquals(booleanValue, shard.getIndexSettings().isFlushOnClose());
}
public void testWriteShardState() throws Exception {
try (NodeEnvironment env = newNodeEnvironment()) {
ShardId id = new ShardId("foo", 1);

9
core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.MockEngineFactoryPlugin;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
@ -66,9 +67,7 @@ import static org.hamcrest.Matchers.notNullValue;
public class CorruptedTranslogIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
// we really need local GW here since this also checks for corruption etc.
// and we need to make sure primaries are not just trashed if we don't have replicas
return pluginList(MockTransportService.TestPlugin.class);
return pluginList(MockTransportService.TestPlugin.class, MockEngineFactoryPlugin.class);
}
public void testCorruptTranslogFiles() throws Exception {
@ -78,8 +77,7 @@ public class CorruptedTranslogIT extends ESIntegTestCase {
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.put("index.refresh_interval", "-1")
.put(MockEngineSupport.FLUSH_ON_CLOSE_RATIO, 0.0d) // never flush - always recover from translog
.put(IndexSettings.INDEX_FLUSH_ON_CLOSE, false) // never flush - always recover from translog
.put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), true) // never flush - always recover from translog
));
ensureYellow();
@ -103,7 +101,6 @@ public class CorruptedTranslogIT extends ESIntegTestCase {
client().prepareSearch("test").setQuery(matchAllQuery()).get();
fail("all shards should be failed due to a corrupted translog");
} catch (SearchPhaseExecutionException e) {
e.printStackTrace();
// Good, all shards should be failed because there is only a
// single shard and its translog is corrupt
}

2
core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
View File

@ -111,7 +111,7 @@ public class RandomExceptionCircuitBreakerIT extends ESIntegTestCase {
.put(indexSettings())
.put(EXCEPTION_TOP_LEVEL_RATIO_KEY, topLevelRate)
.put(EXCEPTION_LOW_LEVEL_RATIO_KEY, lowLevelRate)
.put(MockEngineSupport.WRAP_READER_RATIO, 1.0d);
.put(MockEngineSupport.WRAP_READER_RATIO.getKey(), 1.0d);
logger.info("creating index: [test] using settings: [{}]", settings.build().getAsMap());
client().admin().indices().prepareCreate("test")
.setSettings(settings)

2
core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
View File

@ -92,7 +92,7 @@ public class SearchWithRandomExceptionsIT extends ESIntegTestCase {
.put(indexSettings())
.put(EXCEPTION_TOP_LEVEL_RATIO_KEY, topLevelRate)
.put(EXCEPTION_LOW_LEVEL_RATIO_KEY, lowLevelRate)
.put(MockEngineSupport.WRAP_READER_RATIO, 1.0d);
.put(MockEngineSupport.WRAP_READER_RATIO.getKey(), 1.0d);
logger.info("creating index: [test] using settings: [{}]", settings.build().getAsMap());
assertAcked(prepareCreate("test")
.setSettings(settings)

29
test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
View File

@ -30,6 +30,7 @@ import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.EngineConfig;
@ -50,9 +51,16 @@ import java.util.concurrent.atomic.AtomicBoolean;
*/
public final class MockEngineSupport {
public static final String WRAP_READER_RATIO = "index.engine.mock.random.wrap_reader_ratio";
public static final String READER_WRAPPER_TYPE = "index.engine.mock.random.wrapper";
public static final String FLUSH_ON_CLOSE_RATIO = "index.engine.mock.flush_on_close.ratio";
/**
* Allows tests to wrap an index reader randomly with a given ratio. This is disabled by default ie. <tt>0.0d</tt> since reader wrapping is insanely
* slow if {@link org.apache.lucene.index.AssertingDirectoryReader} is used.
*/
public static final Setting<Double> WRAP_READER_RATIO = Setting.doubleSetting("index.engine.mock.random.wrap_reader_ratio", 0.0d, 0.0d, false, Setting.Scope.INDEX);
/**
* Allows tests to prevent an engine from being flushed on close ie. to test translog recovery...
*/
public static final Setting<Boolean> DISABLE_FLUSH_ON_CLOSE = Setting.boolSetting("index.mock.disable_flush_on_close", false, false, Setting.Scope.INDEX);
private final AtomicBoolean closing = new AtomicBoolean(false);
private final ESLogger logger = Loggers.getLogger(Engine.class);
@ -61,20 +69,24 @@ public final class MockEngineSupport {
private final QueryCachingPolicy filterCachingPolicy;
private final SearcherCloseable searcherCloseable;
private final MockContext mockContext;
private final boolean disableFlushOnClose;
public boolean isFlushOnCloseDisabled() {
return disableFlushOnClose;
}
public static class MockContext {
private final Random random;
private final boolean wrapReader;
private final Class<? extends FilterDirectoryReader> wrapper;
private final Settings indexSettings;
private final double flushOnClose;
public MockContext(Random random, boolean wrapReader, Class<? extends FilterDirectoryReader> wrapper, Settings indexSettings) {
this.random = random;
this.wrapReader = wrapReader;
this.wrapper = wrapper;
this.indexSettings = indexSettings;
flushOnClose = indexSettings.getAsDouble(FLUSH_ON_CLOSE_RATIO, 0.5d);
}
}
@ -85,7 +97,7 @@ public final class MockEngineSupport {
filterCachingPolicy = config.getQueryCachingPolicy();
final long seed = settings.getAsLong(ESIntegTestCase.SETTING_INDEX_SEED, 0l);
Random random = new Random(seed);
final double ratio = settings.getAsDouble(WRAP_READER_RATIO, 0.0d); // DISABLED by default - AssertingDR is crazy slow
final double ratio = WRAP_READER_RATIO.get(settings);
boolean wrapReader = random.nextDouble() < ratio;
if (logger.isTraceEnabled()) {
logger.trace("Using [{}] for shard [{}] seed: [{}] wrapReader: [{}]", this.getClass().getName(), shardId, seed, wrapReader);
@ -93,6 +105,7 @@ public final class MockEngineSupport {
mockContext = new MockContext(random, wrapReader, wrapper, settings);
this.searcherCloseable = new SearcherCloseable();
LuceneTestCase.closeAfterSuite(searcherCloseable); // only one suite closeable per Engine
this.disableFlushOnClose = DISABLE_FLUSH_ON_CLOSE.get(settings);
}
enum CloseAction {
@ -105,9 +118,9 @@ public final class MockEngineSupport {
* Returns the CloseAction to execute on the actual engine. Note this method changes the state on
* the first call and treats subsequent calls as if the engine passed is already closed.
*/
public CloseAction flushOrClose(Engine engine, CloseAction originalAction) throws IOException {
public CloseAction flushOrClose(CloseAction originalAction) throws IOException {
if (closing.compareAndSet(false, true)) { // only do the random thing if we are the first call to this since super.flushOnClose() calls #close() again and then we might end up with a stackoverflow.
if (mockContext.flushOnClose > mockContext.random.nextDouble()) {
if (mockContext.random.nextBoolean()) {
return CloseAction.FLUSH_AND_CLOSE;
} else {
return CloseAction.CLOSE;

16
test/framework/src/main/java/org/elasticsearch/test/engine/MockInternalEngine.java
View File

@ -49,9 +49,9 @@ final class MockInternalEngine extends InternalEngine {
@Override
public void close() throws IOException {
switch (support().flushOrClose(this, MockEngineSupport.CloseAction.CLOSE)) {
switch (support().flushOrClose(MockEngineSupport.CloseAction.CLOSE)) {
case FLUSH_AND_CLOSE:
super.flushAndClose();
flushAndCloseInternal();
break;
case CLOSE:
super.close();
@ -62,16 +62,24 @@ final class MockInternalEngine extends InternalEngine {
@Override
public void flushAndClose() throws IOException {
if (randomizeFlushOnClose) {
switch (support().flushOrClose(this, MockEngineSupport.CloseAction.FLUSH_AND_CLOSE)) {
switch (support().flushOrClose(MockEngineSupport.CloseAction.FLUSH_AND_CLOSE)) {
case FLUSH_AND_CLOSE:
super.flushAndClose();
flushAndCloseInternal();
break;
case CLOSE:
super.close();
break;
}
} else {
flushAndCloseInternal();
}
}
private void flushAndCloseInternal() throws IOException {
if (support().isFlushOnCloseDisabled() == false) {
super.flushAndClose();
} else {
super.close();
}
}