TEST: Randomize soft-deletes settings (#31585)
This change allows us to test our system with/without soft-deletes enabled.
This commit is contained in:
parent
52d9012d31
commit
540da7399c
|
@ -3832,6 +3832,7 @@ public class InternalEngineTests extends EngineTestCase {
|
|||
assertThat(noOp.seqNo(), equalTo((long) (maxSeqNo + 2)));
|
||||
assertThat(noOp.primaryTerm(), equalTo(primaryTerm.get()));
|
||||
assertThat(noOp.reason(), equalTo(reason));
|
||||
if (engine.engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
|
||||
MapperService mapperService = createMapperService("test");
|
||||
List<Translog.Operation> operationsFromLucene = readAllOperationsInLucene(noOpEngine, mapperService);
|
||||
assertThat(operationsFromLucene, hasSize(maxSeqNo + 2 - localCheckpoint)); // fills n gap and 2 manual noop.
|
||||
|
@ -3839,6 +3840,7 @@ public class InternalEngineTests extends EngineTestCase {
|
|||
assertThat(operationsFromLucene.get(i), equalTo(new Translog.NoOp(localCheckpoint + 1 + i, primaryTerm.get(), "filling gaps")));
|
||||
}
|
||||
assertConsistentHistoryBetweenTranslogAndLuceneIndex(noOpEngine, mapperService);
|
||||
}
|
||||
} finally {
|
||||
IOUtils.close(noOpEngine);
|
||||
}
|
||||
|
@ -3895,9 +3897,11 @@ public class InternalEngineTests extends EngineTestCase {
|
|||
engine.forceMerge(randomBoolean(), between(1, 10), randomBoolean(), false, false);
|
||||
}
|
||||
}
|
||||
if (engine.engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
|
||||
List<Translog.Operation> operations = readAllOperationsInLucene(engine, createMapperService("test"));
|
||||
assertThat(operations, hasSize(numOps));
|
||||
}
|
||||
}
|
||||
|
||||
public void testMinGenerationForSeqNo() throws IOException, BrokenBarrierException, InterruptedException {
|
||||
engine.close();
|
||||
|
@ -4837,9 +4841,15 @@ public class InternalEngineTests extends EngineTestCase {
|
|||
private void assertOperationHistoryInLucene(List<Engine.Operation> operations) throws IOException {
|
||||
final MergePolicy keepSoftDeleteDocsMP = new SoftDeletesRetentionMergePolicy(
|
||||
Lucene.SOFT_DELETE_FIELD, () -> new MatchAllDocsQuery(), engine.config().getMergePolicy());
|
||||
Settings.Builder settings = Settings.builder()
|
||||
.put(defaultSettings.getSettings())
|
||||
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
|
||||
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), randomLongBetween(0, 10));
|
||||
final IndexMetaData indexMetaData = IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build();
|
||||
final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetaData);
|
||||
Set<Long> expectedSeqNos = new HashSet<>();
|
||||
try (Store store = createStore();
|
||||
Engine engine = createEngine(config(defaultSettings, store, createTempDir(), keepSoftDeleteDocsMP, null))) {
|
||||
Engine engine = createEngine(config(indexSettings, store, createTempDir(), keepSoftDeleteDocsMP, null))) {
|
||||
for (Engine.Operation op : operations) {
|
||||
if (op instanceof Engine.Index) {
|
||||
Engine.IndexResult indexResult = engine.index((Engine.Index) op);
|
||||
|
|
|
@ -19,7 +19,9 @@
|
|||
|
||||
package org.elasticsearch.index.engine;
|
||||
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.core.internal.io.IOUtils;
|
||||
import org.elasticsearch.index.IndexSettings;
|
||||
import org.elasticsearch.index.VersionType;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
import org.elasticsearch.index.mapper.ParsedDocument;
|
||||
|
@ -49,6 +51,13 @@ public class LuceneChangesSnapshotTests extends EngineTestCase {
|
|||
mapperService = createMapperService("test");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Settings indexSettings() {
|
||||
return Settings.builder().put(super.indexSettings())
|
||||
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) // always enable soft-deletes
|
||||
.build();
|
||||
}
|
||||
|
||||
public void testBasics() throws Exception {
|
||||
long fromSeqNo = randomNonNegativeLong();
|
||||
long toSeqNo = randomLongBetween(fromSeqNo, Long.MAX_VALUE);
|
||||
|
|
|
@ -2333,7 +2333,8 @@ public class IndexShardTests extends IndexShardTestCase {
|
|||
public void testDocStats() throws IOException {
|
||||
IndexShard indexShard = null;
|
||||
try {
|
||||
indexShard = newStartedShard();
|
||||
indexShard = newStartedShard(
|
||||
Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), 0).build());
|
||||
final long numDocs = randomIntBetween(2, 32); // at least two documents so we have docs to delete
|
||||
// Delete at least numDocs/10 documents otherwise the number of deleted docs will be below 10%
|
||||
// and forceMerge will refuse to expunge deletes
|
||||
|
|
|
@ -118,6 +118,7 @@ public class IndexStatsIT extends ESIntegTestCase {
|
|||
return Settings.builder().put(super.indexSettings())
|
||||
.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)
|
||||
.put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), true)
|
||||
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), 0)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
|
|
@ -155,6 +155,20 @@ public abstract class EngineTestCase extends ESTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
protected Settings indexSettings() {
|
||||
// TODO randomize more settings
|
||||
return Settings.builder()
|
||||
.put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), "1h") // make sure this doesn't kick in on us
|
||||
.put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codecName)
|
||||
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||
.put(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.getKey(),
|
||||
between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY)))
|
||||
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
|
||||
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(),
|
||||
randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000))
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
|
@ -169,13 +183,7 @@ public abstract class EngineTestCase extends ESTestCase {
|
|||
} else {
|
||||
codecName = "default";
|
||||
}
|
||||
defaultSettings = IndexSettingsModule.newIndexSettings("test", Settings.builder()
|
||||
.put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), "1h") // make sure this doesn't kick in on us
|
||||
.put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codecName)
|
||||
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||
.put(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.getKey(),
|
||||
between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY)))
|
||||
.build()); // TODO randomize more settings
|
||||
defaultSettings = IndexSettingsModule.newIndexSettings("test", indexSettings());
|
||||
threadPool = new TestThreadPool(getClass().getName());
|
||||
store = createStore();
|
||||
storeReplica = createStore();
|
||||
|
|
|
@ -59,6 +59,7 @@ import org.elasticsearch.common.lease.Releasables;
|
|||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.IndexSettings;
|
||||
import org.elasticsearch.index.engine.EngineFactory;
|
||||
import org.elasticsearch.index.engine.InternalEngineFactory;
|
||||
import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction;
|
||||
|
@ -120,6 +121,9 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
|
|||
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas)
|
||||
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
|
||||
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
|
||||
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(),
|
||||
randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000))
|
||||
.put(indexSettings)
|
||||
.build();
|
||||
IndexMetaData.Builder metaData = IndexMetaData.builder(index.getName())
|
||||
|
|
|
@ -195,6 +195,9 @@ public abstract class IndexShardTestCase extends ESTestCase {
|
|||
Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
|
||||
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
|
||||
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
|
||||
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(),
|
||||
randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000))
|
||||
.put(settings)
|
||||
.build();
|
||||
IndexMetaData.Builder metaData = IndexMetaData.builder(shardRouting.getIndexName())
|
||||
|
@ -365,6 +368,14 @@ public abstract class IndexShardTestCase extends ESTestCase {
|
|||
return newStartedShard(randomBoolean());
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new empty shard and starts it
|
||||
* @param settings the settings to use for this shard
|
||||
*/
|
||||
protected IndexShard newStartedShard(Settings settings) throws IOException {
|
||||
return newStartedShard(randomBoolean(), settings, new InternalEngineFactory());
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new empty shard and starts it.
|
||||
*
|
||||
|
|
|
@ -718,6 +718,10 @@ public abstract class ESIntegTestCase extends ESTestCase {
|
|||
}
|
||||
// always default delayed allocation to 0 to make sure we have tests are not delayed
|
||||
builder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0);
|
||||
builder.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
|
||||
if (randomBoolean()) {
|
||||
builder.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000));
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
|
|
|
@ -41,6 +41,7 @@ import org.elasticsearch.env.Environment;
|
|||
import org.elasticsearch.env.NodeEnvironment;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.IndexService;
|
||||
import org.elasticsearch.index.IndexSettings;
|
||||
import org.elasticsearch.indices.IndicesService;
|
||||
import org.elasticsearch.node.MockNode;
|
||||
import org.elasticsearch.node.Node;
|
||||
|
@ -86,6 +87,14 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
|
|||
.setOrder(0)
|
||||
.setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
|
||||
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get();
|
||||
client().admin().indices()
|
||||
.preparePutTemplate("random-soft-deletes-template")
|
||||
.setPatterns(Collections.singletonList("*"))
|
||||
.setOrder(0)
|
||||
.setSettings(Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean())
|
||||
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(),
|
||||
randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000))
|
||||
).get();
|
||||
}
|
||||
|
||||
private static void stopNode() throws IOException {
|
||||
|
|
Loading…
Reference in New Issue