From 3393f9599ea452e7425535e6cf339d4c067b2aed Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 22 Aug 2019 16:40:06 -0400 Subject: [PATCH] Ignore translog retention policy if soft-deletes enabled (#45473) Since #45136, we use soft-deletes instead of translog in peer recovery. There's no need to retain extra translog to increase a chance of operation-based recoveries. This commit ignores the translog retention policy if soft-deletes is enabled so we can discard translog more quickly. Backport of #45473 Relates #45136 --- .../reference/index-modules/translog.asciidoc | 21 +++- .../test/indices.stats/20_translog.yml | 105 +++++++++++++++++- .../elasticsearch/index/IndexSettings.java | 67 +++++++---- .../translog/TruncateTranslogAction.java | 16 ++- .../index/IndexServiceTests.java | 9 +- .../index/IndexSettingsTests.java | 61 ++++++++++ .../index/engine/NoOpEngineTests.java | 5 +- .../index/engine/ReadOnlyEngineTests.java | 9 +- .../IndexLevelReplicationTests.java | 16 ++- .../index/shard/IndexShardTests.java | 16 ++- .../indices/recovery/RecoveryTests.java | 11 +- .../indices/state/OpenCloseIndexIT.java | 23 ++-- .../xpack/deprecation/DeprecationChecks.java | 3 +- .../deprecation/IndexDeprecationChecks.java | 15 +++ .../IndexDeprecationChecksTests.java | 30 +++++ .../index/engine/FrozenIndexTests.java | 9 +- .../test/indices.freeze/20_stats.yml | 8 +- 17 files changed, 351 insertions(+), 73 deletions(-) diff --git a/docs/reference/index-modules/translog.asciidoc b/docs/reference/index-modules/translog.asciidoc index c01671b4ae6..414ac59f0ba 100644 --- a/docs/reference/index-modules/translog.asciidoc +++ b/docs/reference/index-modules/translog.asciidoc @@ -76,12 +76,23 @@ commit point. Defaults to `512mb`. `index.translog.retention.size`:: -The total size of translog files to keep. Keeping more translog files increases -the chance of performing an operation based sync when recovering replicas. If -the translog files are not sufficient, replica recovery will fall back to a -file based sync. Defaults to `512mb` +When soft deletes is disabled (enabled by default in 7.0 or later), +`index.translog.retention.size` controls the total size of translog files to keep. +Keeping more translog files increases the chance of performing an operation based +sync when recovering replicas. If the translog files are not sufficient, +replica recovery will fall back to a file based sync. Defaults to `512mb` + +Both `index.translog.retention.size` and `index.translog.retention.age` should not +be specified unless soft deletes is disabled as they will be ignored. `index.translog.retention.age`:: -The maximum duration for which translog files will be kept. Defaults to `12h`. +When soft deletes is disabled (enabled by default in 7.0 or later), +`index.translog.retention.age` controls the maximum duration for which translog +files to keep. Keeping more translog files increases the chance of performing an +operation based sync when recovering replicas. If the translog files are not sufficient, +replica recovery will fall back to a file based sync. Defaults to `12h` + +Both `index.translog.retention.size` and `index.translog.retention.age` should not +be specified unless soft deletes is disabled as they will be ignored. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml index 92bbe3f5a36..1444e6153fd 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml @@ -1,14 +1,14 @@ --- -setup: +"Translog retention without soft_deletes": - do: indices.create: - index: test + index: test + body: + settings: + soft_deletes.enabled: false - do: cluster.health: wait_for_no_initializing_shards: true - ---- -"Translog retention": - do: indices.stats: metric: [ translog ] @@ -64,6 +64,53 @@ setup: - lte: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size } - match: { indices.test.primaries.translog.uncommitted_operations: 0 } +--- +"Translog retention with soft_deletes": + - skip: + version: " - 7.3.99" + reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4" + - do: + indices.create: + index: test + body: + settings: + soft_deletes.enabled: true + - do: + cluster.health: + wait_for_no_initializing_shards: true + - do: + indices.stats: + metric: [ translog ] + - set: { indices.test.primaries.translog.size_in_bytes: creation_size } + + - do: + index: + index: test + id: 1 + body: { "foo": "bar" } + + - do: + indices.stats: + metric: [ translog ] + - gt: { indices.test.primaries.translog.size_in_bytes: $creation_size } + - match: { indices.test.primaries.translog.operations: 1 } + - match: { indices.test.primaries.translog.uncommitted_operations: 1 } + # call flush twice to sync the global checkpoint after the last operation so that we can have the safe commit + - do: + indices.flush: + index: test + - do: + indices.flush: + index: test + - do: + indices.stats: + metric: [ translog ] + # after flushing we have one empty translog file while an empty index before flushing has two empty translog files. + - lt: { indices.test.primaries.translog.size_in_bytes: $creation_size } + - match: { indices.test.primaries.translog.operations: 0 } + - lt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size } + - match: { indices.test.primaries.translog.uncommitted_operations: 0 } + --- "Translog last modified age stats": - skip: @@ -81,11 +128,20 @@ setup: - gte: { indices.test.primaries.translog.earliest_last_modified_age: 0 } --- -"Translog stats on closed indices": +"Translog stats on closed indices without soft-deletes": - skip: version: " - 7.2.99" reason: "closed indices have translog stats starting version 7.3.0" + - do: + indices.create: + index: test + body: + settings: + soft_deletes.enabled: false + - do: + cluster.health: + wait_for_no_initializing_shards: true - do: index: index: test @@ -123,3 +179,40 @@ setup: forbid_closed_indices: false - match: { indices.test.primaries.translog.operations: 3 } - match: { indices.test.primaries.translog.uncommitted_operations: 0 } + +--- +"Translog stats on closed indices with soft-deletes": + - skip: + version: " - 7.3.99" + reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4" + - do: + indices.create: + index: test + body: + settings: + soft_deletes.enabled: true + - do: + cluster.health: + wait_for_no_initializing_shards: true + - do: + index: + index: test + id: 1 + body: { "foo": "bar" } + - do: + indices.stats: + metric: [ translog ] + - match: { indices.test.primaries.translog.operations: 1 } + - match: { indices.test.primaries.translog.uncommitted_operations: 1 } + - do: + indices.close: + index: test + wait_for_active_shards: 1 + - is_true: acknowledged + - do: + indices.stats: + metric: [ translog ] + expand_wildcards: all + forbid_closed_indices: false + - match: { indices.test.primaries.translog.operations: 0 } + - match: { indices.test.primaries.translog.uncommitted_operations: 0 } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index ca8a24ea93d..a8e629e2aff 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -195,24 +195,6 @@ public final class IndexSettings { new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES), Property.Dynamic, Property.IndexScope); - /** - * Controls how long translog files that are no longer needed for persistence reasons - * will be kept around before being deleted. A longer retention policy is useful to increase - * the chance of ops based recoveries. - **/ - public static final Setting INDEX_TRANSLOG_RETENTION_AGE_SETTING = - Setting.timeSetting("index.translog.retention.age", TimeValue.timeValueHours(12), TimeValue.timeValueMillis(-1), - Property.Dynamic, Property.IndexScope); - - /** - * Controls how many translog files that are no longer needed for persistence reasons - * will be kept around before being deleted. Keeping more files is useful to increase - * the chance of ops based recoveries. - **/ - public static final Setting INDEX_TRANSLOG_RETENTION_SIZE_SETTING = - Setting.byteSizeSetting("index.translog.retention.size", new ByteSizeValue(512, ByteSizeUnit.MB), Property.Dynamic, - Property.IndexScope); - /** * The maximum size of a translog generation. This is independent of the maximum size of * translog operations that have not been flushed. @@ -258,6 +240,27 @@ public final class IndexSettings { Setting.longSetting("index.soft_deletes.retention.operations", 0, 0, Property.IndexScope, Property.Dynamic); + /** + * Controls how long translog files that are no longer needed for persistence reasons + * will be kept around before being deleted. Keeping more files is useful to increase + * the chance of ops based recoveries for indices with soft-deletes disabled. + * This setting will be ignored if soft-deletes is enabled. + **/ + public static final Setting INDEX_TRANSLOG_RETENTION_AGE_SETTING = + Setting.timeSetting("index.translog.retention.age", + settings -> INDEX_SOFT_DELETES_SETTING.get(settings) ? TimeValue.MINUS_ONE : TimeValue.timeValueHours(12), TimeValue.MINUS_ONE, + Property.Dynamic, Property.IndexScope); + + /** + * Controls how many translog files that are no longer needed for persistence reasons + * will be kept around before being deleted. Keeping more files is useful to increase + * the chance of ops based recoveries for indices with soft-deletes disabled. + * This setting will be ignored if soft-deletes is enabled. + **/ + public static final Setting INDEX_TRANSLOG_RETENTION_SIZE_SETTING = + Setting.byteSizeSetting("index.translog.retention.size", settings -> INDEX_SOFT_DELETES_SETTING.get(settings) ? "-1" : "512MB", + Property.Dynamic, Property.IndexScope); + /** * Controls the maximum length of time since a retention lease is created or renewed before it is considered expired. */ @@ -466,8 +469,6 @@ public final class IndexSettings { syncInterval = INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.get(settings); refreshInterval = scopedSettings.get(INDEX_REFRESH_INTERVAL_SETTING); flushThresholdSize = scopedSettings.get(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING); - translogRetentionAge = scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING); - translogRetentionSize = scopedSettings.get(INDEX_TRANSLOG_RETENTION_SIZE_SETTING); generationThresholdSize = scopedSettings.get(INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING); mergeSchedulerConfig = new MergeSchedulerConfig(this); gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis(); @@ -493,6 +494,8 @@ public final class IndexSettings { this.indexSortConfig = new IndexSortConfig(this); searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER); defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE); + setTranslogRetentionAge(scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING)); + setTranslogRetentionSize(scopedSettings.get(INDEX_TRANSLOG_RETENTION_SIZE_SETTING)); scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio); scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING, @@ -553,11 +556,21 @@ public final class IndexSettings { } private void setTranslogRetentionSize(ByteSizeValue byteSizeValue) { - this.translogRetentionSize = byteSizeValue; + if (softDeleteEnabled && byteSizeValue.getBytes() >= 0) { + // ignore the translog retention settings if soft-deletes enabled + this.translogRetentionSize = new ByteSizeValue(-1); + } else { + this.translogRetentionSize = byteSizeValue; + } } private void setTranslogRetentionAge(TimeValue age) { - this.translogRetentionAge = age; + if (softDeleteEnabled && age.millis() >= 0) { + // ignore the translog retention settings if soft-deletes enabled + this.translogRetentionAge = TimeValue.MINUS_ONE; + } else { + this.translogRetentionAge = age; + } } private void setGenerationThresholdSize(final ByteSizeValue generationThresholdSize) { @@ -734,13 +747,19 @@ public final class IndexSettings { /** * Returns the transaction log retention size which controls how much of the translog is kept around to allow for ops based recoveries */ - public ByteSizeValue getTranslogRetentionSize() { return translogRetentionSize; } + public ByteSizeValue getTranslogRetentionSize() { + assert softDeleteEnabled == false || translogRetentionSize.getBytes() == -1L : translogRetentionSize; + return translogRetentionSize; + } /** * Returns the transaction log retention age which controls the maximum age (time from creation) that translog files will be kept * around */ - public TimeValue getTranslogRetentionAge() { return translogRetentionAge; } + public TimeValue getTranslogRetentionAge() { + assert softDeleteEnabled == false || translogRetentionAge.millis() == -1L : translogRetentionSize; + return translogRetentionAge; + } /** * Returns the generation threshold size. As sequence numbers can cause multiple generations to diff --git a/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java b/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java index 7cf165a5b11..55a24d30991 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java @@ -177,11 +177,19 @@ public class TruncateTranslogAction { final TranslogConfig translogConfig = new TranslogConfig(shardPath.getShardId(), translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE); long primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardPath.getShardId().id()); - final TranslogDeletionPolicy translogDeletionPolicy = - new TranslogDeletionPolicy(indexSettings.getTranslogRetentionSize().getBytes(), - indexSettings.getTranslogRetentionAge().getMillis()); + // We open translog to check for corruption, do not clean anything. + final TranslogDeletionPolicy retainAllTranslogPolicy = new TranslogDeletionPolicy(Long.MAX_VALUE, Long.MAX_VALUE) { + @Override + long minTranslogGenRequired(List readers, TranslogWriter writer) { + long minGen = writer.generation; + for (TranslogReader reader : readers) { + minGen = Math.min(reader.generation, minGen); + } + return minGen; + } + }; try (Translog translog = new Translog(translogConfig, translogUUID, - translogDeletionPolicy, () -> translogGlobalCheckpoint, () -> primaryTerm, seqNo -> {}); + retainAllTranslogPolicy, () -> translogGlobalCheckpoint, () -> primaryTerm, seqNo -> {}); Translog.Snapshot snapshot = translog.newSnapshot()) { //noinspection StatementWithEmptyBody we are just checking that we can iterate through the whole snapshot while (snapshot.next() != null) { diff --git a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java index a4256c7e0cc..afa74512e31 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexServiceTests.java @@ -396,15 +396,20 @@ public class IndexServiceTests extends ESSingleNodeTestCase { final Path translogPath = translog.getConfig().getTranslogPath(); final String translogUuid = translog.getTranslogUUID(); + int translogOps = 0; final int numDocs = scaledRandomIntBetween(10, 100); for (int i = 0; i < numDocs; i++) { client().prepareIndex().setIndex(indexName).setId(String.valueOf(i)).setSource("{\"foo\": \"bar\"}", XContentType.JSON).get(); + translogOps++; if (randomBoolean()) { client().admin().indices().prepareFlush(indexName).get(); + if (indexService.getIndexSettings().isSoftDeleteEnabled()) { + translogOps = 0; + } } } - assertThat(translog.totalOperations(), equalTo(numDocs)); - assertThat(translog.stats().estimatedNumberOfOperations(), equalTo(numDocs)); + assertThat(translog.totalOperations(), equalTo(translogOps)); + assertThat(translog.stats().estimatedNumberOfOperations(), equalTo(translogOps)); assertAcked(client().admin().indices().prepareClose("test").setWaitForActiveShards(ActiveShardCount.DEFAULT)); indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(indexService.index()); diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index b3e6557b187..c79c9268f24 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.translog.Translog; @@ -577,4 +578,64 @@ public class IndexSettingsTests extends ESTestCase { assertFalse(IndexSettings.INDEX_SOFT_DELETES_SETTING.get(settings)); } } + + public void testIgnoreTranslogRetentionSettingsIfSoftDeletesEnabled() { + Settings.Builder settings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT)); + if (randomBoolean()) { + settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue()); + } + if (randomBoolean()) { + settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 1024) + "b"); + } + IndexMetaData metaData = newIndexMeta("index", settings.build()); + IndexSettings indexSettings = new IndexSettings(metaData, Settings.EMPTY); + assertThat(indexSettings.getTranslogRetentionAge().millis(), equalTo(-1L)); + assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L)); + + Settings.Builder newSettings = Settings.builder().put(settings.build()); + if (randomBoolean()) { + newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue()); + } + if (randomBoolean()) { + newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 1024) + "b"); + } + indexSettings.updateIndexMetaData(newIndexMeta("index", newSettings.build())); + assertThat(indexSettings.getTranslogRetentionAge().millis(), equalTo(-1L)); + assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L)); + } + + public void testUpdateTranslogRetentionSettingsWithSoftDeletesDisabled() { + Settings.Builder settings = Settings.builder() + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT); + + TimeValue ageSetting = TimeValue.timeValueHours(12); + if (randomBoolean()) { + ageSetting = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomIntBetween(0, 10000)); + settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), ageSetting); + } + ByteSizeValue sizeSetting = new ByteSizeValue(512, ByteSizeUnit.MB); + if (randomBoolean()) { + sizeSetting = randomBoolean() ? new ByteSizeValue(-1) : new ByteSizeValue(randomIntBetween(0, 1024)); + settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), sizeSetting); + } + IndexMetaData metaData = newIndexMeta("index", settings.build()); + IndexSettings indexSettings = new IndexSettings(metaData, Settings.EMPTY); + assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting)); + assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting)); + + Settings.Builder newSettings = Settings.builder().put(settings.build()); + if (randomBoolean()) { + ageSetting = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomIntBetween(0, 10000)); + newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), ageSetting); + } + if (randomBoolean()) { + sizeSetting = randomBoolean() ? new ByteSizeValue(-1) : new ByteSizeValue(randomIntBetween(0, 1024)); + newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), sizeSetting); + } + indexSettings.updateIndexMetaData(newIndexMeta("index", newSettings.build())); + assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting)); + assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java index bd934f683fb..623bbe0ec50 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java @@ -169,13 +169,14 @@ public class NoOpEngineTests extends EngineTestCase { tracker.updateFromMaster(1L, Collections.singleton(allocationId.getId()), table); tracker.activatePrimaryMode(SequenceNumbers.NO_OPS_PERFORMED); + boolean softDeleteEnabled = engine.config().getIndexSettings().isSoftDeleteEnabled(); final int numDocs = scaledRandomIntBetween(10, 3000); for (int i = 0; i < numDocs; i++) { engine.index(indexForDoc(createParsedDoc(Integer.toString(i), null))); + tracker.updateLocalCheckpoint(allocationId.getId(), i); if (rarely()) { engine.flush(); } - tracker.updateLocalCheckpoint(allocationId.getId(), i); } engine.flush(true, true); @@ -195,7 +196,7 @@ public class NoOpEngineTests extends EngineTestCase { } assertThat(Translog.readMinTranslogGeneration(translogPath, translogUuid), equalTo(minFileGeneration)); - assertThat(noOpEngine.getTranslogStats().estimatedNumberOfOperations(), equalTo(numDocs)); + assertThat(noOpEngine.getTranslogStats().estimatedNumberOfOperations(), equalTo(softDeleteEnabled ? 0 : numDocs)); assertThat(noOpEngine.getTranslogStats().getUncommittedOperations(), equalTo(0)); noOpEngine.trimUnreferencedTranslogFiles(); diff --git a/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java index 506be95c225..c01aca80825 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java @@ -250,7 +250,7 @@ public class ReadOnlyEngineTests extends EngineTestCase { try (Store store = createStore()) { final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get); - + final boolean softDeletesEnabled = config.getIndexSettings().isSoftDeleteEnabled(); final int numDocs = frequently() ? scaledRandomIntBetween(10, 200) : 0; int uncommittedDocs = 0; @@ -259,16 +259,17 @@ public class ReadOnlyEngineTests extends EngineTestCase { ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null); engine.index(new Engine.Index(newUid(doc), doc, i, primaryTerm.get(), 1, null, Engine.Operation.Origin.REPLICA, System.nanoTime(), -1, false, SequenceNumbers.UNASSIGNED_SEQ_NO, 0)); + globalCheckpoint.set(i); if (rarely()) { engine.flush(); uncommittedDocs = 0; } else { uncommittedDocs += 1; } - globalCheckpoint.set(i); } - assertThat(engine.getTranslogStats().estimatedNumberOfOperations(), equalTo(numDocs)); + assertThat(engine.getTranslogStats().estimatedNumberOfOperations(), + equalTo(softDeletesEnabled ? uncommittedDocs : numDocs)); assertThat(engine.getTranslogStats().getUncommittedOperations(), equalTo(uncommittedDocs)); assertThat(engine.getTranslogStats().getTranslogSizeInBytes(), greaterThan(0L)); assertThat(engine.getTranslogStats().getUncommittedSizeInBytes(), greaterThan(0L)); @@ -278,7 +279,7 @@ public class ReadOnlyEngineTests extends EngineTestCase { } try (ReadOnlyEngine readOnlyEngine = new ReadOnlyEngine(config, null, null, true, Function.identity())) { - assertThat(readOnlyEngine.getTranslogStats().estimatedNumberOfOperations(), equalTo(numDocs)); + assertThat(readOnlyEngine.getTranslogStats().estimatedNumberOfOperations(), equalTo(softDeletesEnabled ? 0 : numDocs)); assertThat(readOnlyEngine.getTranslogStats().getUncommittedOperations(), equalTo(0)); assertThat(readOnlyEngine.getTranslogStats().getTranslogSizeInBytes(), greaterThan(0L)); assertThat(readOnlyEngine.getTranslogStats().getUncommittedSizeInBytes(), greaterThan(0L)); diff --git a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java index 3809d002483..2817c51d33a 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java @@ -467,7 +467,12 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase shards.startReplicas(nReplica); for (IndexShard shard : shards) { try (Translog.Snapshot snapshot = getTranslog(shard).newSnapshot()) { - assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps)); + // we flush at the end of peer recovery + if (shard.routingEntry().primary() || shard.indexSettings().isSoftDeleteEnabled() == false) { + assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps)); + } else { + assertThat(snapshot.totalOperations(), equalTo(0)); + } } try (Translog.Snapshot snapshot = shard.getHistoryOperations("test", 0)) { assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps)); @@ -476,11 +481,16 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase // the failure replicated directly from the replication channel. indexResp = shards.index(new IndexRequest(index.getName(), "type", "any").source("{}", XContentType.JSON)); assertThat(indexResp.getFailure().getCause(), equalTo(indexException)); - expectedTranslogOps.add(new Translog.NoOp(1, primaryTerm, indexException.toString())); + Translog.NoOp noop2 = new Translog.NoOp(1, primaryTerm, indexException.toString()); + expectedTranslogOps.add(noop2); for (IndexShard shard : shards) { try (Translog.Snapshot snapshot = getTranslog(shard).newSnapshot()) { - assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps)); + if (shard.routingEntry().primary() || shard.indexSettings().isSoftDeleteEnabled() == false) { + assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps)); + } else { + assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(Collections.singletonList(noop2))); + } } try (Translog.Snapshot snapshot = shard.getHistoryOperations("test", 0)) { assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps)); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index b3799e3fe76..c0bb63e2466 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2136,11 +2136,13 @@ public class IndexShardTests extends IndexShardTestCase { /* This test just verifies that we fill up local checkpoint up to max seen seqID on primary recovery */ public void testRecoverFromStoreWithNoOps() throws IOException { - final IndexShard shard = newStartedShard(true); + final Settings settings = Settings.builder() + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).build(); + final IndexShard shard = newStartedShard(true, settings); indexDoc(shard, "_doc", "0"); indexDoc(shard, "_doc", "1"); // start a replica shard and index the second doc - final IndexShard otherShard = newStartedShard(false); + final IndexShard otherShard = newStartedShard(false, settings); updateMappings(otherShard, shard.indexSettings().getIndexMetaData()); SourceToParse sourceToParse = new SourceToParse(shard.shardId().getIndexName(), "_doc", "1", new BytesArray("{}"), XContentType.JSON); @@ -2179,7 +2181,7 @@ public class IndexShardTests extends IndexShardTestCase { newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); assertTrue(newShard.recoverFromStore()); try (Translog.Snapshot snapshot = getTranslog(newShard).newSnapshot()) { - assertThat(snapshot.totalOperations(), equalTo(2)); + assertThat(snapshot.totalOperations(), equalTo(newShard.indexSettings.isSoftDeleteEnabled() ? 0 : 2)); } } closeShards(newShard, shard); @@ -3801,7 +3803,13 @@ public class IndexShardTests extends IndexShardTestCase { engineResetLatch.await(); assertThat(getShardDocUIDs(shard), equalTo(docBelowGlobalCheckpoint)); assertThat(shard.seqNoStats().getMaxSeqNo(), equalTo(globalCheckpoint)); - assertThat(shard.translogStats().estimatedNumberOfOperations(), equalTo(translogStats.estimatedNumberOfOperations())); + if (shard.indexSettings.isSoftDeleteEnabled()) { + // we might have trimmed some operations if the translog retention policy is ignored (when soft-deletes enabled). + assertThat(shard.translogStats().estimatedNumberOfOperations(), + lessThanOrEqualTo(translogStats.estimatedNumberOfOperations())); + } else { + assertThat(shard.translogStats().estimatedNumberOfOperations(), equalTo(translogStats.estimatedNumberOfOperations())); + } assertThat(shard.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(maxSeqNoBeforeRollback)); done.set(true); thread.join(); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java index 3b338ff824f..b340d8c52be 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java @@ -82,7 +82,7 @@ public class RecoveryTests extends ESIndexLevelReplicationTestCase { shards.startAll(); final IndexShard replica = shards.getReplicas().get(0); boolean softDeletesEnabled = replica.indexSettings().isSoftDeleteEnabled(); - assertThat(getTranslog(replica).totalOperations(), equalTo(softDeletesEnabled ? moreDocs : docs + moreDocs)); + assertThat(getTranslog(replica).totalOperations(), equalTo(softDeletesEnabled ? 0 : docs + moreDocs)); shards.assertAllEqual(docs + moreDocs); } } @@ -298,7 +298,7 @@ public class RecoveryTests extends ESIndexLevelReplicationTestCase { // file based recovery should be made assertThat(newReplica.recoveryState().getIndex().fileDetails(), not(empty())); boolean softDeletesEnabled = replica.indexSettings().isSoftDeleteEnabled(); - assertThat(getTranslog(newReplica).totalOperations(), equalTo(softDeletesEnabled ? nonFlushedDocs : numDocs)); + assertThat(getTranslog(newReplica).totalOperations(), equalTo(softDeletesEnabled ? 0 : numDocs)); // history uuid was restored assertThat(newReplica.getHistoryUUID(), equalTo(historyUUID)); @@ -385,7 +385,12 @@ public class RecoveryTests extends ESIndexLevelReplicationTestCase { shards.recoverReplica(newReplica); try (Translog.Snapshot snapshot = getTranslog(newReplica).newSnapshot()) { - assertThat("Sequence based recovery should keep existing translog", snapshot, SnapshotMatchers.size(initDocs + moreDocs)); + if (newReplica.indexSettings().isSoftDeleteEnabled()) { + assertThat(snapshot.totalOperations(), equalTo(0)); + } else { + assertThat("Sequence based recovery should keep existing translog", + snapshot, SnapshotMatchers.size(initDocs + moreDocs)); + } } assertThat(newReplica.recoveryState().getTranslog().recoveredOperations(), equalTo(uncommittedDocs + moreDocs)); assertThat(newReplica.recoveryState().getIndex().fileDetails(), empty()); diff --git a/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java b/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java index ea447cc998b..e98fedbfcd7 100644 --- a/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java +++ b/server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -352,11 +353,13 @@ public class OpenCloseIndexIT extends ESIntegTestCase { } } - public void testTranslogStats() { + public void testTranslogStats() throws Exception { final String indexName = "test"; createIndex(indexName, Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .build()); + boolean softDeletesEnabled = IndexSettings.INDEX_SOFT_DELETES_SETTING.get( + client().admin().indices().prepareGetSettings(indexName).get().getIndexToSettings().get(indexName)); final int nbDocs = randomIntBetween(0, 50); int uncommittedOps = 0; @@ -372,17 +375,23 @@ public class OpenCloseIndexIT extends ESIntegTestCase { } } - IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().setTranslog(true).get(); - assertThat(stats.getIndex(indexName), notNullValue()); - assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(nbDocs)); - assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(uncommittedOps)); + final int uncommittedTranslogOps = uncommittedOps; + assertBusy(() -> { + IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().setTranslog(true).get(); + assertThat(stats.getIndex(indexName), notNullValue()); + assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo( + softDeletesEnabled ? uncommittedTranslogOps : nbDocs)); + assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(uncommittedTranslogOps)); + }); assertAcked(client().admin().indices().prepareClose("test").setWaitForActiveShards(ActiveShardCount.ONE)); IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN_CLOSED; - stats = client().admin().indices().prepareStats(indexName).setIndicesOptions(indicesOptions).clear().setTranslog(true).get(); + IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName) + .setIndicesOptions(indicesOptions).clear().setTranslog(true).get(); assertThat(stats.getIndex(indexName), notNullValue()); - assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(nbDocs)); + assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), + equalTo(softDeletesEnabled ? 0 : nbDocs)); assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(0)); } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index 194c412ffe2..dfb344f829d 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -49,7 +49,8 @@ public class DeprecationChecks { IndexDeprecationChecks::oldIndicesCheck, IndexDeprecationChecks::tooManyFieldsCheck, IndexDeprecationChecks::chainedMultiFieldsCheck, - IndexDeprecationChecks::deprecatedDateTimeFormat + IndexDeprecationChecks::deprecatedDateTimeFormat, + IndexDeprecationChecks::translogRetentionSettingCheck )); static List> ML_SETTINGS_CHECKS = diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java index ead1b665ba7..38a0d0ad5cc 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java @@ -231,4 +231,19 @@ public class IndexDeprecationChecks { return fields; } + + static DeprecationIssue translogRetentionSettingCheck(IndexMetaData indexMetaData) { + final boolean softDeletesEnabled = IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexMetaData.getSettings()); + if (softDeletesEnabled) { + if (IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.exists(indexMetaData.getSettings()) + || IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.exists(indexMetaData.getSettings())) { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, + "translog retention settings are ignored", + "https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules-translog.html", + "translog retention settings [index.translog.retention.size] and [index.translog.retention.age] are ignored " + + "because translog is no longer used in peer recoveries with soft-deletes enabled (default in 7.0 or later)"); + } + } + return null; + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java index 0655da1db08..e32e24aeafb 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.joda.JodaDeprecationPatterns; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; @@ -27,6 +28,8 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.deprecation.DeprecationChecks.INDEX_SETTINGS_CHECKS; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.collection.IsIterableContainingInOrder.contains; public class IndexDeprecationChecksTests extends ESTestCase { public void testOldIndicesCheck() { @@ -382,4 +385,31 @@ public class IndexDeprecationChecksTests extends ESTestCase { } mappingBuilder.endObject(); } + + public void testTranslogRetentionSettings() { + Settings.Builder settings = settings(Version.CURRENT); + settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue()); + settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 1024) + "b"); + IndexMetaData indexMetaData = IndexMetaData.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetaData)); + assertThat(issues, contains( + new DeprecationIssue(DeprecationIssue.Level.WARNING, + "translog retention settings are ignored", + "https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules-translog.html", + "translog retention settings [index.translog.retention.size] and [index.translog.retention.age] are ignored " + + "because translog is no longer used in peer recoveries with soft-deletes enabled (default in 7.0 or later)") + )); + } + + public void testDefaultTranslogRetentionSettings() { + Settings.Builder settings = settings(Version.CURRENT); + if (randomBoolean()) { + settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue()); + settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 1024) + "b"); + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false); + } + IndexMetaData indexMetaData = IndexMetaData.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build(); + List issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetaData)); + assertThat(issues, empty()); + } } diff --git a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java index 3e93062cf15..812a3800527 100644 --- a/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java +++ b/x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java @@ -411,7 +411,7 @@ public class FrozenIndexTests extends ESSingleNodeTestCase { public void testTranslogStats() throws Exception { final String indexName = "test"; - createIndex(indexName, Settings.builder() + IndexService indexService = createIndex(indexName, Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .build()); @@ -420,7 +420,6 @@ public class FrozenIndexTests extends ESSingleNodeTestCase { for (long i = 0; i < nbDocs; i++) { final IndexResponse indexResponse = client().prepareIndex(indexName, "_doc", Long.toString(i)).setSource("field", i).get(); assertThat(indexResponse.status(), is(RestStatus.CREATED)); - if (rarely()) { client().admin().indices().prepareFlush(indexName).get(); uncommittedOps = 0; @@ -431,7 +430,8 @@ public class FrozenIndexTests extends ESSingleNodeTestCase { IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().setTranslog(true).get(); assertThat(stats.getIndex(indexName), notNullValue()); - assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(nbDocs)); + assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo( + indexService.getIndexSettings().isSoftDeleteEnabled() ? uncommittedOps : nbDocs)); assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(uncommittedOps)); assertAcked(new XPackClient(client()).freeze(new FreezeRequest(indexName))); @@ -440,7 +440,8 @@ public class FrozenIndexTests extends ESSingleNodeTestCase { IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN_CLOSED; stats = client().admin().indices().prepareStats(indexName).setIndicesOptions(indicesOptions).clear().setTranslog(true).get(); assertThat(stats.getIndex(indexName), notNullValue()); - assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(nbDocs)); + assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), + equalTo(indexService.getIndexSettings().isSoftDeleteEnabled() ? 0 : nbDocs)); assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(0)); } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/20_stats.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/20_stats.yml index e73c7793022..b189fba7c79 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/20_stats.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/20_stats.yml @@ -10,8 +10,8 @@ setup: --- "Translog stats on frozen indices": - skip: - version: " - 7.2.99" - reason: "frozen indices have translog stats starting version 7.3.0" + version: " - 7.3.99" + reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4" - do: index: @@ -47,7 +47,7 @@ setup: - do: indices.stats: metric: [ translog ] - - match: { indices.test.primaries.translog.operations: 3 } + - match: { indices.test.primaries.translog.operations: 0 } - match: { indices.test.primaries.translog.uncommitted_operations: 0 } # unfreeze index @@ -60,5 +60,5 @@ setup: - do: indices.stats: metric: [ translog ] - - match: { indices.test.primaries.translog.operations: 3 } + - match: { indices.test.primaries.translog.operations: 0 } - match: { indices.test.primaries.translog.uncommitted_operations: 0 }