Ignore translog retention policy if soft-deletes enabled (#45473)

Since #45136, we use soft-deletes instead of translog in peer recovery.
There's no need to retain extra translog to increase a chance of
operation-based recoveries. This commit ignores the translog retention
policy if soft-deletes is enabled so we can discard translog more
quickly.

Backport of #45473
Relates #45136

docs/reference/index-modules/translog.asciidoc

@@ -76,12 +76,23 @@ commit point. Defaults to `512mb`.
 
 `index.translog.retention.size`::
 
-The total size of translog files to keep. Keeping more translog files increases
+When soft deletes is disabled (enabled by default in 7.0 or later),
-the chance of performing an operation based sync when recovering replicas. If
+`index.translog.retention.size` controls the total size of translog files to keep.
-the translog files are not sufficient, replica recovery will fall back to a
+Keeping more translog files increases the chance of performing an operation based
-file based sync. Defaults to `512mb`
+sync when recovering replicas. If the translog files are not sufficient,
+replica recovery will fall back to a file based sync. Defaults to `512mb`
+
+Both `index.translog.retention.size` and `index.translog.retention.age` should not
+be specified unless soft deletes is disabled as they will be ignored.
 
 
 `index.translog.retention.age`::
 
-The maximum duration for which translog files will be kept. Defaults to `12h`.
+When soft deletes is disabled (enabled by default in 7.0 or later),
+`index.translog.retention.age` controls the maximum duration for which translog
+files to keep. Keeping more translog files increases the chance of performing an
+operation based sync when recovering replicas. If the translog files are not sufficient,
+replica recovery will fall back to a file based sync. Defaults to `12h`
+
+Both `index.translog.retention.size` and `index.translog.retention.age` should not
+be specified unless soft deletes is disabled as they will be ignored.

rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml

@@ -1,14 +1,14 @@
 ---
-setup:
+"Translog retention without soft_deletes":
   - do:
       indices.create:
-          index: test
+        index: test
+        body:
+          settings:
+            soft_deletes.enabled: false
   - do:
       cluster.health:
         wait_for_no_initializing_shards: true
-
----
-"Translog retention":
   - do:
       indices.stats:
         metric: [ translog ]
@@ -64,6 +64,53 @@ setup:
   - lte: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size }
   - match: { indices.test.primaries.translog.uncommitted_operations: 0 }
 
+---
+"Translog retention with soft_deletes":
+  - skip:
+      version: " - 7.3.99"
+      reason:  "start ignoring translog retention policy with soft-deletes enabled in 7.4"
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            soft_deletes.enabled: true
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+  - do:
+      indices.stats:
+        metric: [ translog ]
+  - set: { indices.test.primaries.translog.size_in_bytes: creation_size }
+
+  - do:
+      index:
+        index: test
+        id:    1
+        body:  { "foo": "bar" }
+
+  - do:
+      indices.stats:
+        metric: [ translog ]
+  - gt: { indices.test.primaries.translog.size_in_bytes: $creation_size }
+  - match: { indices.test.primaries.translog.operations: 1 }
+  - match: { indices.test.primaries.translog.uncommitted_operations: 1 }
+  # call flush twice to sync the global checkpoint after the last operation so that we can have the safe commit
+  - do:
+      indices.flush:
+        index: test
+  - do:
+      indices.flush:
+        index: test
+  - do:
+      indices.stats:
+        metric: [ translog ]
+  # after flushing we have one empty translog file while an empty index before flushing has two empty translog files.
+  - lt: { indices.test.primaries.translog.size_in_bytes: $creation_size }
+  - match: { indices.test.primaries.translog.operations: 0 }
+  - lt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size }
+  - match: { indices.test.primaries.translog.uncommitted_operations: 0 }
+
 ---
 "Translog last modified age stats":
   - skip:
@@ -81,11 +128,20 @@ setup:
   - gte: { indices.test.primaries.translog.earliest_last_modified_age: 0 }
 
 ---
-"Translog stats on closed indices":
+"Translog stats on closed indices without soft-deletes":
   - skip:
       version: " - 7.2.99"
       reason:  "closed indices have translog stats starting version 7.3.0"
 
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            soft_deletes.enabled: false
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
   - do:
       index:
         index: test
@@ -123,3 +179,40 @@ setup:
         forbid_closed_indices: false
   - match: { indices.test.primaries.translog.operations: 3 }
   - match: { indices.test.primaries.translog.uncommitted_operations: 0 }
+
+---
+"Translog stats on closed indices with soft-deletes":
+  - skip:
+      version: " - 7.3.99"
+      reason:  "start ignoring translog retention policy with soft-deletes enabled in 7.4"
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            soft_deletes.enabled: true
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+  - do:
+      index:
+        index: test
+        id:    1
+        body:  { "foo": "bar" }
+  - do:
+      indices.stats:
+        metric: [ translog ]
+  - match: { indices.test.primaries.translog.operations: 1 }
+  - match: { indices.test.primaries.translog.uncommitted_operations: 1 }
+  - do:
+      indices.close:
+        index: test
+        wait_for_active_shards: 1
+  - is_true: acknowledged
+  - do:
+      indices.stats:
+        metric: [ translog ]
+        expand_wildcards: all
+        forbid_closed_indices: false
+  - match: { indices.test.primaries.translog.operations: 0 }
+  - match: { indices.test.primaries.translog.uncommitted_operations: 0 }

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -195,24 +195,6 @@ public final class IndexSettings {
             new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES),
             Property.Dynamic, Property.IndexScope);
 
-    /**
-     * Controls how long translog files that are no longer needed for persistence reasons
-     * will be kept around before being deleted. A longer retention policy is useful to increase
-     * the chance of ops based recoveries.
-     **/
-    public static final Setting<TimeValue> INDEX_TRANSLOG_RETENTION_AGE_SETTING =
-        Setting.timeSetting("index.translog.retention.age", TimeValue.timeValueHours(12), TimeValue.timeValueMillis(-1),
-            Property.Dynamic, Property.IndexScope);
-
-    /**
-     * Controls how many translog files that are no longer needed for persistence reasons
-     * will be kept around before being deleted. Keeping more files is useful to increase
-     * the chance of ops based recoveries.
-     **/
-    public static final Setting<ByteSizeValue> INDEX_TRANSLOG_RETENTION_SIZE_SETTING =
-        Setting.byteSizeSetting("index.translog.retention.size", new ByteSizeValue(512, ByteSizeUnit.MB), Property.Dynamic,
-            Property.IndexScope);
-
     /**
      * The maximum size of a translog generation. This is independent of the maximum size of
      * translog operations that have not been flushed.
@@ -258,6 +240,27 @@ public final class IndexSettings {
         Setting.longSetting("index.soft_deletes.retention.operations", 0, 0,
             Property.IndexScope, Property.Dynamic);
 
+    /**
+     * Controls how long translog files that are no longer needed for persistence reasons
+     * will be kept around before being deleted. Keeping more files is useful to increase
+     * the chance of ops based recoveries for indices with soft-deletes disabled.
+     * This setting will be ignored if soft-deletes is enabled.
+     **/
+    public static final Setting<TimeValue> INDEX_TRANSLOG_RETENTION_AGE_SETTING =
+        Setting.timeSetting("index.translog.retention.age",
+            settings -> INDEX_SOFT_DELETES_SETTING.get(settings) ? TimeValue.MINUS_ONE : TimeValue.timeValueHours(12), TimeValue.MINUS_ONE,
+            Property.Dynamic, Property.IndexScope);
+
+    /**
+     * Controls how many translog files that are no longer needed for persistence reasons
+     * will be kept around before being deleted. Keeping more files is useful to increase
+     * the chance of ops based recoveries for indices with soft-deletes disabled.
+     * This setting will be ignored if soft-deletes is enabled.
+     **/
+    public static final Setting<ByteSizeValue> INDEX_TRANSLOG_RETENTION_SIZE_SETTING =
+        Setting.byteSizeSetting("index.translog.retention.size", settings -> INDEX_SOFT_DELETES_SETTING.get(settings) ? "-1" : "512MB",
+            Property.Dynamic, Property.IndexScope);
+
     /**
      * Controls the maximum length of time since a retention lease is created or renewed before it is considered expired.
      */
@@ -466,8 +469,6 @@ public final class IndexSettings {
         syncInterval = INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.get(settings);
         refreshInterval = scopedSettings.get(INDEX_REFRESH_INTERVAL_SETTING);
         flushThresholdSize = scopedSettings.get(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING);
-        translogRetentionAge = scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING);
-        translogRetentionSize = scopedSettings.get(INDEX_TRANSLOG_RETENTION_SIZE_SETTING);
         generationThresholdSize = scopedSettings.get(INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING);
         mergeSchedulerConfig = new MergeSchedulerConfig(this);
         gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis();
@@ -493,6 +494,8 @@ public final class IndexSettings {
         this.indexSortConfig = new IndexSortConfig(this);
         searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER);
         defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE);
+        setTranslogRetentionAge(scopedSettings.get(INDEX_TRANSLOG_RETENTION_AGE_SETTING));
+        setTranslogRetentionSize(scopedSettings.get(INDEX_TRANSLOG_RETENTION_SIZE_SETTING));
 
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio);
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING,
@@ -553,11 +556,21 @@ public final class IndexSettings {
     }
 
     private void setTranslogRetentionSize(ByteSizeValue byteSizeValue) {
-        this.translogRetentionSize = byteSizeValue;
+        if (softDeleteEnabled && byteSizeValue.getBytes() >= 0) {
+            // ignore the translog retention settings if soft-deletes enabled
+            this.translogRetentionSize = new ByteSizeValue(-1);
+        } else {
+            this.translogRetentionSize = byteSizeValue;
+        }
     }
 
     private void setTranslogRetentionAge(TimeValue age) {
-        this.translogRetentionAge = age;
+        if (softDeleteEnabled && age.millis() >= 0) {
+            // ignore the translog retention settings if soft-deletes enabled
+            this.translogRetentionAge = TimeValue.MINUS_ONE;
+        } else {
+            this.translogRetentionAge = age;
+        }
     }
 
     private void setGenerationThresholdSize(final ByteSizeValue generationThresholdSize) {
@@ -734,13 +747,19 @@ public final class IndexSettings {
     /**
      * Returns the transaction log retention size which controls how much of the translog is kept around to allow for ops based recoveries
      */
-    public ByteSizeValue getTranslogRetentionSize() { return translogRetentionSize; }
+    public ByteSizeValue getTranslogRetentionSize() {
+        assert softDeleteEnabled == false || translogRetentionSize.getBytes() == -1L : translogRetentionSize;
+        return translogRetentionSize;
+    }
 
     /**
      * Returns the transaction log retention age which controls the maximum age (time from creation) that translog files will be kept
      * around
      */
-    public TimeValue getTranslogRetentionAge() { return translogRetentionAge; }
+    public TimeValue getTranslogRetentionAge() {
+        assert softDeleteEnabled == false || translogRetentionAge.millis() == -1L : translogRetentionSize;
+        return translogRetentionAge;
+    }
 
     /**
      * Returns the generation threshold size. As sequence numbers can cause multiple generations to

server/src/main/java/org/elasticsearch/index/translog/TruncateTranslogAction.java

@@ -177,11 +177,19 @@ public class TruncateTranslogAction {
             final TranslogConfig translogConfig = new TranslogConfig(shardPath.getShardId(), translogPath,
                 indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
             long primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardPath.getShardId().id());
-            final TranslogDeletionPolicy translogDeletionPolicy =
+            // We open translog to check for corruption, do not clean anything.
-                new TranslogDeletionPolicy(indexSettings.getTranslogRetentionSize().getBytes(),
+            final TranslogDeletionPolicy retainAllTranslogPolicy = new TranslogDeletionPolicy(Long.MAX_VALUE, Long.MAX_VALUE) {
-                    indexSettings.getTranslogRetentionAge().getMillis());
+                @Override
+                long minTranslogGenRequired(List<TranslogReader> readers, TranslogWriter writer) {
+                    long minGen = writer.generation;
+                    for (TranslogReader reader : readers) {
+                        minGen = Math.min(reader.generation, minGen);
+                    }
+                    return minGen;
+                }
+            };
             try (Translog translog = new Translog(translogConfig, translogUUID,
-                translogDeletionPolicy, () -> translogGlobalCheckpoint, () -> primaryTerm, seqNo -> {});
+                retainAllTranslogPolicy, () -> translogGlobalCheckpoint, () -> primaryTerm, seqNo -> {});
                  Translog.Snapshot snapshot = translog.newSnapshot()) {
                 //noinspection StatementWithEmptyBody we are just checking that we can iterate through the whole snapshot
                 while (snapshot.next() != null) {

server/src/test/java/org/elasticsearch/index/IndexServiceTests.java

@@ -396,15 +396,20 @@ public class IndexServiceTests extends ESSingleNodeTestCase {
         final Path translogPath = translog.getConfig().getTranslogPath();
         final String translogUuid = translog.getTranslogUUID();
 
+        int translogOps = 0;
         final int numDocs = scaledRandomIntBetween(10, 100);
         for (int i = 0; i < numDocs; i++) {
             client().prepareIndex().setIndex(indexName).setId(String.valueOf(i)).setSource("{\"foo\": \"bar\"}", XContentType.JSON).get();
+            translogOps++;
             if (randomBoolean()) {
                 client().admin().indices().prepareFlush(indexName).get();
+                if (indexService.getIndexSettings().isSoftDeleteEnabled()) {
+                    translogOps = 0;
+                }
             }
         }
-        assertThat(translog.totalOperations(), equalTo(numDocs));
+        assertThat(translog.totalOperations(), equalTo(translogOps));
-        assertThat(translog.stats().estimatedNumberOfOperations(), equalTo(numDocs));
+        assertThat(translog.stats().estimatedNumberOfOperations(), equalTo(translogOps));
         assertAcked(client().admin().indices().prepareClose("test").setWaitForActiveShards(ActiveShardCount.DEFAULT));
 
         indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(indexService.index());

server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java

@@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.translog.Translog;
@@ -577,4 +578,64 @@ public class IndexSettingsTests extends ESTestCase {
             assertFalse(IndexSettings.INDEX_SOFT_DELETES_SETTING.get(settings));
         }
     }
+
+    public void testIgnoreTranslogRetentionSettingsIfSoftDeletesEnabled() {
+        Settings.Builder settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT));
+        if (randomBoolean()) {
+            settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue());
+        }
+        if (randomBoolean()) {
+            settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 1024) + "b");
+        }
+        IndexMetaData metaData = newIndexMeta("index", settings.build());
+        IndexSettings indexSettings = new IndexSettings(metaData, Settings.EMPTY);
+        assertThat(indexSettings.getTranslogRetentionAge().millis(), equalTo(-1L));
+        assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L));
+
+        Settings.Builder newSettings = Settings.builder().put(settings.build());
+        if (randomBoolean()) {
+            newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue());
+        }
+        if (randomBoolean()) {
+            newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 1024) + "b");
+        }
+        indexSettings.updateIndexMetaData(newIndexMeta("index", newSettings.build()));
+        assertThat(indexSettings.getTranslogRetentionAge().millis(), equalTo(-1L));
+        assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L));
+    }
+
+    public void testUpdateTranslogRetentionSettingsWithSoftDeletesDisabled() {
+        Settings.Builder settings = Settings.builder()
+            .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
+
+        TimeValue ageSetting = TimeValue.timeValueHours(12);
+        if (randomBoolean()) {
+            ageSetting = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomIntBetween(0, 10000));
+            settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), ageSetting);
+        }
+        ByteSizeValue sizeSetting = new ByteSizeValue(512, ByteSizeUnit.MB);
+        if (randomBoolean()) {
+            sizeSetting = randomBoolean() ? new ByteSizeValue(-1) : new ByteSizeValue(randomIntBetween(0, 1024));
+            settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), sizeSetting);
+        }
+        IndexMetaData metaData = newIndexMeta("index", settings.build());
+        IndexSettings indexSettings = new IndexSettings(metaData, Settings.EMPTY);
+        assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting));
+        assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting));
+
+        Settings.Builder newSettings = Settings.builder().put(settings.build());
+        if (randomBoolean()) {
+            ageSetting = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomIntBetween(0, 10000));
+            newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), ageSetting);
+        }
+        if (randomBoolean()) {
+            sizeSetting = randomBoolean() ? new ByteSizeValue(-1) : new ByteSizeValue(randomIntBetween(0, 1024));
+            newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), sizeSetting);
+        }
+        indexSettings.updateIndexMetaData(newIndexMeta("index", newSettings.build()));
+        assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting));
+        assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting));
+    }
 }

server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java

@@ -169,13 +169,14 @@ public class NoOpEngineTests extends EngineTestCase {
         tracker.updateFromMaster(1L, Collections.singleton(allocationId.getId()), table);
         tracker.activatePrimaryMode(SequenceNumbers.NO_OPS_PERFORMED);
 
+        boolean softDeleteEnabled = engine.config().getIndexSettings().isSoftDeleteEnabled();
         final int numDocs = scaledRandomIntBetween(10, 3000);
         for (int i = 0; i < numDocs; i++) {
             engine.index(indexForDoc(createParsedDoc(Integer.toString(i), null)));
+            tracker.updateLocalCheckpoint(allocationId.getId(), i);
             if (rarely()) {
                 engine.flush();
             }
-            tracker.updateLocalCheckpoint(allocationId.getId(), i);
         }
         engine.flush(true, true);
 
@@ -195,7 +196,7 @@ public class NoOpEngineTests extends EngineTestCase {
         }
 
         assertThat(Translog.readMinTranslogGeneration(translogPath, translogUuid), equalTo(minFileGeneration));
-        assertThat(noOpEngine.getTranslogStats().estimatedNumberOfOperations(), equalTo(numDocs));
+        assertThat(noOpEngine.getTranslogStats().estimatedNumberOfOperations(), equalTo(softDeleteEnabled ? 0 : numDocs));
         assertThat(noOpEngine.getTranslogStats().getUncommittedOperations(), equalTo(0));
 
         noOpEngine.trimUnreferencedTranslogFiles();

server/src/test/java/org/elasticsearch/index/engine/ReadOnlyEngineTests.java

@@ -250,7 +250,7 @@ public class ReadOnlyEngineTests extends EngineTestCase {
         try (Store store = createStore()) {
             final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
             EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
-
+            final boolean softDeletesEnabled = config.getIndexSettings().isSoftDeleteEnabled();
             final int numDocs = frequently() ? scaledRandomIntBetween(10, 200) : 0;
             int uncommittedDocs = 0;
 
@@ -259,16 +259,17 @@ public class ReadOnlyEngineTests extends EngineTestCase {
                     ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null);
                     engine.index(new Engine.Index(newUid(doc), doc, i, primaryTerm.get(), 1, null, Engine.Operation.Origin.REPLICA,
                         System.nanoTime(), -1, false, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
+                    globalCheckpoint.set(i);
                     if (rarely()) {
                         engine.flush();
                         uncommittedDocs = 0;
                     } else {
                         uncommittedDocs += 1;
                     }
-                    globalCheckpoint.set(i);
                 }
 
-                assertThat(engine.getTranslogStats().estimatedNumberOfOperations(), equalTo(numDocs));
+                assertThat(engine.getTranslogStats().estimatedNumberOfOperations(),
+                    equalTo(softDeletesEnabled ? uncommittedDocs : numDocs));
                 assertThat(engine.getTranslogStats().getUncommittedOperations(), equalTo(uncommittedDocs));
                 assertThat(engine.getTranslogStats().getTranslogSizeInBytes(), greaterThan(0L));
                 assertThat(engine.getTranslogStats().getUncommittedSizeInBytes(), greaterThan(0L));
@@ -278,7 +279,7 @@ public class ReadOnlyEngineTests extends EngineTestCase {
             }
 
             try (ReadOnlyEngine readOnlyEngine = new ReadOnlyEngine(config, null, null, true, Function.identity())) {
-                assertThat(readOnlyEngine.getTranslogStats().estimatedNumberOfOperations(), equalTo(numDocs));
+                assertThat(readOnlyEngine.getTranslogStats().estimatedNumberOfOperations(), equalTo(softDeletesEnabled ? 0 : numDocs));
                 assertThat(readOnlyEngine.getTranslogStats().getUncommittedOperations(), equalTo(0));
                 assertThat(readOnlyEngine.getTranslogStats().getTranslogSizeInBytes(), greaterThan(0L));
                 assertThat(readOnlyEngine.getTranslogStats().getUncommittedSizeInBytes(), greaterThan(0L));

server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java

@@ -467,7 +467,12 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase
             shards.startReplicas(nReplica);
             for (IndexShard shard : shards) {
                 try (Translog.Snapshot snapshot = getTranslog(shard).newSnapshot()) {
-                    assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps));
+                    // we flush at the end of peer recovery
+                    if (shard.routingEntry().primary() || shard.indexSettings().isSoftDeleteEnabled() == false) {
+                        assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps));
+                    } else {
+                        assertThat(snapshot.totalOperations(), equalTo(0));
+                    }
                 }
                 try (Translog.Snapshot snapshot = shard.getHistoryOperations("test", 0)) {
                     assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps));
@@ -476,11 +481,16 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase
             // the failure replicated directly from the replication channel.
             indexResp = shards.index(new IndexRequest(index.getName(), "type", "any").source("{}", XContentType.JSON));
             assertThat(indexResp.getFailure().getCause(), equalTo(indexException));
-            expectedTranslogOps.add(new Translog.NoOp(1, primaryTerm, indexException.toString()));
+            Translog.NoOp noop2 = new Translog.NoOp(1, primaryTerm, indexException.toString());
+            expectedTranslogOps.add(noop2);
 
             for (IndexShard shard : shards) {
                 try (Translog.Snapshot snapshot = getTranslog(shard).newSnapshot()) {
-                    assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps));
+                    if (shard.routingEntry().primary() || shard.indexSettings().isSoftDeleteEnabled() == false) {
+                        assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps));
+                    } else {
+                        assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(Collections.singletonList(noop2)));
+                    }
                 }
                 try (Translog.Snapshot snapshot = shard.getHistoryOperations("test", 0)) {
                     assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps));

server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java

@@ -2136,11 +2136,13 @@ public class IndexShardTests extends IndexShardTestCase {
 
     /* This test just verifies that we fill up local checkpoint up to max seen seqID on primary recovery */
     public void testRecoverFromStoreWithNoOps() throws IOException {
-        final IndexShard shard = newStartedShard(true);
+        final Settings settings = Settings.builder()
+            .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).build();
+        final IndexShard shard = newStartedShard(true, settings);
         indexDoc(shard, "_doc", "0");
         indexDoc(shard, "_doc", "1");
         // start a replica shard and index the second doc
-        final IndexShard otherShard = newStartedShard(false);
+        final IndexShard otherShard = newStartedShard(false, settings);
         updateMappings(otherShard, shard.indexSettings().getIndexMetaData());
         SourceToParse sourceToParse = new SourceToParse(shard.shardId().getIndexName(), "_doc", "1",
             new BytesArray("{}"), XContentType.JSON);
@@ -2179,7 +2181,7 @@ public class IndexShardTests extends IndexShardTestCase {
             newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null));
             assertTrue(newShard.recoverFromStore());
             try (Translog.Snapshot snapshot = getTranslog(newShard).newSnapshot()) {
-                assertThat(snapshot.totalOperations(), equalTo(2));
+                assertThat(snapshot.totalOperations(), equalTo(newShard.indexSettings.isSoftDeleteEnabled() ? 0 : 2));
             }
         }
         closeShards(newShard, shard);
@@ -3801,7 +3803,13 @@ public class IndexShardTests extends IndexShardTestCase {
         engineResetLatch.await();
         assertThat(getShardDocUIDs(shard), equalTo(docBelowGlobalCheckpoint));
         assertThat(shard.seqNoStats().getMaxSeqNo(), equalTo(globalCheckpoint));
-        assertThat(shard.translogStats().estimatedNumberOfOperations(), equalTo(translogStats.estimatedNumberOfOperations()));
+        if (shard.indexSettings.isSoftDeleteEnabled()) {
+            // we might have trimmed some operations if the translog retention policy is ignored (when soft-deletes enabled).
+            assertThat(shard.translogStats().estimatedNumberOfOperations(),
+                lessThanOrEqualTo(translogStats.estimatedNumberOfOperations()));
+        } else {
+            assertThat(shard.translogStats().estimatedNumberOfOperations(), equalTo(translogStats.estimatedNumberOfOperations()));
+        }
         assertThat(shard.getMaxSeqNoOfUpdatesOrDeletes(), equalTo(maxSeqNoBeforeRollback));
         done.set(true);
         thread.join();

server/src/test/java/org/elasticsearch/indices/recovery/RecoveryTests.java

@@ -82,7 +82,7 @@ public class RecoveryTests extends ESIndexLevelReplicationTestCase {
             shards.startAll();
             final IndexShard replica = shards.getReplicas().get(0);
             boolean softDeletesEnabled = replica.indexSettings().isSoftDeleteEnabled();
-            assertThat(getTranslog(replica).totalOperations(), equalTo(softDeletesEnabled ? moreDocs : docs + moreDocs));
+            assertThat(getTranslog(replica).totalOperations(), equalTo(softDeletesEnabled ? 0 : docs + moreDocs));
             shards.assertAllEqual(docs + moreDocs);
         }
     }
@@ -298,7 +298,7 @@ public class RecoveryTests extends ESIndexLevelReplicationTestCase {
             // file based recovery should be made
             assertThat(newReplica.recoveryState().getIndex().fileDetails(), not(empty()));
             boolean softDeletesEnabled = replica.indexSettings().isSoftDeleteEnabled();
-            assertThat(getTranslog(newReplica).totalOperations(), equalTo(softDeletesEnabled ? nonFlushedDocs : numDocs));
+            assertThat(getTranslog(newReplica).totalOperations(), equalTo(softDeletesEnabled ? 0 : numDocs));
 
             // history uuid was restored
             assertThat(newReplica.getHistoryUUID(), equalTo(historyUUID));
@@ -385,7 +385,12 @@ public class RecoveryTests extends ESIndexLevelReplicationTestCase {
             shards.recoverReplica(newReplica);
 
             try (Translog.Snapshot snapshot = getTranslog(newReplica).newSnapshot()) {
-                assertThat("Sequence based recovery should keep existing translog", snapshot, SnapshotMatchers.size(initDocs + moreDocs));
+                if (newReplica.indexSettings().isSoftDeleteEnabled()) {
+                    assertThat(snapshot.totalOperations(), equalTo(0));
+                } else {
+                    assertThat("Sequence based recovery should keep existing translog",
+                        snapshot, SnapshotMatchers.size(initDocs + moreDocs));
+                }
             }
             assertThat(newReplica.recoveryState().getTranslog().recoveredOperations(), equalTo(uncommittedDocs + moreDocs));
             assertThat(newReplica.recoveryState().getIndex().fileDetails(), empty());

server/src/test/java/org/elasticsearch/indices/state/OpenCloseIndexIT.java

@@ -36,6 +36,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -352,11 +353,13 @@ public class OpenCloseIndexIT extends ESIntegTestCase {
         }
     }
 
-    public void testTranslogStats()  {
+    public void testTranslogStats() throws Exception {
         final String indexName = "test";
         createIndex(indexName, Settings.builder()
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
             .build());
+        boolean softDeletesEnabled = IndexSettings.INDEX_SOFT_DELETES_SETTING.get(
+            client().admin().indices().prepareGetSettings(indexName).get().getIndexToSettings().get(indexName));
 
         final int nbDocs = randomIntBetween(0, 50);
         int uncommittedOps = 0;
@@ -372,17 +375,23 @@ public class OpenCloseIndexIT extends ESIntegTestCase {
             }
         }
 
-        IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().setTranslog(true).get();
+        final int uncommittedTranslogOps = uncommittedOps;
-        assertThat(stats.getIndex(indexName), notNullValue());
+        assertBusy(() -> {
-        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(nbDocs));
+            IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().setTranslog(true).get();
-        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(uncommittedOps));
+            assertThat(stats.getIndex(indexName), notNullValue());
+            assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(
+                softDeletesEnabled ? uncommittedTranslogOps : nbDocs));
+            assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(uncommittedTranslogOps));
+        });
 
         assertAcked(client().admin().indices().prepareClose("test").setWaitForActiveShards(ActiveShardCount.ONE));
 
         IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN_CLOSED;
-        stats = client().admin().indices().prepareStats(indexName).setIndicesOptions(indicesOptions).clear().setTranslog(true).get();
+        IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName)
+            .setIndicesOptions(indicesOptions).clear().setTranslog(true).get();
         assertThat(stats.getIndex(indexName), notNullValue());
-        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(nbDocs));
+        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(),
+            equalTo(softDeletesEnabled ? 0 : nbDocs));
         assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(0));
     }
 }

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

@@ -49,7 +49,8 @@ public class DeprecationChecks {
             IndexDeprecationChecks::oldIndicesCheck,
             IndexDeprecationChecks::tooManyFieldsCheck,
             IndexDeprecationChecks::chainedMultiFieldsCheck,
-            IndexDeprecationChecks::deprecatedDateTimeFormat
+            IndexDeprecationChecks::deprecatedDateTimeFormat,
+            IndexDeprecationChecks::translogRetentionSettingCheck
         ));
 
     static List<BiFunction<DatafeedConfig, NamedXContentRegistry, DeprecationIssue>> ML_SETTINGS_CHECKS =

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java

@@ -231,4 +231,19 @@ public class IndexDeprecationChecks {
 
         return fields;
     }
+
+    static DeprecationIssue translogRetentionSettingCheck(IndexMetaData indexMetaData) {
+        final boolean softDeletesEnabled = IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexMetaData.getSettings());
+        if (softDeletesEnabled) {
+            if (IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.exists(indexMetaData.getSettings())
+                || IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.exists(indexMetaData.getSettings())) {
+                return new DeprecationIssue(DeprecationIssue.Level.WARNING,
+                    "translog retention settings are ignored",
+                    "https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules-translog.html",
+                    "translog retention settings [index.translog.retention.size] and [index.translog.retention.age] are ignored " +
+                        "because translog is no longer used in peer recoveries with soft-deletes enabled (default in 7.0 or later)");
+            }
+        }
+        return null;
+    }
 }

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java

@@ -14,6 +14,7 @@ import org.elasticsearch.common.joda.JodaDeprecationPatterns;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
@@ -27,6 +28,8 @@ import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.deprecation.DeprecationChecks.INDEX_SETTINGS_CHECKS;
 import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.collection.IsIterableContainingInOrder.contains;
 
 public class IndexDeprecationChecksTests extends ESTestCase {
     public void testOldIndicesCheck() {
@@ -382,4 +385,31 @@ public class IndexDeprecationChecksTests extends ESTestCase {
         }
         mappingBuilder.endObject();
     }
+
+    public void testTranslogRetentionSettings() {
+        Settings.Builder settings = settings(Version.CURRENT);
+        settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue());
+        settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 1024) + "b");
+        IndexMetaData indexMetaData = IndexMetaData.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build();
+        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetaData));
+        assertThat(issues, contains(
+            new DeprecationIssue(DeprecationIssue.Level.WARNING,
+                "translog retention settings are ignored",
+                "https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules-translog.html",
+                "translog retention settings [index.translog.retention.size] and [index.translog.retention.age] are ignored " +
+                    "because translog is no longer used in peer recoveries with soft-deletes enabled (default in 7.0 or later)")
+        ));
+    }
+
+    public void testDefaultTranslogRetentionSettings() {
+        Settings.Builder settings = settings(Version.CURRENT);
+        if (randomBoolean()) {
+            settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue());
+            settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 1024) + "b");
+            settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false);
+        }
+        IndexMetaData indexMetaData = IndexMetaData.builder("test").settings(settings).numberOfShards(1).numberOfReplicas(0).build();
+        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetaData));
+        assertThat(issues, empty());
+    }
 }

x-pack/plugin/frozen-indices/src/test/java/org/elasticsearch/index/engine/FrozenIndexTests.java

@@ -411,7 +411,7 @@ public class FrozenIndexTests extends ESSingleNodeTestCase {
 
     public void testTranslogStats() throws Exception {
         final String indexName = "test";
-        createIndex(indexName, Settings.builder()
+        IndexService indexService = createIndex(indexName, Settings.builder()
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
             .build());
 
@@ -420,7 +420,6 @@ public class FrozenIndexTests extends ESSingleNodeTestCase {
         for (long i = 0; i < nbDocs; i++) {
             final IndexResponse indexResponse = client().prepareIndex(indexName, "_doc", Long.toString(i)).setSource("field", i).get();
             assertThat(indexResponse.status(), is(RestStatus.CREATED));
-
             if (rarely()) {
                 client().admin().indices().prepareFlush(indexName).get();
                 uncommittedOps = 0;
@@ -431,7 +430,8 @@ public class FrozenIndexTests extends ESSingleNodeTestCase {
 
         IndicesStatsResponse stats = client().admin().indices().prepareStats(indexName).clear().setTranslog(true).get();
         assertThat(stats.getIndex(indexName), notNullValue());
-        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(nbDocs));
+        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(
+            indexService.getIndexSettings().isSoftDeleteEnabled() ? uncommittedOps : nbDocs));
         assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(uncommittedOps));
 
         assertAcked(new XPackClient(client()).freeze(new FreezeRequest(indexName)));
@@ -440,7 +440,8 @@ public class FrozenIndexTests extends ESSingleNodeTestCase {
         IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN_CLOSED;
         stats = client().admin().indices().prepareStats(indexName).setIndicesOptions(indicesOptions).clear().setTranslog(true).get();
         assertThat(stats.getIndex(indexName), notNullValue());
-        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(nbDocs));
+        assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(),
+            equalTo(indexService.getIndexSettings().isSoftDeleteEnabled() ? 0 : nbDocs));
         assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(0));
     }
 }

x-pack/plugin/src/test/resources/rest-api-spec/test/indices.freeze/20_stats.yml

@@ -10,8 +10,8 @@ setup:
 ---
 "Translog stats on frozen indices":
   - skip:
-      version: " - 7.2.99"
+      version: " - 7.3.99"
-      reason:  "frozen indices have translog stats starting version 7.3.0"
+      reason:  "start ignoring translog retention policy with soft-deletes enabled in 7.4"
 
   - do:
       index:
@@ -47,7 +47,7 @@ setup:
   - do:
       indices.stats:
         metric: [ translog ]
-  - match: { indices.test.primaries.translog.operations: 3 }
+  - match: { indices.test.primaries.translog.operations: 0 }
   - match: { indices.test.primaries.translog.uncommitted_operations: 0 }
 
   # unfreeze index
@@ -60,5 +60,5 @@ setup:
   - do:
       indices.stats:
         metric: [ translog ]
-  - match: { indices.test.primaries.translog.operations: 3 }
+  - match: { indices.test.primaries.translog.operations: 0 }
   - match: { indices.test.primaries.translog.uncommitted_operations: 0 }