diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java index 55c6ff351da..13e1e4a3fea 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java @@ -789,44 +789,6 @@ public class UpdateSettingsIT extends OpenSearchIntegTestCase { assertThat(newSettingsVersion, equalTo(1 + settingsVersion)); } - public void testTranslogPruningSetting() { - createIndex("test"); - ensureGreen("test"); - final String REPLICATION_TRANSLOG_SETTING = "index.plugins.replication.translog.retention_lease.pruning.enabled"; - final long settingsVersion = client().admin() - .cluster() - .prepareState() - .get() - .getState() - .metadata() - .index("test") - .getSettingsVersion(); - GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings("test").get(); - boolean shouldPruneTranslogByRetentionLease = Boolean.parseBoolean( - settingsResponse.getSetting("test", REPLICATION_TRANSLOG_SETTING) - ); - - assertFalse(shouldPruneTranslogByRetentionLease); - assertAcked( - client().admin() - .indices() - .prepareUpdateSettings("test") - .setSettings(Settings.builder().put("index.plugins.replication.translog.retention_lease.pruning.enabled", true)) - ); - final long newSettingsVersion = client().admin() - .cluster() - .prepareState() - .get() - .getState() - .metadata() - .index("test") - .getSettingsVersion(); - assertThat(newSettingsVersion, equalTo(1 + settingsVersion)); - settingsResponse = client().admin().indices().prepareGetSettings("test").get(); - shouldPruneTranslogByRetentionLease = Boolean.parseBoolean(settingsResponse.getSetting("test", REPLICATION_TRANSLOG_SETTING)); - assertTrue(shouldPruneTranslogByRetentionLease); - } - /* * Test that we are able to set the setting index.number_of_replicas to the default. */ diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index fce41b8109b..a923e6d0b9d 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1464,13 +1464,6 @@ public class MetadataCreateIndexService { + "and [index.translog.retention.size] are deprecated and effectively ignored. " + "They will be removed in a future version." ); - } else if (IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.exists(indexSettings)) { - DEPRECATION_LOGGER.deprecate( - "translog_pruning_retention_lease", - "[index.plugins.replication.translog.retention_lease.pruning.enabled] " - + "setting was deprecated in OpenSearch and will be removed in a future release! " - + "See the breaking changes documentation for the next major version." - ); } } } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index f42b7f8c1ad..6ea8dccab9c 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -158,7 +158,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings { EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING, EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING, IndexSettings.INDEX_FLUSH_AFTER_MERGE_THRESHOLD_SIZE_SETTING, - IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING, IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING, IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING, @@ -244,6 +243,9 @@ public final class IndexScopedSettings extends AbstractScopedSettings { case "index.shrink.source.name": case IndexMetadata.INDEX_RESIZE_SOURCE_UUID_KEY: case IndexMetadata.INDEX_RESIZE_SOURCE_NAME_KEY: + // we keep this setting for BWC to support indexes created in 1.1.0 + // this can be removed in OpenSearch 3.0 + case "index.plugins.replication.translog.retention_lease.pruning.enabled": return true; default: return IndexMetadata.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getRawKey().match(key); diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 7466343e170..6006002b38f 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -362,19 +362,6 @@ public final class IndexSettings { Property.Final ); - /** - * Specifies if the index translog should prune based on retention leases. - * @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release - */ - @Deprecated - public static final Setting INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING = Setting.boolSetting( - "index.plugins.replication.translog.retention_lease.pruning.enabled", - false, - Property.IndexScope, - Property.Dynamic, - Property.Deprecated - ); - /** * Controls how many soft-deleted documents will be kept around before being merged away. Keeping more deleted * documents increases the chance of operation-based recoveries and allows querying a longer history of documents. @@ -408,12 +395,10 @@ public final class IndexSettings { * the chance of ops based recoveries for indices with soft-deletes disabled. * This setting will be ignored if soft-deletes is used in peer recoveries (default in 7.4). **/ - @Deprecated - private static final ByteSizeValue DEFAULT_TRANSLOG_RETENTION_SIZE = new ByteSizeValue(512, ByteSizeUnit.MB); public static final Setting INDEX_TRANSLOG_RETENTION_SIZE_SETTING = Setting.byteSizeSetting( "index.translog.retention.size", - settings -> DEFAULT_TRANSLOG_RETENTION_SIZE.getStringRep(), + settings -> shouldDisableTranslogRetention(settings) ? "-1" : "512MB", Property.Dynamic, Property.IndexScope ); @@ -546,8 +531,6 @@ public final class IndexSettings { private final IndexScopedSettings scopedSettings; private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis(); private final boolean softDeleteEnabled; - @Deprecated - private volatile boolean translogPruningByRetentionLease; private volatile long softDeleteRetentionOperations; private volatile long retentionLeaseMillis; @@ -684,10 +667,6 @@ public final class IndexSettings { mergeSchedulerConfig = new MergeSchedulerConfig(this); gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis(); softDeleteEnabled = version.onOrAfter(LegacyESVersion.V_6_5_0) && scopedSettings.get(INDEX_SOFT_DELETES_SETTING); - // @todo move to CCR plugin - this.translogPruningByRetentionLease = version.onOrAfter(Version.V_1_1_0) - && softDeleteEnabled - && scopedSettings.get(INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING); softDeleteRetentionOperations = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING); retentionLeaseMillis = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING).millis(); warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING); @@ -771,10 +750,6 @@ public final class IndexSettings { scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING, this::setGenerationThresholdSize); scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_RETENTION_AGE_SETTING, this::setTranslogRetentionAge); scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_RETENTION_SIZE_SETTING, this::setTranslogRetentionSize); - scopedSettings.addSettingsUpdateConsumer( - INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING, - this::setTranslogPruningByRetentionLease - ); scopedSettings.addSettingsUpdateConsumer(INDEX_REFRESH_INTERVAL_SETTING, this::setRefreshInterval); scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners); scopedSettings.addSettingsUpdateConsumer(MAX_ANALYZED_OFFSET_SETTING, this::setHighlightMaxAnalyzedOffset); @@ -807,20 +782,8 @@ public final class IndexSettings { this.flushAfterMergeThresholdSize = byteSizeValue; } - /** - * Enable or disable translog pruning by retention lease requirement - * - * @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release - */ - @Deprecated - private void setTranslogPruningByRetentionLease(boolean enabled) { - this.translogPruningByRetentionLease = INDEX_SOFT_DELETES_SETTING.get(settings) && enabled; - } - private void setTranslogRetentionSize(ByteSizeValue byteSizeValue) { - if (shouldDisableTranslogRetention(settings) - && shouldPruneTranslogByRetentionLease(settings) == false - && byteSizeValue.getBytes() >= 0) { + if (shouldDisableTranslogRetention(settings) && byteSizeValue.getBytes() >= 0) { // ignore the translog retention settings if soft-deletes enabled this.translogRetentionSize = new ByteSizeValue(-1); } else { @@ -1033,11 +996,7 @@ public final class IndexSettings { * Returns the transaction log retention size which controls how much of the translog is kept around to allow for ops based recoveries */ public ByteSizeValue getTranslogRetentionSize() { - if (shouldDisableTranslogRetention(settings) && shouldPruneTranslogByRetentionLease(settings) == false) { - return new ByteSizeValue(-1); - } else if (shouldPruneTranslogByRetentionLease(settings) && translogRetentionSize.getBytes() == -1) { - return DEFAULT_TRANSLOG_RETENTION_SIZE; - } + assert shouldDisableTranslogRetention(settings) == false || translogRetentionSize.getBytes() == -1L : translogRetentionSize; return translogRetentionSize; } @@ -1303,24 +1262,6 @@ public final class IndexSettings { this.requiredPipeline = requiredPipeline; } - /** - * Returns true if translog ops should be pruned based on retention lease - * @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release - */ - @Deprecated - public boolean shouldPruneTranslogByRetentionLease() { - return translogPruningByRetentionLease; - } - - /** - * Returns true if translog ops should be pruned based on retention lease - * @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release - */ - @Deprecated - public static boolean shouldPruneTranslogByRetentionLease(Settings settings) { - return INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.get(settings); - } - /** * Returns true if soft-delete is enabled. */ diff --git a/server/src/main/java/org/opensearch/index/engine/Engine.java b/server/src/main/java/org/opensearch/index/engine/Engine.java index ce1614832be..d4415e91e2f 100644 --- a/server/src/main/java/org/opensearch/index/engine/Engine.java +++ b/server/src/main/java/org/opensearch/index/engine/Engine.java @@ -1998,22 +1998,7 @@ public abstract class Engine implements Closeable { } public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) { - onSettingsChanged(translogRetentionAge, translogRetentionSize, softDeletesRetentionOps, false); - } - /** - * callback when index settings change - * - * @deprecated EXPERT: this method is specific to CCR and will be moved to a plugin in the next release - */ - @Deprecated - public void onSettingsChanged( - TimeValue translogRetentionAge, - ByteSizeValue translogRetentionSize, - long softDeletesRetentionOps, - boolean translogPruningByRetentionLease - ) { - // @todo this is overly silent; make this abstract and force derived classes to noop in the next release } /** diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index afb5834244b..ff1e7b3fc84 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -2801,12 +2801,7 @@ public class InternalEngine extends Engine { } @Override - public void onSettingsChanged( - TimeValue translogRetentionAge, - ByteSizeValue translogRetentionSize, - long softDeletesRetentionOps, - boolean translogPruningByRetentionLease - ) { + public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) { mergeScheduler.refreshConfig(); // config().isEnableGcDeletes() or config.getGcDeletesInMillis() may have changed: maybePruneDeletes(); @@ -2819,7 +2814,6 @@ public class InternalEngine extends Engine { final TranslogDeletionPolicy translogDeletionPolicy = translog.getDeletionPolicy(); translogDeletionPolicy.setRetentionAgeInMillis(translogRetentionAge.millis()); translogDeletionPolicy.setRetentionSizeInBytes(translogRetentionSize.getBytes()); - translogDeletionPolicy.shouldPruneTranslogByRetentionLease(translogPruningByRetentionLease); softDeletesPolicy.setRetentionOperations(softDeletesRetentionOps); } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 67852a17237..9217a2b67c1 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -2278,11 +2278,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl final boolean disableTranslogRetention = indexSettings.isSoftDeleteEnabled() && useRetentionLeasesInPeerRecovery; engineOrNull.onSettingsChanged( disableTranslogRetention ? TimeValue.MINUS_ONE : indexSettings.getTranslogRetentionAge(), - disableTranslogRetention && !indexSettings.shouldPruneTranslogByRetentionLease() - ? new ByteSizeValue(-1) - : indexSettings.getTranslogRetentionSize(), - indexSettings.getSoftDeleteRetentionOperations(), - indexSettings.shouldPruneTranslogByRetentionLease() + disableTranslogRetention ? new ByteSizeValue(-1) : indexSettings.getTranslogRetentionSize(), + indexSettings.getSoftDeleteRetentionOperations() ); } } diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java b/server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java index 1478b1ae087..e3b5898c730 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java @@ -35,8 +35,6 @@ package org.opensearch.index.translog; import org.apache.lucene.util.Counter; import org.opensearch.Assertions; import org.opensearch.common.lease.Releasable; -import org.opensearch.index.seqno.RetentionLease; -import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.seqno.SequenceNumbers; import java.io.IOException; @@ -45,16 +43,10 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; public class TranslogDeletionPolicy { private final Map openTranslogRef; - /** - * @deprecated EXPERT: this supplier is specific to CCR and will be moved to a plugin in the next release - */ - @Deprecated - private Supplier retentionLeasesSupplier; public void assertNoOpenTranslogRefs() { if (openTranslogRef.isEmpty() == false) { @@ -94,21 +86,6 @@ public class TranslogDeletionPolicy { } } - /** - * Construct a TranslogDeletionPolicy to include pruning by retention leases - * @deprecated EXPERT: this ctor is specific to CCR and will be moved to a plugin in the next release - */ - @Deprecated - public TranslogDeletionPolicy( - long retentionSizeInBytes, - long retentionAgeInMillis, - int retentionTotalFiles, - Supplier retentionLeasesSupplier - ) { - this(retentionSizeInBytes, retentionAgeInMillis, retentionTotalFiles); - this.retentionLeasesSupplier = retentionLeasesSupplier; - } - public synchronized void setLocalCheckpointOfSafeCommit(long newCheckpoint) { if (newCheckpoint < this.localCheckpointOfSafeCommit) { throw new IllegalArgumentException( @@ -135,15 +112,6 @@ public class TranslogDeletionPolicy { this.retentionTotalFiles = retentionTotalFiles; } - /** - * Should the translog be pruned by the retention lease heuristic - * @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release - */ - @Deprecated - public synchronized void shouldPruneTranslogByRetentionLease(boolean translogPruneByRetentionLease) { - this.shouldPruneTranslogByRetentionLease = translogPruneByRetentionLease; - } - /** * acquires the basis generation for a new snapshot. Any translog generation above, and including, the returned generation * will not be deleted until the returned {@link Releasable} is closed. @@ -201,12 +169,6 @@ public class TranslogDeletionPolicy { long minByLocks = getMinTranslogGenRequiredByLocks(); long minByAge = getMinTranslogGenByAge(readers, writer, retentionAgeInMillis, currentTime()); long minBySize = getMinTranslogGenBySize(readers, writer, retentionSizeInBytes); - long minByRetentionLeasesAndSize = Long.MAX_VALUE; - if (shouldPruneTranslogByRetentionLease) { - // If retention size is specified, size takes precedence. - long minByRetentionLeases = getMinTranslogGenByRetentionLease(readers, writer, retentionLeasesSupplier); - minByRetentionLeasesAndSize = Math.max(minBySize, minByRetentionLeases); - } final long minByAgeAndSize; if (minBySize == Long.MIN_VALUE && minByAge == Long.MIN_VALUE) { // both size and age are disabled; @@ -215,36 +177,7 @@ public class TranslogDeletionPolicy { minByAgeAndSize = Math.max(minByAge, minBySize); } long minByNumFiles = getMinTranslogGenByTotalFiles(readers, writer, retentionTotalFiles); - long minByTranslogGenSettings = Math.min(Math.max(minByAgeAndSize, minByNumFiles), minByLocks); - return Math.min(minByTranslogGenSettings, minByRetentionLeasesAndSize); - } - - /** - * Find the minimum translog generation by minimum retaining sequence number - * @deprecated EXPERT: this configuration is specific to CCR and will be moved to a plugin in the next release - */ - @Deprecated - static long getMinTranslogGenByRetentionLease( - List readers, - TranslogWriter writer, - Supplier retentionLeasesSupplier - ) { - long minGen = writer.getGeneration(); - final long minimumRetainingSequenceNumber = retentionLeasesSupplier.get() - .leases() - .stream() - .mapToLong(RetentionLease::retainingSequenceNumber) - .min() - .orElse(Long.MAX_VALUE); - - for (int i = readers.size() - 1; i >= 0; i--) { - final TranslogReader reader = readers.get(i); - if (reader.getCheckpoint().minSeqNo <= minimumRetainingSequenceNumber - && reader.getCheckpoint().maxSeqNo >= minimumRetainingSequenceNumber) { - minGen = Math.min(minGen, reader.getGeneration()); - } - } - return minGen; + return Math.min(Math.max(minByAgeAndSize, minByNumFiles), minByLocks); } static long getMinTranslogGenBySize(List readers, TranslogWriter writer, long retentionSizeInBytes) { diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 2adbe70a64c..eaf1c2488e1 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -1274,31 +1274,6 @@ public class MetadataCreateIndexServiceTests extends OpenSearchTestCase { ); } - public void testTranslogPruningBasedOnRetentionLeaseDeprecation() { - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - request.settings( - Settings.builder() - .put(INDEX_SOFT_DELETES_SETTING.getKey(), true) - .put(IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.getKey(), true) - .build() - ); - aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - Settings.EMPTY, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() - ); - assertWarnings( - "[index.plugins.replication.translog.retention_lease.pruning.enabled] setting " - + "was deprecated in OpenSearch and will be removed in a future release! " - + "See the breaking changes documentation for the next major version." - ); - } - private IndexTemplateMetadata addMatchingTemplate(Consumer configurator) { IndexTemplateMetadata.Builder builder = templateMetadataBuilder("template1", "te*"); configurator.accept(builder); diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index 890360d8a91..f3278a9e3d5 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -802,50 +802,4 @@ public class IndexSettingsTests extends OpenSearchTestCase { assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting)); assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting)); } - - public void testTranslogPruningSettingsWithSoftDeletesEnabled() { - Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_1_1_0); - - ByteSizeValue retentionSize = new ByteSizeValue(512, ByteSizeUnit.MB); - boolean translogPruningEnabled = randomBoolean(); - settings.put( - IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.getKey(), - translogPruningEnabled - ); - IndexMetadata metadata = newIndexMeta("index", settings.build()); - IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); - if (translogPruningEnabled) { - assertTrue(indexSettings.shouldPruneTranslogByRetentionLease()); - assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(retentionSize.getBytes())); - } else { - assertFalse(indexSettings.shouldPruneTranslogByRetentionLease()); - assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L)); - } - assertWarnings( - "[index.plugins.replication.translog.retention_lease.pruning.enabled] setting " - + "was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation " - + "for the next major version." - ); - } - - public void testTranslogPruningSettingsWithSoftDeletesDisabled() { - Settings.Builder settings = Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT); - boolean translogPruningEnabled = randomBoolean(); - ByteSizeValue retentionSize = new ByteSizeValue(512, ByteSizeUnit.MB); - settings.put( - IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.getKey(), - translogPruningEnabled - ); - IndexMetadata metadata = newIndexMeta("index", settings.build()); - IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); - assertFalse(indexSettings.shouldPruneTranslogByRetentionLease()); - assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(retentionSize.getBytes())); - assertWarnings( - "[index.plugins.replication.translog.retention_lease.pruning.enabled] setting " - + "was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation " - + "for the next major version." - ); - } } diff --git a/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java b/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java index 51e13b7301f..0d981b769fe 100644 --- a/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java +++ b/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java @@ -40,8 +40,6 @@ import org.opensearch.common.collect.Tuple; import org.opensearch.common.lease.Releasable; import org.opensearch.common.util.BigArrays; import org.opensearch.core.internal.io.IOUtils; -import org.opensearch.index.seqno.RetentionLease; -import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.mockito.Mockito; @@ -51,9 +49,7 @@ import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; -import java.util.function.Supplier; import static java.lang.Math.min; import static org.hamcrest.Matchers.equalTo; @@ -99,37 +95,6 @@ public class TranslogDeletionPolicyTests extends OpenSearchTestCase { } } - public void testWithRetentionLease() throws IOException { - long now = System.currentTimeMillis(); - Tuple, TranslogWriter> readersAndWriter = createReadersAndWriter(now); - List allGens = new ArrayList<>(readersAndWriter.v1()); - allGens.add(readersAndWriter.v2()); - Supplier retentionLeasesSupplier = createRetentionLeases(now, 0L, allGens.size() * TOTAL_OPS_IN_GEN - 1); - try { - final long minimumRetainingSequenceNumber = retentionLeasesSupplier.get() - .leases() - .stream() - .mapToLong(RetentionLease::retainingSequenceNumber) - .min() - .orElse(Long.MAX_VALUE); - - final long selectedReader = (minimumRetainingSequenceNumber / TOTAL_OPS_IN_GEN); - final long selectedGen = allGens.get((int) selectedReader).generation; - assertThat( - TranslogDeletionPolicy.getMinTranslogGenByRetentionLease( - readersAndWriter.v1(), - readersAndWriter.v2(), - retentionLeasesSupplier - ), - equalTo(selectedGen) - ); - - } finally { - IOUtils.close(readersAndWriter.v1()); - IOUtils.close(readersAndWriter.v2()); - } - } - public void testAgeRetention() throws IOException { long now = System.currentTimeMillis(); Tuple, TranslogWriter> readersAndWriter = createReadersAndWriter(now); @@ -186,49 +151,6 @@ public class TranslogDeletionPolicyTests extends OpenSearchTestCase { } } - public void testBySizeAndRetentionLease() throws Exception { - long now = System.currentTimeMillis(); - Tuple, TranslogWriter> readersAndWriter = createReadersAndWriter(now); - List allGens = new ArrayList<>(readersAndWriter.v1()); - allGens.add(readersAndWriter.v2()); - try { - int selectedReader = randomIntBetween(0, allGens.size() - 1); - final long selectedGeneration = allGens.get(selectedReader).generation; - // Retaining seqno is part of lower gen - long size = allGens.stream().skip(selectedReader).map(BaseTranslogReader::sizeInBytes).reduce(Long::sum).get(); - Supplier retentionLeasesSupplier = createRetentionLeases(now, 0L, selectedGeneration * TOTAL_OPS_IN_GEN - 1); - TranslogDeletionPolicy deletionPolicy = new MockDeletionPolicy( - now, - size, - Integer.MAX_VALUE, - Integer.MAX_VALUE, - retentionLeasesSupplier - ); - assertThat(deletionPolicy.minTranslogGenRequired(readersAndWriter.v1(), readersAndWriter.v2()), equalTo(selectedGeneration)); - assertThat( - TranslogDeletionPolicy.getMinTranslogGenByAge( - readersAndWriter.v1(), - readersAndWriter.v2(), - 100L, - System.currentTimeMillis() - ), - equalTo(readersAndWriter.v2().generation) - ); - - // Retention lease is part of higher gen - retentionLeasesSupplier = createRetentionLeases( - now, - selectedGeneration * TOTAL_OPS_IN_GEN, - allGens.size() * TOTAL_OPS_IN_GEN + TOTAL_OPS_IN_GEN - 1 - ); - deletionPolicy = new MockDeletionPolicy(now, size, Long.MIN_VALUE, Integer.MAX_VALUE, retentionLeasesSupplier); - assertThat(deletionPolicy.minTranslogGenRequired(readersAndWriter.v1(), readersAndWriter.v2()), equalTo(selectedGeneration)); - } finally { - IOUtils.close(readersAndWriter.v1()); - IOUtils.close(readersAndWriter.v2()); - } - } - /** * Tests that age trumps size but recovery trumps both. */ @@ -353,17 +275,6 @@ public class TranslogDeletionPolicyTests extends OpenSearchTestCase { return new Tuple<>(readers, writer); } - private Supplier createRetentionLeases(final Long now, final Long lowestSeqNo, final Long highestSeqNo) - throws IOException { - LinkedList leases = new LinkedList<>(); - final int numberOfLeases = randomIntBetween(1, 5); - for (int i = 0; i < numberOfLeases; i++) { - long seqNo = randomLongBetween(lowestSeqNo, highestSeqNo); - leases.add(new RetentionLease("test_" + i, seqNo, now - (numberOfLeases - i) * 1000, "test")); - } - return () -> new RetentionLeases(1L, 1L, leases); - } - private static class MockDeletionPolicy extends TranslogDeletionPolicy { long now; @@ -373,17 +284,6 @@ public class TranslogDeletionPolicyTests extends OpenSearchTestCase { this.now = now; } - MockDeletionPolicy( - long now, - long retentionSizeInBytes, - long maxRetentionAgeInMillis, - int maxRetentionTotalFiles, - Supplier retentionLeasesSupplier - ) { - super(retentionSizeInBytes, maxRetentionAgeInMillis, maxRetentionTotalFiles, retentionLeasesSupplier); - this.now = now; - } - @Override protected long currentTime() { return now;