Remove deprecated settings and logic for translog pruning by retention lease. (#1416)
The settings and the corresponding logic for translog pruning by retention lease which were added as part of #1100 have been deprecated. This commit removes those deprecated code in favor of an extension point for providing a custom TranslogDeletionPolicy. Signed-off-by: Rabi Panda <adnapibar@gmail.com>
This commit is contained in:
parent
f08ee2c67c
commit
2cb71a33b8
|
@ -789,44 +789,6 @@ public class UpdateSettingsIT extends OpenSearchIntegTestCase {
|
|||
assertThat(newSettingsVersion, equalTo(1 + settingsVersion));
|
||||
}
|
||||
|
||||
public void testTranslogPruningSetting() {
|
||||
createIndex("test");
|
||||
ensureGreen("test");
|
||||
final String REPLICATION_TRANSLOG_SETTING = "index.plugins.replication.translog.retention_lease.pruning.enabled";
|
||||
final long settingsVersion = client().admin()
|
||||
.cluster()
|
||||
.prepareState()
|
||||
.get()
|
||||
.getState()
|
||||
.metadata()
|
||||
.index("test")
|
||||
.getSettingsVersion();
|
||||
GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings("test").get();
|
||||
boolean shouldPruneTranslogByRetentionLease = Boolean.parseBoolean(
|
||||
settingsResponse.getSetting("test", REPLICATION_TRANSLOG_SETTING)
|
||||
);
|
||||
|
||||
assertFalse(shouldPruneTranslogByRetentionLease);
|
||||
assertAcked(
|
||||
client().admin()
|
||||
.indices()
|
||||
.prepareUpdateSettings("test")
|
||||
.setSettings(Settings.builder().put("index.plugins.replication.translog.retention_lease.pruning.enabled", true))
|
||||
);
|
||||
final long newSettingsVersion = client().admin()
|
||||
.cluster()
|
||||
.prepareState()
|
||||
.get()
|
||||
.getState()
|
||||
.metadata()
|
||||
.index("test")
|
||||
.getSettingsVersion();
|
||||
assertThat(newSettingsVersion, equalTo(1 + settingsVersion));
|
||||
settingsResponse = client().admin().indices().prepareGetSettings("test").get();
|
||||
shouldPruneTranslogByRetentionLease = Boolean.parseBoolean(settingsResponse.getSetting("test", REPLICATION_TRANSLOG_SETTING));
|
||||
assertTrue(shouldPruneTranslogByRetentionLease);
|
||||
}
|
||||
|
||||
/*
|
||||
* Test that we are able to set the setting index.number_of_replicas to the default.
|
||||
*/
|
||||
|
|
|
@ -1464,13 +1464,6 @@ public class MetadataCreateIndexService {
|
|||
+ "and [index.translog.retention.size] are deprecated and effectively ignored. "
|
||||
+ "They will be removed in a future version."
|
||||
);
|
||||
} else if (IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.exists(indexSettings)) {
|
||||
DEPRECATION_LOGGER.deprecate(
|
||||
"translog_pruning_retention_lease",
|
||||
"[index.plugins.replication.translog.retention_lease.pruning.enabled] "
|
||||
+ "setting was deprecated in OpenSearch and will be removed in a future release! "
|
||||
+ "See the breaking changes documentation for the next major version."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -158,7 +158,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
|
|||
EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING,
|
||||
EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING,
|
||||
IndexSettings.INDEX_FLUSH_AFTER_MERGE_THRESHOLD_SIZE_SETTING,
|
||||
IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING,
|
||||
IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING,
|
||||
IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING,
|
||||
IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING,
|
||||
|
@ -244,6 +243,9 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
|
|||
case "index.shrink.source.name":
|
||||
case IndexMetadata.INDEX_RESIZE_SOURCE_UUID_KEY:
|
||||
case IndexMetadata.INDEX_RESIZE_SOURCE_NAME_KEY:
|
||||
// we keep this setting for BWC to support indexes created in 1.1.0
|
||||
// this can be removed in OpenSearch 3.0
|
||||
case "index.plugins.replication.translog.retention_lease.pruning.enabled":
|
||||
return true;
|
||||
default:
|
||||
return IndexMetadata.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getRawKey().match(key);
|
||||
|
|
|
@ -362,19 +362,6 @@ public final class IndexSettings {
|
|||
Property.Final
|
||||
);
|
||||
|
||||
/**
|
||||
* Specifies if the index translog should prune based on retention leases.
|
||||
* @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release
|
||||
*/
|
||||
@Deprecated
|
||||
public static final Setting<Boolean> INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING = Setting.boolSetting(
|
||||
"index.plugins.replication.translog.retention_lease.pruning.enabled",
|
||||
false,
|
||||
Property.IndexScope,
|
||||
Property.Dynamic,
|
||||
Property.Deprecated
|
||||
);
|
||||
|
||||
/**
|
||||
* Controls how many soft-deleted documents will be kept around before being merged away. Keeping more deleted
|
||||
* documents increases the chance of operation-based recoveries and allows querying a longer history of documents.
|
||||
|
@ -408,12 +395,10 @@ public final class IndexSettings {
|
|||
* the chance of ops based recoveries for indices with soft-deletes disabled.
|
||||
* This setting will be ignored if soft-deletes is used in peer recoveries (default in 7.4).
|
||||
**/
|
||||
@Deprecated
|
||||
private static final ByteSizeValue DEFAULT_TRANSLOG_RETENTION_SIZE = new ByteSizeValue(512, ByteSizeUnit.MB);
|
||||
|
||||
public static final Setting<ByteSizeValue> INDEX_TRANSLOG_RETENTION_SIZE_SETTING = Setting.byteSizeSetting(
|
||||
"index.translog.retention.size",
|
||||
settings -> DEFAULT_TRANSLOG_RETENTION_SIZE.getStringRep(),
|
||||
settings -> shouldDisableTranslogRetention(settings) ? "-1" : "512MB",
|
||||
Property.Dynamic,
|
||||
Property.IndexScope
|
||||
);
|
||||
|
@ -546,8 +531,6 @@ public final class IndexSettings {
|
|||
private final IndexScopedSettings scopedSettings;
|
||||
private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
|
||||
private final boolean softDeleteEnabled;
|
||||
@Deprecated
|
||||
private volatile boolean translogPruningByRetentionLease;
|
||||
private volatile long softDeleteRetentionOperations;
|
||||
|
||||
private volatile long retentionLeaseMillis;
|
||||
|
@ -684,10 +667,6 @@ public final class IndexSettings {
|
|||
mergeSchedulerConfig = new MergeSchedulerConfig(this);
|
||||
gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis();
|
||||
softDeleteEnabled = version.onOrAfter(LegacyESVersion.V_6_5_0) && scopedSettings.get(INDEX_SOFT_DELETES_SETTING);
|
||||
// @todo move to CCR plugin
|
||||
this.translogPruningByRetentionLease = version.onOrAfter(Version.V_1_1_0)
|
||||
&& softDeleteEnabled
|
||||
&& scopedSettings.get(INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING);
|
||||
softDeleteRetentionOperations = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING);
|
||||
retentionLeaseMillis = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING).millis();
|
||||
warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING);
|
||||
|
@ -771,10 +750,6 @@ public final class IndexSettings {
|
|||
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING, this::setGenerationThresholdSize);
|
||||
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_RETENTION_AGE_SETTING, this::setTranslogRetentionAge);
|
||||
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_RETENTION_SIZE_SETTING, this::setTranslogRetentionSize);
|
||||
scopedSettings.addSettingsUpdateConsumer(
|
||||
INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING,
|
||||
this::setTranslogPruningByRetentionLease
|
||||
);
|
||||
scopedSettings.addSettingsUpdateConsumer(INDEX_REFRESH_INTERVAL_SETTING, this::setRefreshInterval);
|
||||
scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners);
|
||||
scopedSettings.addSettingsUpdateConsumer(MAX_ANALYZED_OFFSET_SETTING, this::setHighlightMaxAnalyzedOffset);
|
||||
|
@ -807,20 +782,8 @@ public final class IndexSettings {
|
|||
this.flushAfterMergeThresholdSize = byteSizeValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable or disable translog pruning by retention lease requirement
|
||||
*
|
||||
* @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release
|
||||
*/
|
||||
@Deprecated
|
||||
private void setTranslogPruningByRetentionLease(boolean enabled) {
|
||||
this.translogPruningByRetentionLease = INDEX_SOFT_DELETES_SETTING.get(settings) && enabled;
|
||||
}
|
||||
|
||||
private void setTranslogRetentionSize(ByteSizeValue byteSizeValue) {
|
||||
if (shouldDisableTranslogRetention(settings)
|
||||
&& shouldPruneTranslogByRetentionLease(settings) == false
|
||||
&& byteSizeValue.getBytes() >= 0) {
|
||||
if (shouldDisableTranslogRetention(settings) && byteSizeValue.getBytes() >= 0) {
|
||||
// ignore the translog retention settings if soft-deletes enabled
|
||||
this.translogRetentionSize = new ByteSizeValue(-1);
|
||||
} else {
|
||||
|
@ -1033,11 +996,7 @@ public final class IndexSettings {
|
|||
* Returns the transaction log retention size which controls how much of the translog is kept around to allow for ops based recoveries
|
||||
*/
|
||||
public ByteSizeValue getTranslogRetentionSize() {
|
||||
if (shouldDisableTranslogRetention(settings) && shouldPruneTranslogByRetentionLease(settings) == false) {
|
||||
return new ByteSizeValue(-1);
|
||||
} else if (shouldPruneTranslogByRetentionLease(settings) && translogRetentionSize.getBytes() == -1) {
|
||||
return DEFAULT_TRANSLOG_RETENTION_SIZE;
|
||||
}
|
||||
assert shouldDisableTranslogRetention(settings) == false || translogRetentionSize.getBytes() == -1L : translogRetentionSize;
|
||||
return translogRetentionSize;
|
||||
}
|
||||
|
||||
|
@ -1303,24 +1262,6 @@ public final class IndexSettings {
|
|||
this.requiredPipeline = requiredPipeline;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> if translog ops should be pruned based on retention lease
|
||||
* @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release
|
||||
*/
|
||||
@Deprecated
|
||||
public boolean shouldPruneTranslogByRetentionLease() {
|
||||
return translogPruningByRetentionLease;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> if translog ops should be pruned based on retention lease
|
||||
* @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release
|
||||
*/
|
||||
@Deprecated
|
||||
public static boolean shouldPruneTranslogByRetentionLease(Settings settings) {
|
||||
return INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.get(settings);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> if soft-delete is enabled.
|
||||
*/
|
||||
|
|
|
@ -1998,22 +1998,7 @@ public abstract class Engine implements Closeable {
|
|||
}
|
||||
|
||||
public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) {
|
||||
onSettingsChanged(translogRetentionAge, translogRetentionSize, softDeletesRetentionOps, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* callback when index settings change
|
||||
*
|
||||
* @deprecated EXPERT: this method is specific to CCR and will be moved to a plugin in the next release
|
||||
*/
|
||||
@Deprecated
|
||||
public void onSettingsChanged(
|
||||
TimeValue translogRetentionAge,
|
||||
ByteSizeValue translogRetentionSize,
|
||||
long softDeletesRetentionOps,
|
||||
boolean translogPruningByRetentionLease
|
||||
) {
|
||||
// @todo this is overly silent; make this abstract and force derived classes to noop in the next release
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -2801,12 +2801,7 @@ public class InternalEngine extends Engine {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void onSettingsChanged(
|
||||
TimeValue translogRetentionAge,
|
||||
ByteSizeValue translogRetentionSize,
|
||||
long softDeletesRetentionOps,
|
||||
boolean translogPruningByRetentionLease
|
||||
) {
|
||||
public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) {
|
||||
mergeScheduler.refreshConfig();
|
||||
// config().isEnableGcDeletes() or config.getGcDeletesInMillis() may have changed:
|
||||
maybePruneDeletes();
|
||||
|
@ -2819,7 +2814,6 @@ public class InternalEngine extends Engine {
|
|||
final TranslogDeletionPolicy translogDeletionPolicy = translog.getDeletionPolicy();
|
||||
translogDeletionPolicy.setRetentionAgeInMillis(translogRetentionAge.millis());
|
||||
translogDeletionPolicy.setRetentionSizeInBytes(translogRetentionSize.getBytes());
|
||||
translogDeletionPolicy.shouldPruneTranslogByRetentionLease(translogPruningByRetentionLease);
|
||||
softDeletesPolicy.setRetentionOperations(softDeletesRetentionOps);
|
||||
}
|
||||
|
||||
|
|
|
@ -2278,11 +2278,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
|
|||
final boolean disableTranslogRetention = indexSettings.isSoftDeleteEnabled() && useRetentionLeasesInPeerRecovery;
|
||||
engineOrNull.onSettingsChanged(
|
||||
disableTranslogRetention ? TimeValue.MINUS_ONE : indexSettings.getTranslogRetentionAge(),
|
||||
disableTranslogRetention && !indexSettings.shouldPruneTranslogByRetentionLease()
|
||||
? new ByteSizeValue(-1)
|
||||
: indexSettings.getTranslogRetentionSize(),
|
||||
indexSettings.getSoftDeleteRetentionOperations(),
|
||||
indexSettings.shouldPruneTranslogByRetentionLease()
|
||||
disableTranslogRetention ? new ByteSizeValue(-1) : indexSettings.getTranslogRetentionSize(),
|
||||
indexSettings.getSoftDeleteRetentionOperations()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -35,8 +35,6 @@ package org.opensearch.index.translog;
|
|||
import org.apache.lucene.util.Counter;
|
||||
import org.opensearch.Assertions;
|
||||
import org.opensearch.common.lease.Releasable;
|
||||
import org.opensearch.index.seqno.RetentionLease;
|
||||
import org.opensearch.index.seqno.RetentionLeases;
|
||||
import org.opensearch.index.seqno.SequenceNumbers;
|
||||
|
||||
import java.io.IOException;
|
||||
|
@ -45,16 +43,10 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
public class TranslogDeletionPolicy {
|
||||
|
||||
private final Map<Object, RuntimeException> openTranslogRef;
|
||||
/**
|
||||
* @deprecated EXPERT: this supplier is specific to CCR and will be moved to a plugin in the next release
|
||||
*/
|
||||
@Deprecated
|
||||
private Supplier<RetentionLeases> retentionLeasesSupplier;
|
||||
|
||||
public void assertNoOpenTranslogRefs() {
|
||||
if (openTranslogRef.isEmpty() == false) {
|
||||
|
@ -94,21 +86,6 @@ public class TranslogDeletionPolicy {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a TranslogDeletionPolicy to include pruning by retention leases
|
||||
* @deprecated EXPERT: this ctor is specific to CCR and will be moved to a plugin in the next release
|
||||
*/
|
||||
@Deprecated
|
||||
public TranslogDeletionPolicy(
|
||||
long retentionSizeInBytes,
|
||||
long retentionAgeInMillis,
|
||||
int retentionTotalFiles,
|
||||
Supplier<RetentionLeases> retentionLeasesSupplier
|
||||
) {
|
||||
this(retentionSizeInBytes, retentionAgeInMillis, retentionTotalFiles);
|
||||
this.retentionLeasesSupplier = retentionLeasesSupplier;
|
||||
}
|
||||
|
||||
public synchronized void setLocalCheckpointOfSafeCommit(long newCheckpoint) {
|
||||
if (newCheckpoint < this.localCheckpointOfSafeCommit) {
|
||||
throw new IllegalArgumentException(
|
||||
|
@ -135,15 +112,6 @@ public class TranslogDeletionPolicy {
|
|||
this.retentionTotalFiles = retentionTotalFiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Should the translog be pruned by the retention lease heuristic
|
||||
* @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release
|
||||
*/
|
||||
@Deprecated
|
||||
public synchronized void shouldPruneTranslogByRetentionLease(boolean translogPruneByRetentionLease) {
|
||||
this.shouldPruneTranslogByRetentionLease = translogPruneByRetentionLease;
|
||||
}
|
||||
|
||||
/**
|
||||
* acquires the basis generation for a new snapshot. Any translog generation above, and including, the returned generation
|
||||
* will not be deleted until the returned {@link Releasable} is closed.
|
||||
|
@ -201,12 +169,6 @@ public class TranslogDeletionPolicy {
|
|||
long minByLocks = getMinTranslogGenRequiredByLocks();
|
||||
long minByAge = getMinTranslogGenByAge(readers, writer, retentionAgeInMillis, currentTime());
|
||||
long minBySize = getMinTranslogGenBySize(readers, writer, retentionSizeInBytes);
|
||||
long minByRetentionLeasesAndSize = Long.MAX_VALUE;
|
||||
if (shouldPruneTranslogByRetentionLease) {
|
||||
// If retention size is specified, size takes precedence.
|
||||
long minByRetentionLeases = getMinTranslogGenByRetentionLease(readers, writer, retentionLeasesSupplier);
|
||||
minByRetentionLeasesAndSize = Math.max(minBySize, minByRetentionLeases);
|
||||
}
|
||||
final long minByAgeAndSize;
|
||||
if (minBySize == Long.MIN_VALUE && minByAge == Long.MIN_VALUE) {
|
||||
// both size and age are disabled;
|
||||
|
@ -215,36 +177,7 @@ public class TranslogDeletionPolicy {
|
|||
minByAgeAndSize = Math.max(minByAge, minBySize);
|
||||
}
|
||||
long minByNumFiles = getMinTranslogGenByTotalFiles(readers, writer, retentionTotalFiles);
|
||||
long minByTranslogGenSettings = Math.min(Math.max(minByAgeAndSize, minByNumFiles), minByLocks);
|
||||
return Math.min(minByTranslogGenSettings, minByRetentionLeasesAndSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the minimum translog generation by minimum retaining sequence number
|
||||
* @deprecated EXPERT: this configuration is specific to CCR and will be moved to a plugin in the next release
|
||||
*/
|
||||
@Deprecated
|
||||
static long getMinTranslogGenByRetentionLease(
|
||||
List<TranslogReader> readers,
|
||||
TranslogWriter writer,
|
||||
Supplier<RetentionLeases> retentionLeasesSupplier
|
||||
) {
|
||||
long minGen = writer.getGeneration();
|
||||
final long minimumRetainingSequenceNumber = retentionLeasesSupplier.get()
|
||||
.leases()
|
||||
.stream()
|
||||
.mapToLong(RetentionLease::retainingSequenceNumber)
|
||||
.min()
|
||||
.orElse(Long.MAX_VALUE);
|
||||
|
||||
for (int i = readers.size() - 1; i >= 0; i--) {
|
||||
final TranslogReader reader = readers.get(i);
|
||||
if (reader.getCheckpoint().minSeqNo <= minimumRetainingSequenceNumber
|
||||
&& reader.getCheckpoint().maxSeqNo >= minimumRetainingSequenceNumber) {
|
||||
minGen = Math.min(minGen, reader.getGeneration());
|
||||
}
|
||||
}
|
||||
return minGen;
|
||||
return Math.min(Math.max(minByAgeAndSize, minByNumFiles), minByLocks);
|
||||
}
|
||||
|
||||
static long getMinTranslogGenBySize(List<TranslogReader> readers, TranslogWriter writer, long retentionSizeInBytes) {
|
||||
|
|
|
@ -1274,31 +1274,6 @@ public class MetadataCreateIndexServiceTests extends OpenSearchTestCase {
|
|||
);
|
||||
}
|
||||
|
||||
public void testTranslogPruningBasedOnRetentionLeaseDeprecation() {
|
||||
request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
|
||||
request.settings(
|
||||
Settings.builder()
|
||||
.put(INDEX_SOFT_DELETES_SETTING.getKey(), true)
|
||||
.put(IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.getKey(), true)
|
||||
.build()
|
||||
);
|
||||
aggregateIndexSettings(
|
||||
ClusterState.EMPTY_STATE,
|
||||
request,
|
||||
Settings.EMPTY,
|
||||
null,
|
||||
Settings.EMPTY,
|
||||
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
|
||||
randomShardLimitService(),
|
||||
Collections.emptySet()
|
||||
);
|
||||
assertWarnings(
|
||||
"[index.plugins.replication.translog.retention_lease.pruning.enabled] setting "
|
||||
+ "was deprecated in OpenSearch and will be removed in a future release! "
|
||||
+ "See the breaking changes documentation for the next major version."
|
||||
);
|
||||
}
|
||||
|
||||
private IndexTemplateMetadata addMatchingTemplate(Consumer<IndexTemplateMetadata.Builder> configurator) {
|
||||
IndexTemplateMetadata.Builder builder = templateMetadataBuilder("template1", "te*");
|
||||
configurator.accept(builder);
|
||||
|
|
|
@ -802,50 +802,4 @@ public class IndexSettingsTests extends OpenSearchTestCase {
|
|||
assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting));
|
||||
assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting));
|
||||
}
|
||||
|
||||
public void testTranslogPruningSettingsWithSoftDeletesEnabled() {
|
||||
Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_1_1_0);
|
||||
|
||||
ByteSizeValue retentionSize = new ByteSizeValue(512, ByteSizeUnit.MB);
|
||||
boolean translogPruningEnabled = randomBoolean();
|
||||
settings.put(
|
||||
IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.getKey(),
|
||||
translogPruningEnabled
|
||||
);
|
||||
IndexMetadata metadata = newIndexMeta("index", settings.build());
|
||||
IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY);
|
||||
if (translogPruningEnabled) {
|
||||
assertTrue(indexSettings.shouldPruneTranslogByRetentionLease());
|
||||
assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(retentionSize.getBytes()));
|
||||
} else {
|
||||
assertFalse(indexSettings.shouldPruneTranslogByRetentionLease());
|
||||
assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L));
|
||||
}
|
||||
assertWarnings(
|
||||
"[index.plugins.replication.translog.retention_lease.pruning.enabled] setting "
|
||||
+ "was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation "
|
||||
+ "for the next major version."
|
||||
);
|
||||
}
|
||||
|
||||
public void testTranslogPruningSettingsWithSoftDeletesDisabled() {
|
||||
Settings.Builder settings = Settings.builder()
|
||||
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)
|
||||
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT);
|
||||
boolean translogPruningEnabled = randomBoolean();
|
||||
ByteSizeValue retentionSize = new ByteSizeValue(512, ByteSizeUnit.MB);
|
||||
settings.put(
|
||||
IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.getKey(),
|
||||
translogPruningEnabled
|
||||
);
|
||||
IndexMetadata metadata = newIndexMeta("index", settings.build());
|
||||
IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY);
|
||||
assertFalse(indexSettings.shouldPruneTranslogByRetentionLease());
|
||||
assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(retentionSize.getBytes()));
|
||||
assertWarnings(
|
||||
"[index.plugins.replication.translog.retention_lease.pruning.enabled] setting "
|
||||
+ "was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation "
|
||||
+ "for the next major version."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -40,8 +40,6 @@ import org.opensearch.common.collect.Tuple;
|
|||
import org.opensearch.common.lease.Releasable;
|
||||
import org.opensearch.common.util.BigArrays;
|
||||
import org.opensearch.core.internal.io.IOUtils;
|
||||
import org.opensearch.index.seqno.RetentionLease;
|
||||
import org.opensearch.index.seqno.RetentionLeases;
|
||||
import org.opensearch.index.shard.ShardId;
|
||||
import org.opensearch.test.OpenSearchTestCase;
|
||||
import org.mockito.Mockito;
|
||||
|
@ -51,9 +49,7 @@ import java.nio.channels.FileChannel;
|
|||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static java.lang.Math.min;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
|
@ -99,37 +95,6 @@ public class TranslogDeletionPolicyTests extends OpenSearchTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
public void testWithRetentionLease() throws IOException {
|
||||
long now = System.currentTimeMillis();
|
||||
Tuple<List<TranslogReader>, TranslogWriter> readersAndWriter = createReadersAndWriter(now);
|
||||
List<BaseTranslogReader> allGens = new ArrayList<>(readersAndWriter.v1());
|
||||
allGens.add(readersAndWriter.v2());
|
||||
Supplier<RetentionLeases> retentionLeasesSupplier = createRetentionLeases(now, 0L, allGens.size() * TOTAL_OPS_IN_GEN - 1);
|
||||
try {
|
||||
final long minimumRetainingSequenceNumber = retentionLeasesSupplier.get()
|
||||
.leases()
|
||||
.stream()
|
||||
.mapToLong(RetentionLease::retainingSequenceNumber)
|
||||
.min()
|
||||
.orElse(Long.MAX_VALUE);
|
||||
|
||||
final long selectedReader = (minimumRetainingSequenceNumber / TOTAL_OPS_IN_GEN);
|
||||
final long selectedGen = allGens.get((int) selectedReader).generation;
|
||||
assertThat(
|
||||
TranslogDeletionPolicy.getMinTranslogGenByRetentionLease(
|
||||
readersAndWriter.v1(),
|
||||
readersAndWriter.v2(),
|
||||
retentionLeasesSupplier
|
||||
),
|
||||
equalTo(selectedGen)
|
||||
);
|
||||
|
||||
} finally {
|
||||
IOUtils.close(readersAndWriter.v1());
|
||||
IOUtils.close(readersAndWriter.v2());
|
||||
}
|
||||
}
|
||||
|
||||
public void testAgeRetention() throws IOException {
|
||||
long now = System.currentTimeMillis();
|
||||
Tuple<List<TranslogReader>, TranslogWriter> readersAndWriter = createReadersAndWriter(now);
|
||||
|
@ -186,49 +151,6 @@ public class TranslogDeletionPolicyTests extends OpenSearchTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
public void testBySizeAndRetentionLease() throws Exception {
|
||||
long now = System.currentTimeMillis();
|
||||
Tuple<List<TranslogReader>, TranslogWriter> readersAndWriter = createReadersAndWriter(now);
|
||||
List<BaseTranslogReader> allGens = new ArrayList<>(readersAndWriter.v1());
|
||||
allGens.add(readersAndWriter.v2());
|
||||
try {
|
||||
int selectedReader = randomIntBetween(0, allGens.size() - 1);
|
||||
final long selectedGeneration = allGens.get(selectedReader).generation;
|
||||
// Retaining seqno is part of lower gen
|
||||
long size = allGens.stream().skip(selectedReader).map(BaseTranslogReader::sizeInBytes).reduce(Long::sum).get();
|
||||
Supplier<RetentionLeases> retentionLeasesSupplier = createRetentionLeases(now, 0L, selectedGeneration * TOTAL_OPS_IN_GEN - 1);
|
||||
TranslogDeletionPolicy deletionPolicy = new MockDeletionPolicy(
|
||||
now,
|
||||
size,
|
||||
Integer.MAX_VALUE,
|
||||
Integer.MAX_VALUE,
|
||||
retentionLeasesSupplier
|
||||
);
|
||||
assertThat(deletionPolicy.minTranslogGenRequired(readersAndWriter.v1(), readersAndWriter.v2()), equalTo(selectedGeneration));
|
||||
assertThat(
|
||||
TranslogDeletionPolicy.getMinTranslogGenByAge(
|
||||
readersAndWriter.v1(),
|
||||
readersAndWriter.v2(),
|
||||
100L,
|
||||
System.currentTimeMillis()
|
||||
),
|
||||
equalTo(readersAndWriter.v2().generation)
|
||||
);
|
||||
|
||||
// Retention lease is part of higher gen
|
||||
retentionLeasesSupplier = createRetentionLeases(
|
||||
now,
|
||||
selectedGeneration * TOTAL_OPS_IN_GEN,
|
||||
allGens.size() * TOTAL_OPS_IN_GEN + TOTAL_OPS_IN_GEN - 1
|
||||
);
|
||||
deletionPolicy = new MockDeletionPolicy(now, size, Long.MIN_VALUE, Integer.MAX_VALUE, retentionLeasesSupplier);
|
||||
assertThat(deletionPolicy.minTranslogGenRequired(readersAndWriter.v1(), readersAndWriter.v2()), equalTo(selectedGeneration));
|
||||
} finally {
|
||||
IOUtils.close(readersAndWriter.v1());
|
||||
IOUtils.close(readersAndWriter.v2());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests that age trumps size but recovery trumps both.
|
||||
*/
|
||||
|
@ -353,17 +275,6 @@ public class TranslogDeletionPolicyTests extends OpenSearchTestCase {
|
|||
return new Tuple<>(readers, writer);
|
||||
}
|
||||
|
||||
private Supplier<RetentionLeases> createRetentionLeases(final Long now, final Long lowestSeqNo, final Long highestSeqNo)
|
||||
throws IOException {
|
||||
LinkedList<RetentionLease> leases = new LinkedList<>();
|
||||
final int numberOfLeases = randomIntBetween(1, 5);
|
||||
for (int i = 0; i < numberOfLeases; i++) {
|
||||
long seqNo = randomLongBetween(lowestSeqNo, highestSeqNo);
|
||||
leases.add(new RetentionLease("test_" + i, seqNo, now - (numberOfLeases - i) * 1000, "test"));
|
||||
}
|
||||
return () -> new RetentionLeases(1L, 1L, leases);
|
||||
}
|
||||
|
||||
private static class MockDeletionPolicy extends TranslogDeletionPolicy {
|
||||
|
||||
long now;
|
||||
|
@ -373,17 +284,6 @@ public class TranslogDeletionPolicyTests extends OpenSearchTestCase {
|
|||
this.now = now;
|
||||
}
|
||||
|
||||
MockDeletionPolicy(
|
||||
long now,
|
||||
long retentionSizeInBytes,
|
||||
long maxRetentionAgeInMillis,
|
||||
int maxRetentionTotalFiles,
|
||||
Supplier<RetentionLeases> retentionLeasesSupplier
|
||||
) {
|
||||
super(retentionSizeInBytes, maxRetentionAgeInMillis, maxRetentionTotalFiles, retentionLeasesSupplier);
|
||||
this.now = now;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected long currentTime() {
|
||||
return now;
|
||||
|
|
Loading…
Reference in New Issue