honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable soft-deletes by default on 7.0.0 or later (#36141) 

This change enables soft-deletes by default on ES 7.0.0 or later.

Relates #33222

Co-authored-by: Jason Tedor <jason@tedor.me>
This commit is contained in:
Nhat Nguyen 2018-12-11 18:58:49 -05:00 committed by GitHub
parent 797f985067
commit 51800de2a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 58 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
docs/reference/ccr/getting-started.asciidoc
View File

@ -156,12 +156,12 @@ remote cluster.
[[ccr-getting-started-leader-index]]
=== Creating a leader index
Leader indices require special index settings to ensure that the operations that
need to be replicated are available when the
follower requests them from the leader. These settings are used to enable soft
deletes on the leader index and to control how many soft deletes are retained. A
_soft delete_ occurs whenever a document is deleted or updated. Soft deletes can
be enabled only on new indices created on or after {es} 6.5.0.
Leader indices require a special index setting to ensure that the operations
that need to be replicated are available when the follower requests them from
the leader. This setting is used to control how many soft deletes are retained.
A _soft delete_ occurs whenever a document is deleted or updated. Soft deletes
can be enabled only on new indices created on or after {es} 6.5.0, and enabled
by default on new indices created on or after {es} 7.0.0.
In the following example, we will create a leader index in the remote cluster:
@ -174,9 +174,8 @@ PUT /server-metrics
"number_of_shards" : 1,
"number_of_replicas" : 0,
"soft_deletes" : {
"enabled" : true, <1>
"retention" : {
"operations" : 1024 <2>
"operations" : 1024 <1>
}
}
}
@ -212,8 +211,7 @@ PUT /server-metrics
--------------------------------------------------
// CONSOLE
// TEST[continued]
<1> Enables soft deletes on the leader index.
<2> Sets that up to 1024 soft deletes will be retained.
<1> Sets that up to 1024 soft deletes will be retained.
[float]
[[ccr-getting-started-follower-index]]

13
docs/reference/ccr/requirements.asciidoc
View File

@ -16,10 +16,8 @@ shards and made available to the follower shard tasks as it replays the history
of operations.
Soft deletes must be enabled for indices that you want to use as leader
indices. Enabling soft deletes requires the addition of some index settings at
index creation time. You must add these settings to your create index
requests or to the index templates that you use to manage the creation of new
indices.
indices. Soft deletes are enabled by default on new indices created on
or after {es} 7.0.0.
IMPORTANT: This means that {ccr} can not be used on existing indices. If you have
existing data that you want to replicate from another cluster, you must
@ -34,7 +32,7 @@ enabled.
Whether or not soft deletes are enabled on the index. Soft deletes can only be
configured at index creation and only on indices created on or after 6.5.0. The
default value is `false`.
default value is `true`.
`index.soft_deletes.retention.operations`::
@ -49,8 +47,8 @@ For more information about index settings, see {ref}/index-modules.html[Index mo
==== Setting soft deletes on indices created by APM Server or Beats
If you want to replicate indices created by APM Server or Beats, and are
allowing APM Server or Beats to manage index templates, you need to enable
soft deletes on the underlying index templates. To enable soft deletes on the
allowing APM Server or Beats to manage index templates, you need to configure
soft deletes on the underlying index templates. To configure soft deletes on the
underlying index templates, add the following changes to the relevant APM Server
or Beats configuration file.
@ -58,7 +56,6 @@ or Beats configuration file.
----------------------------------------------------------------------
setup.template.overwrite: true
setup.template.settings:
index.soft_deletes.enabled: true
index.soft_deletes.retention.operations: 1024
----------------------------------------------------------------------

3
docs/reference/indices/flush.asciidoc
View File

@ -101,7 +101,8 @@ which returns something similar to:
"translog_generation" : "2",
"max_seq_no" : "-1",
"sync_id" : "AVvFY-071siAOuFGEO9P", <1>
"max_unsafe_auto_id_timestamp" : "-1"
"max_unsafe_auto_id_timestamp" : "-1",
"min_retained_seq_no": "0"
},
"num_docs" : 0
}

2
qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
View File

@ -970,7 +970,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
mappingsAndSettings.startObject("settings");
mappingsAndSettings.field("number_of_shards", 1);
mappingsAndSettings.field("number_of_replicas", 1);
if (getOldClusterVersion().onOrAfter(Version.V_6_5_0)) {
if (getOldClusterVersion().onOrAfter(Version.V_6_5_0) && randomBoolean()) {
mappingsAndSettings.field("soft_deletes.enabled", true);
}
mappingsAndSettings.endObject();

2
qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
View File

@ -285,7 +285,7 @@ public class RecoveryIT extends AbstractRollingTestCase {
// before timing out
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
if (getNodeId(v -> v.onOrAfter(Version.V_6_5_0)) != null) {
if (getNodeId(v -> v.onOrAfter(Version.V_6_5_0)) != null && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
}
createIndex(index, settings.build());

3
server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java
View File

@ -172,8 +172,7 @@ public class TransportResizeAction extends TransportMasterNodeAction<ResizeReque
throw new IllegalArgumentException("cannot provide index.number_of_routing_shards on resize");
}
}
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.exists(metaData.getSettings()) &&
IndexSettings.INDEX_SOFT_DELETES_SETTING.get(metaData.getSettings()) &&
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(metaData.getSettings()) &&
IndexSettings.INDEX_SOFT_DELETES_SETTING.exists(targetIndexSettings) &&
IndexSettings.INDEX_SOFT_DELETES_SETTING.get(targetIndexSettings) == false) {
throw new IllegalArgumentException("Can't disable [index.soft_deletes.enabled] setting on resize");

2
server/src/main/java/org/elasticsearch/index/IndexSettings.java
View File

@ -245,7 +245,7 @@ public final class IndexSettings {
* Specifies if the index should use soft-delete instead of hard-delete for update/delete operations.
*/
public static final Setting<Boolean> INDEX_SOFT_DELETES_SETTING =
Setting.boolSetting("index.soft_deletes.enabled", false, Property.IndexScope, Property.Final);
Setting.boolSetting("index.soft_deletes.enabled", true, Property.IndexScope, Property.Final);
/**
* Controls how many soft-deleted documents will be kept around before being merged away. Keeping more deleted

2
server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java
View File

@ -225,7 +225,7 @@ public class RemoveCorruptedShardDataCommandTests extends IndexShardTestCase {
final Throwable cause = exception.getCause() instanceof EngineException ? exception.getCause().getCause() : exception.getCause();
assertThat(cause, instanceOf(TranslogCorruptedException.class));
closeShards(corruptedShard);
closeShard(corruptedShard, false); // translog is corrupted already - do not check consistency
final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
final MockTerminal t = new MockTerminal();

7
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
View File

@ -10,6 +10,7 @@ import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.client.Client;
@ -23,6 +24,7 @@ import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.collect.CopyOnWriteHashMap;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.common.util.concurrent.CountDown;
@ -416,7 +418,10 @@ public class AutoFollowCoordinator implements ClusterStateListener {
// has a leader index uuid custom metadata entry that matches with uuid of leaderIndexMetaData variable
// If so then handle it differently: not follow it, but just add an entry to
// AutoFollowMetadata#followedLeaderIndexUUIDs
if (leaderIndexMetaData.getSettings().getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)) {
final Settings leaderIndexSettings = leaderIndexMetaData.getSettings();
// soft deletes are enabled by default on indices created on 7.0.0 or later
if (leaderIndexSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(),
IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(leaderIndexSettings).onOrAfter(Version.V_7_0_0))) {
leaderIndicesToFollow.add(leaderIndexMetaData.getIndex());
}
}

5
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
View File

@ -8,6 +8,7 @@ package org.elasticsearch.xpack.ccr.action;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.ActiveShardCount;
@ -122,7 +123,9 @@ public final class TransportPutFollowAction
listener.onFailure(new IllegalArgumentException("leader index [" + request.getLeaderIndex() + "] does not exist"));
return;
}
if (leaderIndexMetaData.getSettings().getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) == false) {
// soft deletes are enabled by default on indices created on 7.0.0 or later
if (leaderIndexMetaData.getSettings().getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(),
IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(leaderIndexMetaData.getSettings()).onOrAfter(Version.V_7_0_0)) == false) {
listener.onFailure(
new IllegalArgumentException("leader index [" + request.getLeaderIndex() + "] does not have soft deletes enabled"));
return;

10
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.ccr.action;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
@ -211,12 +212,13 @@ public class TransportResumeFollowAction extends TransportMasterNodeAction<Resum
"] as history uuid");
}
}
if (leaderIndex.getSettings().getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) == false) {
// soft deletes are enabled by default on indices created on 7.0.0 or later
if (leaderIndex.getSettings().getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(),
IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(leaderIndex.getSettings()).onOrAfter(Version.V_7_0_0)) == false) {
throw new IllegalArgumentException("leader index [" + leaderIndex.getIndex().getName() +
"] does not have soft deletes enabled");
}
if (followIndex.getSettings().getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) == false) {
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(followIndex.getSettings()) == false) {
throw new IllegalArgumentException("follower index [" + request.getFollowerIndex() + "] does not have soft deletes enabled");
}
if (leaderIndex.getNumberOfShards() != followIndex.getNumberOfShards()) {
@ -443,6 +445,8 @@ public class TransportResumeFollowAction extends TransportMasterNodeAction<Resum
Settings.Builder settings = Settings.builder().put(originalSettings);
// Remove settings that are always going to be different between leader and follow index:
settings.remove(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey());
// soft deletes setting is checked manually
settings.remove(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey());
settings.remove(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey());
settings.remove(IndexMetaData.SETTING_INDEX_UUID);
settings.remove(IndexMetaData.SETTING_INDEX_PROVIDED_NAME);

35
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
View File

@ -17,6 +17,7 @@ import org.elasticsearch.index.MapperTestUtils;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.xpack.ccr.Ccr;
import org.elasticsearch.xpack.ccr.CcrSettings;
import org.elasticsearch.xpack.ccr.IndexFollowingIT;
@ -73,25 +74,34 @@ public class TransportResumeFollowActionTests extends ESTestCase {
}
{
// should fail because leader index does not have soft deletes enabled
IndexMetaData leaderIMD = createIMD("index1", 5, Settings.EMPTY, null);
IndexMetaData leaderIMD = createIMD("index1", 5, Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "false").build(), null);
IndexMetaData followIMD = createIMD("index2", 5, Settings.EMPTY, customMetaData);
Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null));
assertThat(e.getMessage(), equalTo("leader index [index1] does not have soft deletes enabled"));
}
{
// should fail because leader index does not have soft deletes enabled (by default).
Version prevVersion = VersionUtils.randomVersionBetween(
random(), Version.V_6_5_0, VersionUtils.getPreviousVersion(Version.V_7_0_0));
IndexMetaData leaderIMD = IndexMetaData.builder("index1").settings(settings(prevVersion)).numberOfShards(1)
.numberOfReplicas(0).setRoutingNumShards(1).putMapping("_doc", "{\"properties\": {}}").build();
IndexMetaData followIMD = createIMD("index2", 5, Settings.EMPTY, customMetaData);
Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null));
assertThat(e.getMessage(), equalTo("leader index [index1] does not have soft deletes enabled"));
}
{
// should fail because the follower index does not have soft deletes enabled
IndexMetaData leaderIMD = createIMD("index1", 5, Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), null);
IndexMetaData followIMD = createIMD("index2", 5, Settings.EMPTY, customMetaData);
IndexMetaData leaderIMD = createIMD("index1", 5, Settings.EMPTY, null);
IndexMetaData followIMD = createIMD("index2", 5, Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "false").build(), customMetaData);
Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null));
assertThat(e.getMessage(), equalTo("follower index [index2] does not have soft deletes enabled"));
}
{
// should fail because the number of primary shards between leader and follow index are not equal
IndexMetaData leaderIMD = createIMD("index1", 5, Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), null);
IndexMetaData followIMD = createIMD("index2", 4,
Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), customMetaData);
IndexMetaData leaderIMD = createIMD("index1", 5, Settings.EMPTY, null);
IndexMetaData followIMD = createIMD("index2", 4, Settings.EMPTY, customMetaData);
Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null));
assertThat(e.getMessage(),
equalTo("leader index primary shards [5] does not match with the number of shards of the follow index [4]"));
@ -100,17 +110,14 @@ public class TransportResumeFollowActionTests extends ESTestCase {
// should fail, because leader index is closed
IndexMetaData leaderIMD = createIMD("index1", State.CLOSE, "{}", 5, Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), null);
IndexMetaData followIMD = createIMD("index2", State.OPEN, "{}", 5,
Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), customMetaData);
IndexMetaData followIMD = createIMD("index2", State.OPEN, "{}", 5, Settings.EMPTY, customMetaData);
Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null));
assertThat(e.getMessage(), equalTo("leader and follow index must be open"));
}
{
// should fail, because index.xpack.ccr.following_index setting has not been enabled in leader index
IndexMetaData leaderIMD = createIMD("index1", 1,
Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), null);
IndexMetaData followIMD = createIMD("index2", 1,
Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true").build(), customMetaData);
IndexMetaData leaderIMD = createIMD("index1", 1, Settings.EMPTY, null);
IndexMetaData followIMD = createIMD("index2", 1, Settings.EMPTY, customMetaData);
MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, "index2");
mapperService.updateMapping(null, followIMD);
Exception e = expectThrows(IllegalArgumentException.class,