honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deprecate indices without soft-deletes (#50502) (#50634) 

Soft-deletes will be enabled for all indices in 8.0. Hence, we should
deprecate new indices without soft-deletes in 7.x.

Backport of #50502
This commit is contained in:
Nhat Nguyen 2020-01-06 08:44:30 -05:00 committed by GitHub
parent ec0ec61881
commit b71490b06b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 113 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/reference/index-modules/history-retention.asciidoc
View File

@ -65,6 +65,9 @@ there>>. {ccr-cap} will not function if soft deletes are disabled.
configured at index creation and only on indices created on or after 6.5.0.
The default value is `true`.
deprecated::[7.6, Creating indices with soft-deletes disabled is
deprecated and will be removed in future Elasticsearch versions.]
`index.soft_deletes.retention_lease.period`::
The maximum length of time to retain a shard history retention lease before

31
qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
View File

@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
@ -306,7 +307,7 @@ public class RecoveryIT extends AbstractRollingTestCase {
}
}
public void testRecoveryWithSoftDeletes() throws Exception {
public void testRecovery() throws Exception {
final String index = "recover_with_soft_deletes";
if (CLUSTER_TYPE == ClusterType.OLD) {
Settings.Builder settings = Settings.builder()
@ -318,8 +319,8 @@ public class RecoveryIT extends AbstractRollingTestCase {
// before timing out
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
if (getNodeId(v -> v.onOrAfter(Version.V_6_5_0)) != null && randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
if (randomBoolean()) {
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
}
createIndex(index, settings.build());
int numDocs = randomInt(10);
@ -350,7 +351,7 @@ public class RecoveryIT extends AbstractRollingTestCase {
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(1, 2)) // triggers nontrivial promotion
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
createIndex(index, settings.build());
int numDocs = randomInt(10);
indexDocs(index, 0, numDocs);
@ -373,7 +374,7 @@ public class RecoveryIT extends AbstractRollingTestCase {
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, 1))
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
.put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean());
createIndex(index, settings.build());
int numDocs = randomInt(10);
indexDocs(index, 0, numDocs);
@ -784,6 +785,26 @@ public class RecoveryIT extends AbstractRollingTestCase {
}
}
public void testSoftDeletesDisabledWarning() throws Exception {
final String indexName = "test_soft_deletes_disabled_warning";
if (CLUSTER_TYPE == ClusterType.OLD) {
boolean softDeletesEnabled = true;
Settings.Builder settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1);
if (randomBoolean()) {
softDeletesEnabled = randomBoolean();
settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled);
}
Request request = new Request("PUT", "/" + indexName);
request.setJsonEntity("{\"settings\": " + Strings.toString(settings.build()) + "}");
if (softDeletesEnabled == false) {
expectSoftDeletesWarning(request, indexName);
}
client().performRequest(request);
}
ensureGreen(indexName);
indexDocs(indexName, randomInt(100), randomInt(100));
}
@SuppressWarnings("unchecked")
private Map<String, Object> getIndexSettingsAsMap(String index) throws IOException {
Map<String, Object> indexSettings = getIndexSettings(index);

17
rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml
View File

@ -154,3 +154,20 @@
- match: { error.type: "illegal_argument_exception" }
- match: { error.reason: "The mapping definition cannot be nested under a type [_doc] unless include_type_name is set to true." }
---
"Create index without soft deletes":
- skip:
version: " - 7.5.99"
reason: "indices without soft deletes are deprecated in 7.6"
features: "warnings"
- do:
warnings:
- Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
Please do not specify value for setting [index.soft_deletes.enabled] of index [test_index].
indices.create:
index: test_index
body:
settings:
soft_deletes.enabled: false

17
rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml
View File

@ -1,11 +1,19 @@
---
"Translog retention without soft_deletes":
- skip:
version: " - 7.5.99"
reason: "indices without soft deletes are deprecated in 7.6"
features: "warnings"
- do:
indices.create:
index: test
body:
settings:
soft_deletes.enabled: false
warnings:
- Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
Please do not specify value for setting [index.soft_deletes.enabled] of index [test].
- do:
cluster.health:
wait_for_no_initializing_shards: true
@ -132,8 +140,9 @@
---
"Translog stats on closed indices without soft-deletes":
- skip:
version: " - 7.2.99"
reason: "closed indices have translog stats starting version 7.3.0"
version: " - 7.5.99"
reason: "indices without soft deletes are deprecated in 7.6"
features: "warnings"
- do:
indices.create:
@ -141,6 +150,10 @@
body:
settings:
soft_deletes.enabled: false
warnings:
- Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions.
Please do not specify value for setting [index.soft_deletes.enabled] of index [test].
- do:
cluster.health:
wait_for_no_initializing_shards: true

8
server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -53,6 +53,7 @@ import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.ValidationException;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
@ -63,6 +64,7 @@ import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.MapperService.MergeReason;
@ -104,6 +106,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF
*/
public class MetaDataCreateIndexService {
private static final Logger logger = LogManager.getLogger(MetaDataCreateIndexService.class);
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger);
public static final int MAX_INDEX_NAME_BYTES = 255;
@ -455,6 +458,11 @@ public class MetaDataCreateIndexService {
* that will be used to create this index.
*/
MetaDataCreateIndexService.checkShardLimit(indexSettings, currentState);
if (indexSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) {
DEPRECATION_LOGGER.deprecatedAndMaybeLog("soft_deletes_disabled",
"Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " +
"Please do not specify value for setting [index.soft_deletes.enabled] of index [" + request.index() + "].");
}
return indexSettings;
}

15
server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
View File

@ -858,6 +858,21 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
assertThat(targetRoutingNumberOfShards, is(6));
}
public void testSoftDeletesDisabledDeprecation() {
request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build());
aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Collections.emptyList(), Collections.emptyMap(),
null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
assertWarnings("Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. "
+ "Please do not specify value for setting [index.soft_deletes.enabled] of index [test].");
request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
if (randomBoolean()) {
request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true).build());
}
aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Collections.emptyList(), Collections.emptyMap(),
null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
}
private IndexTemplateMetaData addMatchingTemplate(Consumer<IndexTemplateMetaData.Builder> configurator) {
IndexTemplateMetaData.Builder builder = templateMetaDataBuilder("template1", "te*");
configurator.accept(builder);

39
test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
View File

@ -52,6 +52,7 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.seqno.ReplicationTracker;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.snapshots.SnapshotState;
@ -969,23 +970,27 @@ public abstract class ESRestTestCase extends ESTestCase {
}
protected static void createIndex(String name, Settings settings) throws IOException {
Request request = new Request("PUT", "/" + name);
request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings) + "}");
client().performRequest(request);
createIndex(name, settings, null);
}
protected static void createIndex(String name, Settings settings, String mapping) throws IOException {
Request request = new Request("PUT", "/" + name);
request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings)
+ ", \"mappings\" : {" + mapping + "} }");
client().performRequest(request);
createIndex(name, settings, mapping, null);
}
protected static void createIndex(String name, Settings settings, String mapping, String aliases) throws IOException {
Request request = new Request("PUT", "/" + name);
request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings)
+ ", \"mappings\" : {" + mapping + "}"
+ ", \"aliases\": {" + aliases + "} }");
String entity = "{\"settings\": " + Strings.toString(settings);
if (mapping != null) {
entity += ",\"mappings\" : {" + mapping + "}";
}
if (aliases != null) {
entity += ",\"aliases\": {" + aliases + "}";
}
entity += "}";
if (settings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) {
expectSoftDeletesWarning(request, name);
}
request.setJsonEntity(entity);
client().performRequest(request);
}
@ -1004,6 +1009,20 @@ public abstract class ESRestTestCase extends ESTestCase {
client().performRequest(request);
}
protected static void expectSoftDeletesWarning(Request request, String indexName) {
final List<String> expectedWarnings = Collections.singletonList(
"Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. " +
"Please do not specify value for setting [index.soft_deletes.enabled] of index [" + indexName + "].");
final Builder requestOptions = RequestOptions.DEFAULT.toBuilder();
if (nodeVersions.stream().allMatch(version -> version.onOrAfter(Version.V_7_6_0))) {
requestOptions.setWarningsHandler(warnings -> warnings.equals(expectedWarnings) == false);
request.setOptions(requestOptions);
} else if (nodeVersions.stream().anyMatch(version -> version.onOrAfter(Version.V_7_6_0))) {
requestOptions.setWarningsHandler(warnings -> warnings.isEmpty() == false && warnings.equals(expectedWarnings) == false);
request.setOptions(requestOptions);
}
}
protected static Map<String, Object> getIndexSettings(String index) throws IOException {
Request request = new Request("GET", "/" + index + "/_settings");
request.addParameter("flat_settings", "true");