honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add migration tool checks for `_field_names` disabling (#46972) 

This change adds a check to the migration tool that warns about the deprecated
`enabled` setting for the `_field_names` field on 7.x indices and issues a
warning for templates containing this setting, which has been removed
with 8.0.

Relates to #42854, #46681
This commit is contained in:
Christoph Büscher 2019-09-25 10:15:10 +02:00 committed by GitHub
parent db63e78b68
commit 0c187e0a10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 180 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java
View File

@ -12,15 +12,19 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.ingest.IngestService;
import org.elasticsearch.ingest.PipelineConfiguration;
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@ -91,6 +95,53 @@ public class ClusterDeprecationChecks {
return null;
}
/**
* Check templates that use `_field_names` explicitly, which was deprecated in https://github.com/elastic/elasticsearch/pull/42854
* and will throw an error on new indices in 8.0
*/
@SuppressWarnings("unchecked")
static DeprecationIssue checkTemplatesWithFieldNamesDisabled(ClusterState state) {
Set<String> templatesContainingFieldNames = new HashSet<>();
state.getMetaData().getTemplates().forEach((templateCursor) -> {
String templateName = templateCursor.key;
templateCursor.value.getMappings().forEach((mappingCursor) -> {
String type = mappingCursor.key;
// there should be the type name at this level, but there was a bug where mappings could be stored without a type (#45120)
// to make sure, we try to detect this like we try to do in MappingMetaData#sourceAsMap()
Map<String, Object> mapping = XContentHelper.convertToMap(mappingCursor.value.compressedReference(), true).v2();
if (mapping.size() == 1 && mapping.containsKey(type)) {
// the type name is the root value, reduce it
mapping = (Map<String, Object>) mapping.get(type);
}
if (mapContainsFieldNamesDisabled(mapping)) {
templatesContainingFieldNames.add(templateName);
}
});
});
if (templatesContainingFieldNames.isEmpty() == false) {
return new DeprecationIssue(DeprecationIssue.Level.WARNING, "Index templates contain _field_names settings.",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html#fieldnames-enabling",
"Index templates " + templatesContainingFieldNames + " use the deprecated `enable` setting for the `"
+ FieldNamesFieldMapper.NAME + "` field. Using this setting in new index mappings will throw an error "
+ "in the next major version and needs to be removed from existing mappings and templates.");
}
return null;
}
/**
* check for "_field_names" entries in the map that contain another property "enabled" in the sub-map
*/
static boolean mapContainsFieldNamesDisabled(Map<?, ?> map) {
Object fieldNamesMapping = map.get(FieldNamesFieldMapper.NAME);
if (fieldNamesMapping != null) {
if (((Map<?, ?>) fieldNamesMapping).keySet().contains("enabled")) {
return true;
}
}
return false;
}
static DeprecationIssue checkPollIntervalTooLow(ClusterState state) {
String pollIntervalString = state.metaData().settings().get(LIFECYCLE_POLL_INTERVAL_SETTING.getKey());
if (Strings.isNullOrEmpty(pollIntervalString)) {

6
x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
View File

@ -35,7 +35,8 @@ public class DeprecationChecks {
Collections.unmodifiableList(Arrays.asList(
ClusterDeprecationChecks::checkUserAgentPipelines,
ClusterDeprecationChecks::checkTemplatesWithTooManyFields,
ClusterDeprecationChecks::checkPollIntervalTooLow
ClusterDeprecationChecks::checkPollIntervalTooLow,
ClusterDeprecationChecks::checkTemplatesWithFieldNamesDisabled
));
@ -51,7 +52,8 @@ public class DeprecationChecks {
IndexDeprecationChecks::tooManyFieldsCheck,
IndexDeprecationChecks::chainedMultiFieldsCheck,
IndexDeprecationChecks::deprecatedDateTimeFormat,
IndexDeprecationChecks::translogRetentionSettingCheck
IndexDeprecationChecks::translogRetentionSettingCheck,
IndexDeprecationChecks::fieldNamesDisabledCheck
));
static List<BiFunction<DatafeedConfig, NamedXContentRegistry, DeprecationIssue>> ML_SETTINGS_CHECKS =

16
x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java
View File

@ -7,6 +7,7 @@ package org.elasticsearch.xpack.deprecation;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
@ -185,6 +186,21 @@ public class IndexDeprecationChecks {
return false;
}
/**
* warn about existing explicit "_field_names" settings in existing mappings
*/
static DeprecationIssue fieldNamesDisabledCheck(IndexMetaData indexMetaData) {
MappingMetaData mapping = indexMetaData.mapping();
if ((mapping != null) && ClusterDeprecationChecks.mapContainsFieldNamesDisabled(mapping.getSourceAsMap())) {
return new DeprecationIssue(DeprecationIssue.Level.WARNING,
"Index mapping contains explicit `_field_names` enabling settings.",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" +
"#fieldnames-enabling",
"The index mapping contains a deprecated `enabled` setting for `_field_names` that should be removed moving foward.");
}
return null;
}
private static final Set<String> TYPES_THAT_DONT_COUNT;
static {
HashSet<String> typesThatDontCount = new HashSet<>();

79
x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java
View File

@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.ingest.IngestService;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
@ -158,6 +159,84 @@ public class ClusterDeprecationChecksTests extends ESTestCase {
assertEquals(singletonList(expected), issues);
}
public void testTemplatesWithFieldNamesDisabled() throws IOException {
XContentBuilder goodMappingBuilder = jsonBuilder();
goodMappingBuilder.startObject();
{
goodMappingBuilder.startObject("_doc");
{
goodMappingBuilder.startObject("properties");
{
addRandomFields(10, goodMappingBuilder);
}
goodMappingBuilder.endObject();
}
goodMappingBuilder.endObject();
}
goodMappingBuilder.endObject();
assertFieldNamesEnabledTemplate(goodMappingBuilder, false);
XContentBuilder badMappingBuilder = jsonBuilder();
badMappingBuilder.startObject();
{
// we currently always store a type level internally
badMappingBuilder.startObject("_doc");
{
badMappingBuilder.startObject(FieldNamesFieldMapper.NAME);
{
badMappingBuilder.field("enabled", randomBoolean());
}
badMappingBuilder.endObject();
}
badMappingBuilder.endObject();
}
badMappingBuilder.endObject();
assertFieldNamesEnabledTemplate(badMappingBuilder, true);
// however, there was a bug where mappings could be stored without a type (#45120)
// so we also should try to check these cases
XContentBuilder badMappingWithoutTypeBuilder = jsonBuilder();
badMappingWithoutTypeBuilder.startObject();
{
badMappingWithoutTypeBuilder.startObject(FieldNamesFieldMapper.NAME);
{
badMappingWithoutTypeBuilder.field("enabled", randomBoolean());
}
badMappingWithoutTypeBuilder.endObject();
}
badMappingWithoutTypeBuilder.endObject();
assertFieldNamesEnabledTemplate(badMappingWithoutTypeBuilder, true);
}
private void assertFieldNamesEnabledTemplate(XContentBuilder templateBuilder, boolean expectIssue) throws IOException {
String badTemplateName = randomAlphaOfLength(5);
final ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5)))
.metaData(MetaData.builder()
.put(IndexTemplateMetaData.builder(badTemplateName)
.patterns(Collections.singletonList(randomAlphaOfLength(5)))
.putMapping("_doc", Strings.toString(templateBuilder))
.build())
.build())
.build();
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(state));
if (expectIssue) {
assertEquals(1, issues.size());
DeprecationIssue issue = issues.get(0);
assertEquals(DeprecationIssue.Level.WARNING, issue.getLevel());
assertEquals("https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html#fieldnames-enabling"
, issue.getUrl());
assertEquals("Index templates contain _field_names settings.", issue.getMessage());
assertEquals("Index templates [" + badTemplateName + "] "
+ "use the deprecated `enable` setting for the `" + FieldNamesFieldMapper.NAME +
"` field. Using this setting in new index mappings will throw an error in the next major version and " +
"needs to be removed from existing mappings and templates.", issue.getDetails());
} else {
assertTrue(issues.isEmpty());
}
}
public void testPollIntervalTooLow() {
{
final String tooLowInterval = randomTimeValue(1, 999, "ms", "micros", "nanos");

32
x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java
View File

@ -11,10 +11,11 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.joda.JodaDeprecationPatterns;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
@ -27,8 +28,8 @@ import java.util.concurrent.atomic.AtomicInteger;
import static java.util.Collections.singletonList;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.xpack.deprecation.DeprecationChecks.INDEX_SETTINGS_CHECKS;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.collection.IsIterableContainingInOrder.contains;
public class IndexDeprecationChecksTests extends ESTestCase {
@ -161,6 +162,7 @@ public class IndexDeprecationChecksTests extends ESTestCase {
"The names of fields that contain chained multi-fields: [[type: _doc, field: invalid-field]]");
assertEquals(singletonList(expected), issues);
}
public void testDefinedPatternsDoNotWarn() throws IOException {
String simpleMapping = "{\n" +
"\"properties\" : {\n" +
@ -412,4 +414,30 @@ public class IndexDeprecationChecksTests extends ESTestCase {
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetaData));
assertThat(issues, empty());
}
public void testFieldNamesEnabling() throws IOException {
XContentBuilder xContent = XContentFactory.jsonBuilder().startObject()
.startObject(FieldNamesFieldMapper.NAME)
.field("enabled", randomBoolean())
.endObject()
.endObject();
String mapping = BytesReference.bytes(xContent).utf8ToString();
IndexMetaData simpleIndex = IndexMetaData.builder(randomAlphaOfLengthBetween(5, 10))
.settings(settings(
VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT)))
.numberOfShards(1)
.numberOfReplicas(0)
.putMapping("_doc", mapping).build();
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(simpleIndex));
assertEquals(1, issues.size());
DeprecationIssue issue = issues.get(0);
assertEquals(DeprecationIssue.Level.WARNING, issue.getLevel());
assertEquals("https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html#fieldnames-enabling"
, issue.getUrl());
assertEquals("Index mapping contains explicit `_field_names` enabling settings.", issue.getMessage());
assertEquals("The index mapping contains a deprecated `enabled` setting for `_field_names` that should be removed moving foward.",
issue.getDetails());
}
}