honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x] [ML] Add upgrade mappings assertions to full cluster restart tests (#62293) (#62305) 

Refactors the index mapping checks in the rolling upgrade tests
and use that shared code in the full cluster restart tests.
This commit is contained in:
David Kyle 2020-09-22 13:09:51 +01:00 committed by GitHub
parent df2b5dd4d1
commit 31fbc6800f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 319 additions and 308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
x-pack/qa/full-cluster-restart/build.gradle
View File

@ -8,7 +8,8 @@ apply from : "$rootDir/gradle/bwc-test.gradle"
dependencies {
// TODO: Remove core dependency and change tests to not use builders that are part of xpack-core.
// Currently needed for ml tests are using the building for datafeed and job config)
// Currently needed for MlConfigIndexMappingsFullClusterRestartIT and SLM classes used in
// FullClusterRestartIT
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
testImplementation project(path: ':qa:full-cluster-restart', configuration: 'testArtifacts')

57
x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
View File

@ -10,22 +10,11 @@ import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.WarningFailureException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
import org.elasticsearch.xpack.core.ml.MlConfigIndex;
import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
import org.elasticsearch.xpack.core.ml.job.config.Detector;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.test.rest.IndexMappingTemplateAsserter;
import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
import org.junit.Before;
@ -33,7 +22,6 @@ import org.junit.Before;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@ -63,7 +51,6 @@ public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClust
}
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/54415")
public void testMlConfigIndexMappingsAfterMigration() throws Exception {
assumeTrue("This test only makes sense in version 6.6.0 and above", getOldClusterVersion().onOrAfter(Version.V_6_6_0));
if (isRunningAgainstOldCluster()) {
@ -82,16 +69,7 @@ public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClust
createAnomalyDetectorJob(NEW_CLUSTER_JOB_ID);
// assert that the mappings are updated
Map<String, Object> dataFrameAnalysisMappings = getDataFrameAnalysisMappings();
// Remove renamed fields
if (getOldClusterVersion().before(Version.V_7_7_0)) {
dataFrameAnalysisMappings = XContentMapValues.filter(dataFrameAnalysisMappings, null, new String[] {
"*.properties.maximum_number_trees" // This was renamed to max_trees
});
}
assertThat(dataFrameAnalysisMappings, equalTo(loadDataFrameAnalysisMappings()));
IndexMappingTemplateAsserter.assertMlMappingsMatchTemplates(client());
}
}
@ -102,15 +80,21 @@ public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClust
}
private void createAnomalyDetectorJob(String jobId) throws IOException {
Detector.Builder detector = new Detector.Builder("metric", "responsetime");
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()))
.setBucketSpan(TimeValue.timeValueMinutes(10));
Job.Builder job = new Job.Builder(jobId)
.setAnalysisConfig(analysisConfig)
.setDataDescription(new DataDescription.Builder());
String jobConfig =
"{\n" +
" \"job_id\": \"" + jobId + "\",\n" +
" \"analysis_config\": {\n" +
" \"bucket_span\": \"10m\",\n" +
" \"detectors\": [{\n" +
" \"function\": \"metric\",\n" +
" \"field_name\": \"responsetime\"\n" +
" }]\n" +
" },\n" +
" \"data_description\": {}\n" +
"}";
Request putJobRequest = new Request("PUT", "/_ml/anomaly_detectors/" + jobId);
putJobRequest.setJsonEntity(Strings.toString(job));
putJobRequest.setJsonEntity(jobConfig);
Response putJobResponse = client().performRequest(putJobRequest);
assertThat(putJobResponse.getStatusLine().getStatusCode(), equalTo(200));
}
@ -141,15 +125,4 @@ public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClust
mappings = (Map<String, Object>) XContentMapValues.extractValue(mappings, "analysis", "properties");
return mappings;
}
@SuppressWarnings("unchecked")
private Map<String, Object> loadDataFrameAnalysisMappings() throws IOException {
String mapping = MlConfigIndex.mapping();
try (XContentParser parser = JsonXContent.jsonXContent.createParser(
NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, new BytesArray(mapping).streamInput())) {
Map<String, Object> mappings = parser.map();
mappings = (Map<String, Object>) XContentMapValues.extractValue(mappings, "_doc", "properties", "analysis", "properties");
return mappings;
}
}
}

207
x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java
View File

@ -8,28 +8,17 @@ package org.elasticsearch.upgrades;
import org.elasticsearch.Version;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.xpack.test.SecuritySettingsSourceField;
import org.junit.Before;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue;
@ -110,7 +99,6 @@ public abstract class AbstractUpgradeTestCase extends ESRestTestCase {
if (expectedTemplates.isEmpty()) {
return;
}
assertBusy(() -> {
final Request catRequest = new Request("GET", "_cat/templates?h=n&s=n");
final Response catResponse = adminClient().performRequest(catRequest);
@ -127,199 +115,4 @@ public abstract class AbstractUpgradeTestCase extends ESRestTestCase {
}
});
}
/**
* Compares the mappings from the template and the index and asserts they
* are the same.
*
* The test is intended to catch cases where an index mapping has been
* updated dynamically or a write occurred before the template was put.
* The assertion error message details the differences in the mappings.
*
* The Mappings, which are maps of maps, are flattened with the keys built
* from the keys of the sub-maps appended to the parent key.
* This makes diffing the 2 maps easier and diffs more comprehensible.
*
* The _meta field is not compared as it contains version numbers that
* change even when the mappings don't.
*
* Mistakes happen and some indices may be stuck with the incorrect mappings
* that cannot be fixed without re-index. In this case use the {@code exceptions}
* parameter to filter out fields in the index mapping that are not in the
* template. Each exception should be a '.' separated path to the value
* e.g. {@code properties.analysis.analysis_field.type}.
*
* @param templateName The template
* @param indexName The index
* @param notAnErrorIfIndexDoesNotExist The index may or may not have been created from
* the template. If {@code true} then the missing
* index does not cause an error
* @param exceptions List of keys to ignore in the index mappings.
* The key is a '.' separated path.
* @throws IOException Yes
*/
@SuppressWarnings("unchecked")
protected void assertLegacyTemplateMatchesIndexMappings(String templateName,
String indexName,
boolean notAnErrorIfIndexDoesNotExist,
Set<String> exceptions) throws IOException {
Request getTemplate = new Request("GET", "_template/" + templateName);
Response templateResponse = client().performRequest(getTemplate);
assertEquals("missing template [" + templateName + "]", 200, templateResponse.getStatusLine().getStatusCode());
Map<String, Object> templateMappings = (Map<String, Object>) XContentMapValues.extractValue(entityAsMap(templateResponse),
templateName, "mappings");
assertNotNull(templateMappings);
Request getIndexMapping = new Request("GET", indexName + "/_mapping");
Response indexMappingResponse;
try {
indexMappingResponse = client().performRequest(getIndexMapping);
} catch (ResponseException e) {
if (e.getResponse().getStatusLine().getStatusCode() == 404 && notAnErrorIfIndexDoesNotExist) {
return;
} else {
throw e;
}
}
assertEquals("error getting mappings for index [" + indexName + "]",
200, indexMappingResponse.getStatusLine().getStatusCode());
Map<String, Object> indexMappings = (Map<String, Object>) XContentMapValues.extractValue(entityAsMap(indexMappingResponse),
indexName, "mappings");
assertNotNull(indexMappings);
// ignore the _meta field
indexMappings.remove("_meta");
templateMappings.remove("_meta");
// We cannot do a simple comparison of mappings e.g
// Objects.equals(indexMappings, templateMappings) because some
// templates use strings for the boolean values - "true" and "false"
// which are automatically converted to Booleans causing the equality
// to fail.
boolean mappingsAreTheSame = true;
// flatten the map of maps
Map<String, Object> flatTemplateMap = flattenMap(templateMappings);
Map<String, Object> flatIndexMap = flattenMap(indexMappings);
SortedSet<String> keysInTemplateMissingFromIndex = new TreeSet<>(flatTemplateMap.keySet());
keysInTemplateMissingFromIndex.removeAll(flatIndexMap.keySet());
SortedSet<String> keysInIndexMissingFromTemplate = new TreeSet<>(flatIndexMap.keySet());
keysInIndexMissingFromTemplate.removeAll(flatTemplateMap.keySet());
// In the case of object fields the 'type: object' mapping is set by default.
// If this does not explicitly appear in the template it is not an error
// as ES has added the default to the index mappings
keysInIndexMissingFromTemplate.removeIf(key -> key.endsWith("type") && "object".equals(flatIndexMap.get(key)));
// Remove the exceptions
keysInIndexMissingFromTemplate.removeAll(exceptions);
StringBuilder errorMesssage = new StringBuilder("Error the template mappings [")
.append(templateName)
.append("] and index mappings [")
.append(indexName)
.append("] are not the same")
.append(System.lineSeparator());
if (keysInTemplateMissingFromIndex.isEmpty() == false) {
mappingsAreTheSame = false;
errorMesssage.append("Keys in the template missing from the index mapping: ")
.append(keysInTemplateMissingFromIndex)
.append(System.lineSeparator());
}
if (keysInIndexMissingFromTemplate.isEmpty() == false) {
mappingsAreTheSame = false;
errorMesssage.append("Keys in the index missing from the template mapping: ")
.append(keysInIndexMissingFromTemplate)
.append(System.lineSeparator());
}
// find values that are different for the same key
Set<String> commonKeys = new TreeSet<>(flatIndexMap.keySet());
commonKeys.retainAll(flatTemplateMap.keySet());
for (String key : commonKeys) {
Object template = flatTemplateMap.get(key);
Object index = flatIndexMap.get(key);
if (Objects.equals(template, index) == false) {
// Both maybe be booleans but different representations
if (areBooleanObjectsAndEqual(index, template)) {
continue;
}
mappingsAreTheSame = false;
errorMesssage.append("Values for key [").append(key).append("] are different").append(System.lineSeparator());
errorMesssage.append(" template value [").append(template).append("] ").append(template.getClass().getSimpleName())
.append(System.lineSeparator());
errorMesssage.append(" index value [").append(index).append("] ").append(index.getClass().getSimpleName())
.append(System.lineSeparator());
}
}
if (mappingsAreTheSame == false) {
fail(errorMesssage.toString());
}
}
protected void assertLegacyTemplateMatchesIndexMappings(String templateName,
String indexName) throws IOException {
assertLegacyTemplateMatchesIndexMappings(templateName, indexName, false, Collections.emptySet());
}
private boolean areBooleanObjectsAndEqual(Object a, Object b) {
Boolean left;
Boolean right;
if (a instanceof Boolean) {
left = (Boolean)a;
} else if (a instanceof String && isBooleanValueString((String)a)) {
left = Boolean.parseBoolean((String)a);
} else {
return false;
}
if (b instanceof Boolean) {
right = (Boolean)b;
} else if (b instanceof String && isBooleanValueString((String)b)) {
right = Boolean.parseBoolean((String)b);
} else {
return false;
}
return left.equals(right);
}
/* Boolean.parseBoolean is not strict anything that isn't
* "true" is returned as false. Here we want to know if
* s is a boolean.
*/
private boolean isBooleanValueString(String s) {
return s.equalsIgnoreCase("true") || s.equalsIgnoreCase("false");
}
private Map<String, Object> flattenMap(Map<String, Object> map) {
return new TreeMap<>(flatten("", map).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
}
private Stream<Map.Entry<String, Object>> flatten(String path, Map<String, Object> map) {
return map.entrySet()
.stream()
.flatMap((e) -> extractValue(path, e));
}
@SuppressWarnings("unchecked")
private Stream<Map.Entry<String, Object>> extractValue(String path, Map.Entry<String, Object> entry) {
String nextPath = path.isEmpty() ? entry.getKey() : path + "." + entry.getKey();
if (entry.getValue() instanceof Map<?, ?>) {
return flatten(nextPath, (Map<String, Object>) entry.getValue());
} else {
return Stream.of(new AbstractMap.SimpleEntry<>(nextPath, entry.getValue()));
}
}
}

59
x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java
View File

@ -14,15 +14,14 @@ import org.elasticsearch.client.ml.job.config.Detector;
import org.elasticsearch.client.ml.job.config.Job;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.xpack.test.rest.IndexMappingTemplateAsserter;
import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -56,7 +55,7 @@ public class MlMappingsUpgradeIT extends AbstractUpgradeTestCase {
assertUpgradedAnnotationsMappings();
closeAndReopenTestJob();
assertUpgradedConfigMappings();
assertMappingsMatchTemplates();
IndexMappingTemplateAsserter.assertMlMappingsMatchTemplates(client());
break;
default:
throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]");
@ -179,58 +178,4 @@ public class MlMappingsUpgradeIT extends AbstractUpgradeTestCase {
extractValue("mappings.properties.model_plot_config.properties.annotations_enabled.type", indexLevel));
});
}
/**
* Assert that the mappings of the ml indices are the same as in the
* templates. If different this is either a consequence of an unintended
* write (dynamic update) or the mappings have not been updated after
* upgrade.
*
* A failure here will be very difficult to reproduce as it may be a side
* effect of one of the other tests running in the cluster.
*
* @throws IOException On error
*/
private void assertMappingsMatchTemplates() throws IOException {
// Keys that have been dynamically mapped in the .ml-config index
// but are not in the template. These can only be fixed with
// re-index and should be addressed at the next major upgrade.
// For now this serves as documentation of the missing fields
Set<String> configIndexExceptions = new HashSet<>();
configIndexExceptions.add("properties.allow_lazy_start.type");
configIndexExceptions.add("properties.analysis.properties.classification.properties.randomize_seed.type");
configIndexExceptions.add("properties.analysis.properties.outlier_detection.properties.compute_feature_influence.type");
configIndexExceptions.add("properties.analysis.properties.outlier_detection.properties.outlier_fraction.type");
configIndexExceptions.add("properties.analysis.properties.outlier_detection.properties.standardization_enabled.type");
configIndexExceptions.add("properties.analysis.properties.regression.properties.randomize_seed.type");
configIndexExceptions.add("properties.deleting.type");
configIndexExceptions.add("properties.model_memory_limit.type");
// fields from previous versions that have been removed
// renamed to max_trees in 7.7
configIndexExceptions.add("properties.analysis.properties.classification.properties.maximum_number_trees.type");
configIndexExceptions.add("properties.analysis.properties.regression.properties.maximum_number_trees.type");
configIndexExceptions.add("properties.established_model_memory.type");
configIndexExceptions.add("properties.last_data_time.type");
configIndexExceptions.add("properties.types.type");
// Excluding those from stats index as some have been renamed and other removed.
Set<String> statsIndexException = new HashSet<>();
statsIndexException.add("properties.hyperparameters.properties.regularization_depth_penalty_multiplier.type");
statsIndexException.add("properties.hyperparameters.properties.regularization_leaf_weight_penalty_multiplier.type");
statsIndexException.add("properties.hyperparameters.properties.regularization_soft_tree_depth_limit.type");
statsIndexException.add("properties.hyperparameters.properties.regularization_soft_tree_depth_tolerance.type");
statsIndexException.add("properties.hyperparameters.properties.regularization_tree_size_penalty_multiplier.type");
assertLegacyTemplateMatchesIndexMappings(".ml-config", ".ml-config", false, configIndexExceptions);
// the true parameter means the index may not have been created
assertLegacyTemplateMatchesIndexMappings(".ml-meta", ".ml-meta", true, Collections.emptySet());
assertLegacyTemplateMatchesIndexMappings(".ml-stats", ".ml-stats-000001", true, statsIndexException);
assertLegacyTemplateMatchesIndexMappings(".ml-state", ".ml-state-000001");
// AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/61908")
// assertLegacyTemplateMatchesIndexMappings(".ml-notifications-000001", ".ml-notifications-000001");
assertLegacyTemplateMatchesIndexMappings(".ml-inference-000003", ".ml-inference-000003", true, Collections.emptySet());
// .ml-annotations-6 does not use a template
// .ml-anomalies-shared uses a template but will have dynamically updated mappings as new jobs are opened
}
}

4
x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java
View File

@ -36,6 +36,7 @@ import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.xpack.test.rest.IndexMappingTemplateAsserter;
import org.junit.Before;
import java.io.IOException;
@ -157,7 +158,8 @@ public class TransformSurvivesUpgradeIT extends AbstractUpgradeTestCase {
case UPGRADED:
client().performRequest(waitForYellow);
verifyContinuousTransformHandlesData(3);
assertLegacyTemplateMatchesIndexMappings(".transform-internal-005", ".transform-internal-005");
IndexMappingTemplateAsserter.assertLegacyTemplateMatchesIndexMappings(client(),
".transform-internal-005", ".transform-internal-005");
cleanUpTransforms();
break;
default:

297
x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/IndexMappingTemplateAsserter.java Normal file
View File

@ -0,0 +1,297 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.test.rest;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.test.rest.ESRestTestCase;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
/**
* Utilities for checking that the current index mappings match
* the mappings defined in the template.
*
* The tests are intended to catch cases where an index mapping has been
* updated dynamically or a write occurred before the template was put
* causing the index to have the wrong mappings.
*
* These assertions are usually part of upgrade testing.
*/
public class IndexMappingTemplateAsserter {
/**
* Assert that the mappings of the ml indices are the same as in the
* templates. If different this is either a consequence of an unintended
* write (dynamic update) or the mappings have not been updated after
* upgrade.
*
* A failure here will be very difficult to reproduce as it may be a side
* effect of a different test running in the cluster.
*
* @param client The rest client
* @throws IOException On error
*/
public static void assertMlMappingsMatchTemplates(RestClient client) throws IOException {
// Keys that have been dynamically mapped in the .ml-config index
// but are not in the template. These can only be fixed with
// re-index and should be addressed at the next major upgrade.
// For now this serves as documentation of the missing fields
Set<String> configIndexExceptions = new HashSet<>();
configIndexExceptions.add("properties.allow_lazy_start.type");
configIndexExceptions.add("properties.analysis.properties.classification.properties.randomize_seed.type");
configIndexExceptions.add("properties.analysis.properties.outlier_detection.properties.compute_feature_influence.type");
configIndexExceptions.add("properties.analysis.properties.outlier_detection.properties.outlier_fraction.type");
configIndexExceptions.add("properties.analysis.properties.outlier_detection.properties.standardization_enabled.type");
configIndexExceptions.add("properties.analysis.properties.regression.properties.randomize_seed.type");
configIndexExceptions.add("properties.deleting.type");
configIndexExceptions.add("properties.model_memory_limit.type");
// fields from previous versions that have been removed
// renamed to max_trees in 7.7
configIndexExceptions.add("properties.analysis.properties.classification.properties.maximum_number_trees.type");
configIndexExceptions.add("properties.analysis.properties.regression.properties.maximum_number_trees.type");
configIndexExceptions.add("properties.established_model_memory.type");
configIndexExceptions.add("properties.last_data_time.type");
configIndexExceptions.add("properties.types.type");
// Excluding those from stats index as some have been renamed and other removed.
Set<String> statsIndexException = new HashSet<>();
statsIndexException.add("properties.hyperparameters.properties.regularization_depth_penalty_multiplier.type");
statsIndexException.add("properties.hyperparameters.properties.regularization_leaf_weight_penalty_multiplier.type");
statsIndexException.add("properties.hyperparameters.properties.regularization_soft_tree_depth_limit.type");
statsIndexException.add("properties.hyperparameters.properties.regularization_soft_tree_depth_tolerance.type");
statsIndexException.add("properties.hyperparameters.properties.regularization_tree_size_penalty_multiplier.type");
assertLegacyTemplateMatchesIndexMappings(client, ".ml-config", ".ml-config", false, configIndexExceptions);
// the true parameter means the index may not have been created
assertLegacyTemplateMatchesIndexMappings(client, ".ml-meta", ".ml-meta", true, Collections.emptySet());
assertLegacyTemplateMatchesIndexMappings(client, ".ml-stats", ".ml-stats-000001", true, statsIndexException);
assertLegacyTemplateMatchesIndexMappings(client, ".ml-state", ".ml-state-000001", true, Collections.emptySet());
// Depending on the order Full Cluster restart tests are run there may not be an notifications index yet
assertLegacyTemplateMatchesIndexMappings(client,
".ml-notifications-000001", ".ml-notifications-000001", true, Collections.emptySet());
assertLegacyTemplateMatchesIndexMappings(client,
".ml-inference-000003", ".ml-inference-000003", true, Collections.emptySet());
// .ml-annotations-6 does not use a template
// .ml-anomalies-shared uses a template but will have dynamically updated mappings as new jobs are opened
}
/**
* Compares the mappings from the template and the index and asserts they
* are the same. The assertion error message details the differences in
* the mappings.
*
* The Mappings, which are maps of maps, are flattened with the keys built
* from the keys of the sub-maps appended to the parent key.
* This makes diffing the 2 maps easier and diffs more comprehensible.
*
* The _meta field is not compared as it contains version numbers that
* change even when the mappings don't.
*
* Mistakes happen and some indices may be stuck with the incorrect mappings
* that cannot be fixed without re-index. In this case use the {@code exceptions}
* parameter to filter out fields in the index mapping that are not in the
* template. Each exception should be a '.' separated path to the value
* e.g. {@code properties.analysis.analysis_field.type}.
*
* @param client The rest client to use
* @param templateName The template
* @param indexName The index
* @param notAnErrorIfIndexDoesNotExist The index may or may not have been created from
* the template. If {@code true} then the missing
* index does not cause an error
* @param exceptions List of keys to ignore in the index mappings.
* Each key is a '.' separated path.
* @throws IOException Yes
*/
@SuppressWarnings("unchecked")
public static void assertLegacyTemplateMatchesIndexMappings(RestClient client,
String templateName,
String indexName,
boolean notAnErrorIfIndexDoesNotExist,
Set<String> exceptions) throws IOException {
Request getTemplate = new Request("GET", "_template/" + templateName);
Response templateResponse = client.performRequest(getTemplate);
assertEquals("missing template [" + templateName + "]", 200, templateResponse.getStatusLine().getStatusCode());
Map<String, Object> templateMappings = (Map<String, Object>) XContentMapValues.extractValue(
ESRestTestCase.entityAsMap(templateResponse),
templateName, "mappings");
assertNotNull(templateMappings);
Request getIndexMapping = new Request("GET", indexName + "/_mapping");
Response indexMappingResponse;
try {
indexMappingResponse = client.performRequest(getIndexMapping);
} catch (ResponseException e) {
if (e.getResponse().getStatusLine().getStatusCode() == 404 && notAnErrorIfIndexDoesNotExist) {
return;
} else {
throw e;
}
}
assertEquals("error getting mappings for index [" + indexName + "]",
200, indexMappingResponse.getStatusLine().getStatusCode());
Map<String, Object> indexMappings = (Map<String, Object>) XContentMapValues.extractValue(
ESRestTestCase.entityAsMap(indexMappingResponse),
indexName, "mappings");
assertNotNull(indexMappings);
// ignore the _meta field
indexMappings.remove("_meta");
templateMappings.remove("_meta");
// We cannot do a simple comparison of mappings e.g
// Objects.equals(indexMappings, templateMappings) because some
// templates use strings for the boolean values - "true" and "false"
// which are automatically converted to Booleans causing the equality
// to fail.
boolean mappingsAreTheSame = true;
// flatten the map of maps
Map<String, Object> flatTemplateMap = flattenMap(templateMappings);
Map<String, Object> flatIndexMap = flattenMap(indexMappings);
SortedSet<String> keysInTemplateMissingFromIndex = new TreeSet<>(flatTemplateMap.keySet());
keysInTemplateMissingFromIndex.removeAll(flatIndexMap.keySet());
SortedSet<String> keysInIndexMissingFromTemplate = new TreeSet<>(flatIndexMap.keySet());
keysInIndexMissingFromTemplate.removeAll(flatTemplateMap.keySet());
// In the case of object fields the 'type: object' mapping is set by default.
// If this does not explicitly appear in the template it is not an error
// as ES has added the default to the index mappings
keysInIndexMissingFromTemplate.removeIf(key -> key.endsWith("type") && "object".equals(flatIndexMap.get(key)));
// Remove the exceptions
keysInIndexMissingFromTemplate.removeAll(exceptions);
StringBuilder errorMesssage = new StringBuilder("Error the template mappings [")
.append(templateName)
.append("] and index mappings [")
.append(indexName)
.append("] are not the same")
.append(System.lineSeparator());
if (keysInTemplateMissingFromIndex.isEmpty() == false) {
mappingsAreTheSame = false;
errorMesssage.append("Keys in the template missing from the index mapping: ")
.append(keysInTemplateMissingFromIndex)
.append(System.lineSeparator());
}
if (keysInIndexMissingFromTemplate.isEmpty() == false) {
mappingsAreTheSame = false;
errorMesssage.append("Keys in the index missing from the template mapping: ")
.append(keysInIndexMissingFromTemplate)
.append(System.lineSeparator());
}
// find values that are different for the same key
Set<String> commonKeys = new TreeSet<>(flatIndexMap.keySet());
commonKeys.retainAll(flatTemplateMap.keySet());
for (String key : commonKeys) {
Object template = flatTemplateMap.get(key);
Object index = flatIndexMap.get(key);
if (Objects.equals(template, index) == false) {
// Both maybe be booleans but different representations
if (areBooleanObjectsAndEqual(index, template)) {
continue;
}
mappingsAreTheSame = false;
errorMesssage.append("Values for key [").append(key).append("] are different").append(System.lineSeparator());
errorMesssage.append(" template value [").append(template).append("] ").append(template.getClass().getSimpleName())
.append(System.lineSeparator());
errorMesssage.append(" index value [").append(index).append("] ").append(index.getClass().getSimpleName())
.append(System.lineSeparator());
}
}
if (mappingsAreTheSame == false) {
fail(errorMesssage.toString());
}
}
public static void assertLegacyTemplateMatchesIndexMappings(RestClient client,
String templateName,
String indexName) throws IOException {
assertLegacyTemplateMatchesIndexMappings(client, templateName, indexName, false, Collections.emptySet());
}
private static boolean areBooleanObjectsAndEqual(Object a, Object b) {
Boolean left;
Boolean right;
if (a instanceof Boolean) {
left = (Boolean)a;
} else if (a instanceof String && isBooleanValueString((String)a)) {
left = Boolean.parseBoolean((String)a);
} else {
return false;
}
if (b instanceof Boolean) {
right = (Boolean)b;
} else if (b instanceof String && isBooleanValueString((String)b)) {
right = Boolean.parseBoolean((String)b);
} else {
return false;
}
return left.equals(right);
}
/* Boolean.parseBoolean is not strict. Anything that isn't
* "true" is returned as false. Here we want to know if
* the string is a boolean value.
*/
private static boolean isBooleanValueString(String s) {
return s.equalsIgnoreCase("true") || s.equalsIgnoreCase("false");
}
private static Map<String, Object> flattenMap(Map<String, Object> map) {
return new TreeMap<>(flatten("", map).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
}
private static Stream<Map.Entry<String, Object>> flatten(String path, Map<String, Object> map) {
return map.entrySet()
.stream()
.flatMap((e) -> extractValue(path, e));
}
@SuppressWarnings("unchecked")
private static Stream<Map.Entry<String, Object>> extractValue(String path, Map.Entry<String, Object> entry) {
String nextPath = path.isEmpty() ? entry.getKey() : path + "." + entry.getKey();
if (entry.getValue() instanceof Map<?, ?>) {
return flatten(nextPath, (Map<String, Object>) entry.getValue());
} else {
return Stream.of(new AbstractMap.SimpleEntry<>(nextPath, entry.getValue()));
}
}
}