[7.x] Disallow merging existing mapping field definitions in templates (#57701) (#57822)

Backports the following commits to 7.x:

    Disallow merging existing mapping field definitions in templates (#57701)
This commit is contained in:
Lee Hinman 2020-06-08 12:56:09 -06:00 committed by GitHub
parent 16fcb64c99
commit 6e8cf0973f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 340 additions and 47 deletions

View File

@ -66,9 +66,6 @@ PUT _index_template/template_1
"number_of_shards": 1 "number_of_shards": 1
}, },
"mappings": { "mappings": {
"_source": {
"enabled": false
},
"properties": { "properties": {
"host_name": { "host_name": {
"type": "keyword" "type": "keyword"

View File

@ -14,7 +14,7 @@
number_of_replicas: 1 number_of_replicas: 1
mappings: mappings:
properties: properties:
field2: field1:
type: text type: text
aliases: aliases:
aliasname: aliasname:
@ -47,9 +47,8 @@
index.number_of_shards: 2 index.number_of_shards: 2
mappings: mappings:
properties: properties:
field: field3:
type: keyword type: integer
ignore_above: 255
aliases: aliases:
my_alias: {} my_alias: {}
aliasname: aliasname:
@ -78,8 +77,9 @@
- match: {bar-baz.settings.index.number_of_shards: "2"} - match: {bar-baz.settings.index.number_of_shards: "2"}
- match: {bar-baz.settings.index.number_of_replicas: "0"} - match: {bar-baz.settings.index.number_of_replicas: "0"}
- match: {bar-baz.settings.index.priority: "17"} - match: {bar-baz.settings.index.priority: "17"}
- match: {bar-baz.mappings.properties.field: {type: keyword, ignore_above: 255}} - match: {bar-baz.mappings.properties.field1: {type: text}}
- match: {bar-baz.mappings.properties.field2: {type: keyword}} - match: {bar-baz.mappings.properties.field2: {type: keyword}}
- match: {bar-baz.mappings.properties.field3: {type: integer}}
- match: {bar-baz.mappings.properties.foo: {type: keyword}} - match: {bar-baz.mappings.properties.foo: {type: keyword}}
- match: {bar-baz.aliases.aliasname: {filter: {match_all: {}}}} - match: {bar-baz.aliases.aliasname: {filter: {match_all: {}}}}
- match: {bar-baz.aliases.my_alias: {}} - match: {bar-baz.aliases.my_alias: {}}

View File

@ -595,7 +595,7 @@ public class MetadataCreateIndexService {
nonProperties = innerTemplateNonProperties; nonProperties = innerTemplateNonProperties;
if (maybeProperties != null) { if (maybeProperties != null) {
properties = mergeIgnoringDots(properties, maybeProperties); properties = mergeFailingOnReplacement(properties, maybeProperties);
} }
} }
} }
@ -609,7 +609,7 @@ public class MetadataCreateIndexService {
nonProperties = innerRequestNonProperties; nonProperties = innerRequestNonProperties;
if (maybeRequestProperties != null) { if (maybeRequestProperties != null) {
properties = mergeIgnoringDots(properties, maybeRequestProperties); properties = mergeFailingOnReplacement(properties, maybeRequestProperties);
} }
} }
@ -619,18 +619,18 @@ public class MetadataCreateIndexService {
} }
/** /**
* Add the objects in the second map to the first, where the keys in the {@code second} map have * Add the objects in the second map to the first, A duplicated field is treated as illegal and
* higher predecence and overwrite the keys in the {@code first} map. In the event of a key with * an exception is thrown.
* a dot in it (ie, "foo.bar"), the keys are treated as only the prefix counting towards
* equality. If the {@code second} map has a key such as "foo", all keys starting from "foo." in
* the {@code first} map are discarded.
*/ */
static Map<String, Object> mergeIgnoringDots(Map<String, Object> first, Map<String, Object> second) { static Map<String, Object> mergeFailingOnReplacement(Map<String, Object> first, Map<String, Object> second) {
Objects.requireNonNull(first, "merging requires two non-null maps but the first map was null"); Objects.requireNonNull(first, "merging requires two non-null maps but the first map was null");
Objects.requireNonNull(second, "merging requires two non-null maps but the second map was null"); Objects.requireNonNull(second, "merging requires two non-null maps but the second map was null");
Map<String, Object> results = new HashMap<>(first); Map<String, Object> results = new HashMap<>(first);
Set<String> prefixes = second.keySet().stream().map(MetadataCreateIndexService::prefix).collect(Collectors.toSet()); Set<String> prefixes = second.keySet().stream().map(MetadataCreateIndexService::prefix).collect(Collectors.toSet());
results.keySet().removeIf(k -> prefixes.contains(prefix(k))); List<String> matchedPrefixes = results.keySet().stream().filter(k -> prefixes.contains(prefix(k))).collect(Collectors.toList());
if (matchedPrefixes.size() > 0) {
throw new IllegalArgumentException("mapping fields " + matchedPrefixes + " cannot be replaced during template composition");
}
results.putAll(second); results.putAll(second);
return results; return results;
} }

View File

@ -197,15 +197,21 @@ public class MetadataIndexTemplateService {
validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry); validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry);
// Collect all the composable (index) templates that use this component template, we'll use
// this for validating that they're still going to be valid after this component template
// has been updated
final Map<String, ComposableIndexTemplate> templatesUsingComponent = currentState.metadata().templatesV2().entrySet().stream()
.filter(e -> e.getValue().composedOf().contains(name))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
// if we're updating a component template, let's check if it's part of any V2 template that will yield the CT update invalid // if we're updating a component template, let's check if it's part of any V2 template that will yield the CT update invalid
if (create == false && finalSettings != null) { if (create == false && finalSettings != null) {
// if the CT is specifying the `index.hidden` setting it cannot be part of any global template // if the CT is specifying the `index.hidden` setting it cannot be part of any global template
if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(finalSettings)) { if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(finalSettings)) {
Map<String, ComposableIndexTemplate> existingTemplates = currentState.metadata().templatesV2();
List<String> globalTemplatesThatUseThisComponent = new ArrayList<>(); List<String> globalTemplatesThatUseThisComponent = new ArrayList<>();
for (Map.Entry<String, ComposableIndexTemplate> entry : existingTemplates.entrySet()) { for (Map.Entry<String, ComposableIndexTemplate> entry : templatesUsingComponent.entrySet()) {
ComposableIndexTemplate templateV2 = entry.getValue(); ComposableIndexTemplate templateV2 = entry.getValue();
if (templateV2.composedOf().contains(name) && templateV2.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) { if (templateV2.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) {
// global templates don't support configuring the `index.hidden` setting so we don't need to resolve the settings as // global templates don't support configuring the `index.hidden` setting so we don't need to resolve the settings as
// no other component template can remove this setting from the resolved settings, so just invalidate this update // no other component template can remove this setting from the resolved settings, so just invalidate this update
globalTemplatesThatUseThisComponent.add(entry.getKey()); globalTemplatesThatUseThisComponent.add(entry.getKey());
@ -235,6 +241,32 @@ public class MetadataIndexTemplateService {
stringMappings == null ? null : new CompressedXContent(stringMappings), template.template().aliases()); stringMappings == null ? null : new CompressedXContent(stringMappings), template.template().aliases());
final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata()); final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata());
validate(name, finalComponentTemplate); validate(name, finalComponentTemplate);
if (templatesUsingComponent.size() > 0) {
ClusterState tempStateWithComponentTemplateAdded = ClusterState.builder(currentState)
.metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate))
.build();
Exception validationFailure = null;
for (Map.Entry<String, ComposableIndexTemplate> entry : templatesUsingComponent.entrySet()) {
final String composableTemplateName = entry.getKey();
final ComposableIndexTemplate composableTemplate = entry.getValue();
try {
validateCompositeTemplate(tempStateWithComponentTemplateAdded, composableTemplateName,
composableTemplate, indicesService, xContentRegistry);
} catch (Exception e) {
if (validationFailure == null) {
validationFailure = new IllegalArgumentException("updating component template [" + name +
"] results in invalid composable template [" + composableTemplateName + "] after templates are merged", e);
} else {
validationFailure.addSuppressed(e);
}
}
}
if (validationFailure != null) {
throw validationFailure;
}
}
logger.info("adding component template [{}]", name); logger.info("adding component template [{}]", name);
return ClusterState.builder(currentState) return ClusterState.builder(currentState)
.metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate)) .metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate))
@ -424,7 +456,6 @@ public class MetadataIndexTemplateService {
// adjusted (to add _doc) and it should be validated // adjusted (to add _doc) and it should be validated
CompressedXContent mappings = innerTemplate.mappings(); CompressedXContent mappings = innerTemplate.mappings();
String stringMappings = mappings == null ? null : mappings.string(); String stringMappings = mappings == null ? null : mappings.string();
validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry);
// Mappings in index templates don't include _doc, so update the mappings to include this single type // Mappings in index templates don't include _doc, so update the mappings to include this single type
if (stringMappings != null) { if (stringMappings != null) {
@ -443,6 +474,17 @@ public class MetadataIndexTemplateService {
} }
validate(name, finalIndexTemplate); validate(name, finalIndexTemplate);
// Finally, right before adding the template, we need to ensure that the composite settings,
// mappings, and aliases are valid after it's been composed with the component templates
try {
validateCompositeTemplate(currentState, name, finalIndexTemplate, indicesService, xContentRegistry);
} catch (Exception e) {
throw new IllegalArgumentException("composable template [" + name +
"] template after composition " +
(finalIndexTemplate.composedOf().size() > 0 ? "with component templates " + finalIndexTemplate.composedOf() + " " : "") +
"is invalid", e);
}
logger.info("adding index template [{}]", name); logger.info("adding index template [{}]", name);
return ClusterState.builder(currentState) return ClusterState.builder(currentState)
.metadata(Metadata.builder(currentState.metadata()).put(name, finalIndexTemplate)) .metadata(Metadata.builder(currentState.metadata()).put(name, finalIndexTemplate))
@ -787,7 +829,6 @@ public class MetadataIndexTemplateService {
return Collections.emptyList(); return Collections.emptyList();
} }
final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates(); final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates();
// TODO: more fine-grained merging of component template mappings, ie, merge fields as distint entities
List<CompressedXContent> mappings = template.composedOf().stream() List<CompressedXContent> mappings = template.composedOf().stream()
.map(componentTemplates::get) .map(componentTemplates::get)
.filter(Objects::nonNull) .filter(Objects::nonNull)
@ -894,6 +935,73 @@ public class MetadataIndexTemplateService {
return Collections.unmodifiableList(aliases); return Collections.unmodifiableList(aliases);
} }
/**
* Given a state and a composable template, validate that the final composite template
* generated by the composable template and all of its component templates contains valid
* settings, mappings, and aliases.
*/
private static void validateCompositeTemplate(final ClusterState state,
final String templateName,
final ComposableIndexTemplate template,
final IndicesService indicesService,
final NamedXContentRegistry xContentRegistry) throws Exception {
final ClusterState stateWithTemplate = ClusterState.builder(state)
.metadata(Metadata.builder(state.metadata()).put(templateName, template))
.build();
final String temporaryIndexName = "validate-template-" + UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT);
Settings resolvedSettings = resolveSettings(stateWithTemplate.metadata(), templateName);
// use the provided values, otherwise just pick valid dummy values
int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(resolvedSettings);
int dummyShards = resolvedSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS,
dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1);
int shardReplicas = resolvedSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
// Create the final aggregate settings, which will be used to create the temporary index metadata to validate everything
Settings finalResolvedSettings = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
.put(resolvedSettings)
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas)
.put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
.build();
// Validate index metadata (settings)
final ClusterState stateWithIndex = ClusterState.builder(stateWithTemplate)
.metadata(Metadata.builder(stateWithTemplate.metadata())
.put(IndexMetadata.builder(temporaryIndexName).settings(finalResolvedSettings))
.build())
.build();
final IndexMetadata tmpIndexMetadata = stateWithIndex.metadata().index(temporaryIndexName);
indicesService.withTempIndexService(tmpIndexMetadata,
tempIndexService -> {
// Validate aliases
MetadataCreateIndexService.resolveAndValidateAliases(temporaryIndexName, Collections.emptySet(),
MetadataIndexTemplateService.resolveAliases(stateWithIndex.metadata(), templateName), stateWithIndex.metadata(),
new AliasValidator(),
// the context is only used for validation so it's fine to pass fake values for the
// shard id and the current timestamp
xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null));
// Parse mappings to ensure they are valid after being composed
List<CompressedXContent> mappings = resolveMappings(stateWithIndex, templateName);
final Map<String, Map<String, Object>> finalMappings;
try {
MapperService dummyMapperService = tempIndexService.mapperService();
for (CompressedXContent mapping : mappings) {
// TODO: Eventually change this to:
// dummyMapperService.merge(MapperService.SINGLE_MAPPING_NAME, mapping, MergeReason.INDEX_TEMPLATE);
dummyMapperService.merge(MapperService.SINGLE_MAPPING_NAME, mapping, MergeReason.MAPPING_UPDATE);
}
} catch (Exception e) {
throw new IllegalArgumentException("invalid composite mappings for [" + templateName + "]", e);
}
return null;
});
}
private static void validateTemplate(Settings validateSettings, String mappings, private static void validateTemplate(Settings validateSettings, String mappings,
IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception { IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
validateTemplate(validateSettings, Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, mappings), validateTemplate(validateSettings, Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, mappings),

View File

@ -88,7 +88,7 @@ public class ComponentTemplateTests extends AbstractDiffableSerializationTestCas
public static Map<String, AliasMetadata> randomAliases() { public static Map<String, AliasMetadata> randomAliases() {
String aliasName = randomAlphaOfLength(5); String aliasName = randomAlphaOfLength(5);
AliasMetadata aliasMeta = AliasMetadata.builder(aliasName) AliasMetadata aliasMeta = AliasMetadata.builder(aliasName)
.filter(Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2))) .filter("{\"term\":{\"year\":" + randomIntBetween(1, 3000) + "}}")
.routing(randomBoolean() ? null : randomAlphaOfLength(3)) .routing(randomBoolean() ? null : randomAlphaOfLength(3))
.isHidden(randomBoolean() ? null : randomBoolean()) .isHidden(randomBoolean() ? null : randomBoolean())
.writeIndex(randomBoolean() ? null : randomBoolean()) .writeIndex(randomBoolean() ? null : randomBoolean())

View File

@ -103,7 +103,7 @@ public class ComposableIndexTemplateTests extends AbstractDiffableSerializationT
private static Map<String, AliasMetadata> randomAliases() { private static Map<String, AliasMetadata> randomAliases() {
String aliasName = randomAlphaOfLength(5); String aliasName = randomAlphaOfLength(5);
AliasMetadata aliasMeta = AliasMetadata.builder(aliasName) AliasMetadata aliasMeta = AliasMetadata.builder(aliasName)
.filter(Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2))) .filter("{\"term\":{\"year\":" + randomIntBetween(1, 3000) + "}}")
.routing(randomBoolean() ? null : randomAlphaOfLength(3)) .routing(randomBoolean() ? null : randomAlphaOfLength(3))
.isHidden(randomBoolean() ? null : randomBoolean()) .isHidden(randomBoolean() ? null : randomBoolean())
.writeIndex(randomBoolean() ? null : randomBoolean()) .writeIndex(randomBoolean() ? null : randomBoolean())

View File

@ -964,6 +964,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/57393")
public void testMappingsMergingIsSmart() throws Exception { public void testMappingsMergingIsSmart() throws Exception {
Template ctt1 = new Template(null, Template ctt1 = new Template(null,
new CompressedXContent("{\"_doc\":{\"_source\":{\"enabled\": false},\"_meta\":{\"ct1\":{\"ver\": \"text\"}}," + new CompressedXContent("{\"_doc\":{\"_source\":{\"enabled\": false},\"_meta\":{\"ct1\":{\"ver\": \"text\"}}," +
@ -1028,6 +1029,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/57393")
public void testMappingsMergingHandlesDots() throws Exception { public void testMappingsMergingHandlesDots() throws Exception {
Template ctt1 = new Template(null, Template ctt1 = new Template(null,
new CompressedXContent("{\"_doc\":{\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"type\": \"long\"}}}}}}"), null); new CompressedXContent("{\"_doc\":{\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"type\": \"long\"}}}}}}"), null);
@ -1062,33 +1064,31 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
equalTo(Collections.singletonMap("properties", Collections.singletonMap("bar", Collections.singletonMap("type", "long"))))); equalTo(Collections.singletonMap("properties", Collections.singletonMap("bar", Collections.singletonMap("type", "long")))));
} }
public void testMergeIgnoringDots() throws Exception { public void testMappingsMergingThrowsOnConflictDots() throws Exception {
Map<String, Object> first = new HashMap<>(); Template ctt1 = new Template(null,
first.put("foo", Collections.singletonMap("type", "long")); new CompressedXContent("{\"_doc\":{\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"type\": \"long\"}}}}}}"), null);
Map<String, Object> second = new HashMap<>(); Template ctt2 = new Template(null,
second.put("foo.bar", Collections.singletonMap("type", "long")); new CompressedXContent("{\"_doc\":{\"properties\":{\"foo.bar\":{\"type\": \"text\",\"analyzer\":\"english\"}}}}"), null);
Map<String, Object> results = MetadataCreateIndexService.mergeIgnoringDots(first, second);
assertThat(results, equalTo(second));
results = MetadataCreateIndexService.mergeIgnoringDots(second, first); ComponentTemplate ct1 = new ComponentTemplate(ctt1, null, null);
assertThat(results, equalTo(first)); ComponentTemplate ct2 = new ComponentTemplate(ctt2, null, null);
second.clear(); ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("index"),
Map<String, Object> inner = new HashMap<>(); null, Arrays.asList("ct2", "ct1"), null, null, null, null);
inner.put("type", "text");
inner.put("analyzer", "english");
second.put("foo", inner);
results = MetadataCreateIndexService.mergeIgnoringDots(first, second); ClusterState state = ClusterState.builder(ClusterState.EMPTY_STATE)
assertThat(results, equalTo(second)); .metadata(Metadata.builder(Metadata.EMPTY_METADATA)
.put("ct1", ct1)
.put("ct2", ct2)
.put("index-template", template)
.build())
.build();
first.put("baz", 3); IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
second.put("egg", 7); () -> MetadataCreateIndexService.resolveV2Mappings("{}", state,
"index-template", new NamedXContentRegistry(Collections.emptyList())));
results = MetadataCreateIndexService.mergeIgnoringDots(first, second); assertThat(e.getMessage(), containsString("mapping fields [foo.bar] cannot be replaced during template composition"));
Map<String, Object> expected = new HashMap<>(second);
expected.put("baz", 3);
assertThat(results, equalTo(expected));
} }
private IndexTemplateMetadata addMatchingTemplate(Consumer<IndexTemplateMetadata.Builder> configurator) { private IndexTemplateMetadata addMatchingTemplate(Consumer<IndexTemplateMetadata.Builder> configurator) {

View File

@ -66,10 +66,12 @@ import static org.hamcrest.CoreMatchers.containsStringIgnoringCase;
import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.matchesRegex;
public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
public void testIndexTemplateInvalidNumberOfShards() { public void testIndexTemplateInvalidNumberOfShards() {
@ -672,7 +674,8 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
} }
} }
public void testResolveMappings() throws Exception { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/57393")
public void testResolveConflictingMappings() throws Exception {
final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
ClusterState state = ClusterState.EMPTY_STATE; ClusterState state = ClusterState.EMPTY_STATE;
@ -738,6 +741,67 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
Collections.singletonMap("field", Collections.singletonMap("type", "keyword")))))); Collections.singletonMap("field", Collections.singletonMap("type", "keyword"))))));
} }
public void testResolveMappings() throws Exception {
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
ClusterState state = ClusterState.EMPTY_STATE;
ComponentTemplate ct1 = new ComponentTemplate(new Template(null,
new CompressedXContent("{\n" +
" \"properties\": {\n" +
" \"field1\": {\n" +
" \"type\": \"keyword\"\n" +
" }\n" +
" }\n" +
" }"), null), null, null);
ComponentTemplate ct2 = new ComponentTemplate(new Template(null,
new CompressedXContent("{\n" +
" \"properties\": {\n" +
" \"field2\": {\n" +
" \"type\": \"text\"\n" +
" }\n" +
" }\n" +
" }"), null), null, null);
state = service.addComponentTemplate(state, true, "ct_high", ct1);
state = service.addComponentTemplate(state, true, "ct_low", ct2);
ComposableIndexTemplate it = new ComposableIndexTemplate(Arrays.asList("i*"),
new Template(null,
new CompressedXContent("{\n" +
" \"properties\": {\n" +
" \"field3\": {\n" +
" \"type\": \"integer\"\n" +
" }\n" +
" }\n" +
" }"), null),
Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null);
state = service.addIndexTemplateV2(state, true, "my-template", it);
List<CompressedXContent> mappings = MetadataIndexTemplateService.resolveMappings(state, "my-template");
assertNotNull(mappings);
assertThat(mappings.size(), equalTo(3));
List<Map<String, Object>> parsedMappings = mappings.stream()
.map(m -> {
try {
return MapperService.parseMapping(new NamedXContentRegistry(Collections.emptyList()), m.string());
} catch (Exception e) {
logger.error(e);
fail("failed to parse mappings: " + m.string());
return null;
}
})
.collect(Collectors.toList());
assertThat(parsedMappings.get(0),
equalTo(Collections.singletonMap("_doc",
Collections.singletonMap("properties", Collections.singletonMap("field2", Collections.singletonMap("type", "text"))))));
assertThat(parsedMappings.get(1),
equalTo(Collections.singletonMap("_doc",
Collections.singletonMap("properties", Collections.singletonMap("field1", Collections.singletonMap("type", "keyword"))))));
assertThat(parsedMappings.get(2),
equalTo(Collections.singletonMap("_doc",
Collections.singletonMap("properties", Collections.singletonMap("field3", Collections.singletonMap("type", "integer"))))));
}
public void testResolveSettings() throws Exception { public void testResolveSettings() throws Exception {
final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
ClusterState state = ClusterState.EMPTY_STATE; ClusterState state = ClusterState.EMPTY_STATE;
@ -864,6 +928,130 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
containsString("component templates [ct] cannot be removed as they are still in use by index templates [template]")); containsString("component templates [ct] cannot be removed as they are still in use by index templates [template]"));
} }
/**
* Tests that we check that settings/mappings/etc are valid even after template composition,
* when adding/updating a composable index template
*/
public void testIndexTemplateFailsToOverrideComponentTemplateMappingField() throws Exception {
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
ClusterState state = ClusterState.EMPTY_STATE;
ComponentTemplate ct1 = new ComponentTemplate(new Template(null,
new CompressedXContent("{\n" +
" \"properties\": {\n" +
" \"field2\": {\n" +
" \"type\": \"object\",\n" +
" \"properties\": {\n" +
" \"foo\": {\n" +
" \"type\": \"integer\"\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
" }"), null), null, null);
ComponentTemplate ct2 = new ComponentTemplate(new Template(null,
new CompressedXContent("{\n" +
" \"properties\": {\n" +
" \"field1\": {\n" +
" \"type\": \"text\"\n" +
" }\n" +
" }\n" +
" }"), null), null, null);
state = service.addComponentTemplate(state, true, "c1", ct1);
state = service.addComponentTemplate(state, true, "c2", ct2);
ComposableIndexTemplate it = new ComposableIndexTemplate(Arrays.asList("i*"),
new Template(null, new CompressedXContent("{\n" +
" \"properties\": {\n" +
" \"field2\": {\n" +
" \"type\": \"text\"\n" +
" }\n" +
" }\n" +
" }"), null),
randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null);
final ClusterState finalState = state;
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> service.addIndexTemplateV2(finalState, randomBoolean(), "my-template", it));
assertThat(e.getMessage(),
matchesRegex("composable template \\[my-template\\] template after composition with component templates .+ is invalid"));
assertNotNull(e.getCause());
assertThat(e.getCause().getMessage(),
containsString("invalid composite mappings for [my-template]"));
assertNotNull(e.getCause().getCause());
assertThat(e.getCause().getCause().getMessage(),
anyOf(containsString("mapping fields [field2] cannot be replaced during template composition"),
containsString("Can't merge a non object mapping [field2] with an object mapping [field2]")));
}
/**
* Tests that we check that settings/mappings/etc are valid even after template composition,
* when updating a component template
*/
public void testUpdateComponentTemplateFailsIfResolvedIndexTemplatesWouldBeInvalid() throws Exception {
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
ClusterState state = ClusterState.EMPTY_STATE;
ComponentTemplate ct1 = new ComponentTemplate(new Template(null,
new CompressedXContent("{\n" +
" \"properties\": {\n" +
" \"field2\": {\n" +
" \"type\": \"object\",\n" +
" \"properties\": {\n" +
" \"foo\": {\n" +
" \"type\": \"integer\"\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
" }"), null), null, null);
ComponentTemplate ct2 = new ComponentTemplate(new Template(null,
new CompressedXContent("{\n" +
" \"properties\": {\n" +
" \"field1\": {\n" +
" \"type\": \"text\"\n" +
" }\n" +
" }\n" +
" }"), null), null, null);
state = service.addComponentTemplate(state, true, "c1", ct1);
state = service.addComponentTemplate(state, true, "c2", ct2);
ComposableIndexTemplate it = new ComposableIndexTemplate(Arrays.asList("i*"),
new Template(null, null, null),
randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null);
// Great, the templates aren't invalid
state = service.addIndexTemplateV2(state, randomBoolean(), "my-template", it);
ComponentTemplate changedCt2 = new ComponentTemplate(new Template(null,
new CompressedXContent("{\n" +
" \"properties\": {\n" +
" \"field2\": {\n" +
" \"type\": \"text\"\n" +
" }\n" +
" }\n" +
" }"), null), null, null);
final ClusterState finalState = state;
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> service.addComponentTemplate(finalState, false, "c2", changedCt2));
assertThat(e.getMessage(),
containsString("updating component template [c2] results in invalid " +
"composable template [my-template] after templates are merged"));
assertNotNull(e.getCause());
assertThat(e.getCause().getMessage(),
containsString("invalid composite mappings for [my-template]"));
assertNotNull(e.getCause().getCause());
assertThat(e.getCause().getCause().getMessage(),
anyOf(containsString("mapping fields [field2] cannot be replaced during template composition"),
containsString("Can't merge a non object mapping [field2] with an object mapping [field2]"),
containsString("mapper [field2] cannot be changed from type [text] to [ObjectMapper]")));
}
private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) { private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
MetadataCreateIndexService createIndexService = new MetadataCreateIndexService( MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(
Settings.EMPTY, Settings.EMPTY,