mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-23 05:15:04 +00:00
Backports the following commits to 7.x: Disallow merging existing mapping field definitions in templates (#57701)
This commit is contained in:
parent
16fcb64c99
commit
6e8cf0973f
@ -66,9 +66,6 @@ PUT _index_template/template_1
|
|||||||
"number_of_shards": 1
|
"number_of_shards": 1
|
||||||
},
|
},
|
||||||
"mappings": {
|
"mappings": {
|
||||||
"_source": {
|
|
||||||
"enabled": false
|
|
||||||
},
|
|
||||||
"properties": {
|
"properties": {
|
||||||
"host_name": {
|
"host_name": {
|
||||||
"type": "keyword"
|
"type": "keyword"
|
||||||
|
@ -14,7 +14,7 @@
|
|||||||
number_of_replicas: 1
|
number_of_replicas: 1
|
||||||
mappings:
|
mappings:
|
||||||
properties:
|
properties:
|
||||||
field2:
|
field1:
|
||||||
type: text
|
type: text
|
||||||
aliases:
|
aliases:
|
||||||
aliasname:
|
aliasname:
|
||||||
@ -47,9 +47,8 @@
|
|||||||
index.number_of_shards: 2
|
index.number_of_shards: 2
|
||||||
mappings:
|
mappings:
|
||||||
properties:
|
properties:
|
||||||
field:
|
field3:
|
||||||
type: keyword
|
type: integer
|
||||||
ignore_above: 255
|
|
||||||
aliases:
|
aliases:
|
||||||
my_alias: {}
|
my_alias: {}
|
||||||
aliasname:
|
aliasname:
|
||||||
@ -78,8 +77,9 @@
|
|||||||
- match: {bar-baz.settings.index.number_of_shards: "2"}
|
- match: {bar-baz.settings.index.number_of_shards: "2"}
|
||||||
- match: {bar-baz.settings.index.number_of_replicas: "0"}
|
- match: {bar-baz.settings.index.number_of_replicas: "0"}
|
||||||
- match: {bar-baz.settings.index.priority: "17"}
|
- match: {bar-baz.settings.index.priority: "17"}
|
||||||
- match: {bar-baz.mappings.properties.field: {type: keyword, ignore_above: 255}}
|
- match: {bar-baz.mappings.properties.field1: {type: text}}
|
||||||
- match: {bar-baz.mappings.properties.field2: {type: keyword}}
|
- match: {bar-baz.mappings.properties.field2: {type: keyword}}
|
||||||
|
- match: {bar-baz.mappings.properties.field3: {type: integer}}
|
||||||
- match: {bar-baz.mappings.properties.foo: {type: keyword}}
|
- match: {bar-baz.mappings.properties.foo: {type: keyword}}
|
||||||
- match: {bar-baz.aliases.aliasname: {filter: {match_all: {}}}}
|
- match: {bar-baz.aliases.aliasname: {filter: {match_all: {}}}}
|
||||||
- match: {bar-baz.aliases.my_alias: {}}
|
- match: {bar-baz.aliases.my_alias: {}}
|
||||||
|
@ -595,7 +595,7 @@ public class MetadataCreateIndexService {
|
|||||||
nonProperties = innerTemplateNonProperties;
|
nonProperties = innerTemplateNonProperties;
|
||||||
|
|
||||||
if (maybeProperties != null) {
|
if (maybeProperties != null) {
|
||||||
properties = mergeIgnoringDots(properties, maybeProperties);
|
properties = mergeFailingOnReplacement(properties, maybeProperties);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -609,7 +609,7 @@ public class MetadataCreateIndexService {
|
|||||||
nonProperties = innerRequestNonProperties;
|
nonProperties = innerRequestNonProperties;
|
||||||
|
|
||||||
if (maybeRequestProperties != null) {
|
if (maybeRequestProperties != null) {
|
||||||
properties = mergeIgnoringDots(properties, maybeRequestProperties);
|
properties = mergeFailingOnReplacement(properties, maybeRequestProperties);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -619,18 +619,18 @@ public class MetadataCreateIndexService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Add the objects in the second map to the first, where the keys in the {@code second} map have
|
* Add the objects in the second map to the first, A duplicated field is treated as illegal and
|
||||||
* higher predecence and overwrite the keys in the {@code first} map. In the event of a key with
|
* an exception is thrown.
|
||||||
* a dot in it (ie, "foo.bar"), the keys are treated as only the prefix counting towards
|
|
||||||
* equality. If the {@code second} map has a key such as "foo", all keys starting from "foo." in
|
|
||||||
* the {@code first} map are discarded.
|
|
||||||
*/
|
*/
|
||||||
static Map<String, Object> mergeIgnoringDots(Map<String, Object> first, Map<String, Object> second) {
|
static Map<String, Object> mergeFailingOnReplacement(Map<String, Object> first, Map<String, Object> second) {
|
||||||
Objects.requireNonNull(first, "merging requires two non-null maps but the first map was null");
|
Objects.requireNonNull(first, "merging requires two non-null maps but the first map was null");
|
||||||
Objects.requireNonNull(second, "merging requires two non-null maps but the second map was null");
|
Objects.requireNonNull(second, "merging requires two non-null maps but the second map was null");
|
||||||
Map<String, Object> results = new HashMap<>(first);
|
Map<String, Object> results = new HashMap<>(first);
|
||||||
Set<String> prefixes = second.keySet().stream().map(MetadataCreateIndexService::prefix).collect(Collectors.toSet());
|
Set<String> prefixes = second.keySet().stream().map(MetadataCreateIndexService::prefix).collect(Collectors.toSet());
|
||||||
results.keySet().removeIf(k -> prefixes.contains(prefix(k)));
|
List<String> matchedPrefixes = results.keySet().stream().filter(k -> prefixes.contains(prefix(k))).collect(Collectors.toList());
|
||||||
|
if (matchedPrefixes.size() > 0) {
|
||||||
|
throw new IllegalArgumentException("mapping fields " + matchedPrefixes + " cannot be replaced during template composition");
|
||||||
|
}
|
||||||
results.putAll(second);
|
results.putAll(second);
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
@ -197,15 +197,21 @@ public class MetadataIndexTemplateService {
|
|||||||
|
|
||||||
validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry);
|
validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry);
|
||||||
|
|
||||||
|
// Collect all the composable (index) templates that use this component template, we'll use
|
||||||
|
// this for validating that they're still going to be valid after this component template
|
||||||
|
// has been updated
|
||||||
|
final Map<String, ComposableIndexTemplate> templatesUsingComponent = currentState.metadata().templatesV2().entrySet().stream()
|
||||||
|
.filter(e -> e.getValue().composedOf().contains(name))
|
||||||
|
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||||
|
|
||||||
// if we're updating a component template, let's check if it's part of any V2 template that will yield the CT update invalid
|
// if we're updating a component template, let's check if it's part of any V2 template that will yield the CT update invalid
|
||||||
if (create == false && finalSettings != null) {
|
if (create == false && finalSettings != null) {
|
||||||
// if the CT is specifying the `index.hidden` setting it cannot be part of any global template
|
// if the CT is specifying the `index.hidden` setting it cannot be part of any global template
|
||||||
if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(finalSettings)) {
|
if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(finalSettings)) {
|
||||||
Map<String, ComposableIndexTemplate> existingTemplates = currentState.metadata().templatesV2();
|
|
||||||
List<String> globalTemplatesThatUseThisComponent = new ArrayList<>();
|
List<String> globalTemplatesThatUseThisComponent = new ArrayList<>();
|
||||||
for (Map.Entry<String, ComposableIndexTemplate> entry : existingTemplates.entrySet()) {
|
for (Map.Entry<String, ComposableIndexTemplate> entry : templatesUsingComponent.entrySet()) {
|
||||||
ComposableIndexTemplate templateV2 = entry.getValue();
|
ComposableIndexTemplate templateV2 = entry.getValue();
|
||||||
if (templateV2.composedOf().contains(name) && templateV2.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) {
|
if (templateV2.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) {
|
||||||
// global templates don't support configuring the `index.hidden` setting so we don't need to resolve the settings as
|
// global templates don't support configuring the `index.hidden` setting so we don't need to resolve the settings as
|
||||||
// no other component template can remove this setting from the resolved settings, so just invalidate this update
|
// no other component template can remove this setting from the resolved settings, so just invalidate this update
|
||||||
globalTemplatesThatUseThisComponent.add(entry.getKey());
|
globalTemplatesThatUseThisComponent.add(entry.getKey());
|
||||||
@ -235,6 +241,32 @@ public class MetadataIndexTemplateService {
|
|||||||
stringMappings == null ? null : new CompressedXContent(stringMappings), template.template().aliases());
|
stringMappings == null ? null : new CompressedXContent(stringMappings), template.template().aliases());
|
||||||
final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata());
|
final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata());
|
||||||
validate(name, finalComponentTemplate);
|
validate(name, finalComponentTemplate);
|
||||||
|
|
||||||
|
if (templatesUsingComponent.size() > 0) {
|
||||||
|
ClusterState tempStateWithComponentTemplateAdded = ClusterState.builder(currentState)
|
||||||
|
.metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate))
|
||||||
|
.build();
|
||||||
|
Exception validationFailure = null;
|
||||||
|
for (Map.Entry<String, ComposableIndexTemplate> entry : templatesUsingComponent.entrySet()) {
|
||||||
|
final String composableTemplateName = entry.getKey();
|
||||||
|
final ComposableIndexTemplate composableTemplate = entry.getValue();
|
||||||
|
try {
|
||||||
|
validateCompositeTemplate(tempStateWithComponentTemplateAdded, composableTemplateName,
|
||||||
|
composableTemplate, indicesService, xContentRegistry);
|
||||||
|
} catch (Exception e) {
|
||||||
|
if (validationFailure == null) {
|
||||||
|
validationFailure = new IllegalArgumentException("updating component template [" + name +
|
||||||
|
"] results in invalid composable template [" + composableTemplateName + "] after templates are merged", e);
|
||||||
|
} else {
|
||||||
|
validationFailure.addSuppressed(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (validationFailure != null) {
|
||||||
|
throw validationFailure;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
logger.info("adding component template [{}]", name);
|
logger.info("adding component template [{}]", name);
|
||||||
return ClusterState.builder(currentState)
|
return ClusterState.builder(currentState)
|
||||||
.metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate))
|
.metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate))
|
||||||
@ -424,7 +456,6 @@ public class MetadataIndexTemplateService {
|
|||||||
// adjusted (to add _doc) and it should be validated
|
// adjusted (to add _doc) and it should be validated
|
||||||
CompressedXContent mappings = innerTemplate.mappings();
|
CompressedXContent mappings = innerTemplate.mappings();
|
||||||
String stringMappings = mappings == null ? null : mappings.string();
|
String stringMappings = mappings == null ? null : mappings.string();
|
||||||
validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry);
|
|
||||||
|
|
||||||
// Mappings in index templates don't include _doc, so update the mappings to include this single type
|
// Mappings in index templates don't include _doc, so update the mappings to include this single type
|
||||||
if (stringMappings != null) {
|
if (stringMappings != null) {
|
||||||
@ -443,6 +474,17 @@ public class MetadataIndexTemplateService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
validate(name, finalIndexTemplate);
|
validate(name, finalIndexTemplate);
|
||||||
|
|
||||||
|
// Finally, right before adding the template, we need to ensure that the composite settings,
|
||||||
|
// mappings, and aliases are valid after it's been composed with the component templates
|
||||||
|
try {
|
||||||
|
validateCompositeTemplate(currentState, name, finalIndexTemplate, indicesService, xContentRegistry);
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IllegalArgumentException("composable template [" + name +
|
||||||
|
"] template after composition " +
|
||||||
|
(finalIndexTemplate.composedOf().size() > 0 ? "with component templates " + finalIndexTemplate.composedOf() + " " : "") +
|
||||||
|
"is invalid", e);
|
||||||
|
}
|
||||||
logger.info("adding index template [{}]", name);
|
logger.info("adding index template [{}]", name);
|
||||||
return ClusterState.builder(currentState)
|
return ClusterState.builder(currentState)
|
||||||
.metadata(Metadata.builder(currentState.metadata()).put(name, finalIndexTemplate))
|
.metadata(Metadata.builder(currentState.metadata()).put(name, finalIndexTemplate))
|
||||||
@ -787,7 +829,6 @@ public class MetadataIndexTemplateService {
|
|||||||
return Collections.emptyList();
|
return Collections.emptyList();
|
||||||
}
|
}
|
||||||
final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates();
|
final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates();
|
||||||
// TODO: more fine-grained merging of component template mappings, ie, merge fields as distint entities
|
|
||||||
List<CompressedXContent> mappings = template.composedOf().stream()
|
List<CompressedXContent> mappings = template.composedOf().stream()
|
||||||
.map(componentTemplates::get)
|
.map(componentTemplates::get)
|
||||||
.filter(Objects::nonNull)
|
.filter(Objects::nonNull)
|
||||||
@ -894,6 +935,73 @@ public class MetadataIndexTemplateService {
|
|||||||
return Collections.unmodifiableList(aliases);
|
return Collections.unmodifiableList(aliases);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Given a state and a composable template, validate that the final composite template
|
||||||
|
* generated by the composable template and all of its component templates contains valid
|
||||||
|
* settings, mappings, and aliases.
|
||||||
|
*/
|
||||||
|
private static void validateCompositeTemplate(final ClusterState state,
|
||||||
|
final String templateName,
|
||||||
|
final ComposableIndexTemplate template,
|
||||||
|
final IndicesService indicesService,
|
||||||
|
final NamedXContentRegistry xContentRegistry) throws Exception {
|
||||||
|
final ClusterState stateWithTemplate = ClusterState.builder(state)
|
||||||
|
.metadata(Metadata.builder(state.metadata()).put(templateName, template))
|
||||||
|
.build();
|
||||||
|
|
||||||
|
final String temporaryIndexName = "validate-template-" + UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT);
|
||||||
|
Settings resolvedSettings = resolveSettings(stateWithTemplate.metadata(), templateName);
|
||||||
|
|
||||||
|
// use the provided values, otherwise just pick valid dummy values
|
||||||
|
int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(resolvedSettings);
|
||||||
|
int dummyShards = resolvedSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS,
|
||||||
|
dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1);
|
||||||
|
int shardReplicas = resolvedSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
|
||||||
|
|
||||||
|
|
||||||
|
// Create the final aggregate settings, which will be used to create the temporary index metadata to validate everything
|
||||||
|
Settings finalResolvedSettings = Settings.builder()
|
||||||
|
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||||
|
.put(resolvedSettings)
|
||||||
|
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards)
|
||||||
|
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas)
|
||||||
|
.put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
|
||||||
|
.build();
|
||||||
|
|
||||||
|
// Validate index metadata (settings)
|
||||||
|
final ClusterState stateWithIndex = ClusterState.builder(stateWithTemplate)
|
||||||
|
.metadata(Metadata.builder(stateWithTemplate.metadata())
|
||||||
|
.put(IndexMetadata.builder(temporaryIndexName).settings(finalResolvedSettings))
|
||||||
|
.build())
|
||||||
|
.build();
|
||||||
|
final IndexMetadata tmpIndexMetadata = stateWithIndex.metadata().index(temporaryIndexName);
|
||||||
|
indicesService.withTempIndexService(tmpIndexMetadata,
|
||||||
|
tempIndexService -> {
|
||||||
|
// Validate aliases
|
||||||
|
MetadataCreateIndexService.resolveAndValidateAliases(temporaryIndexName, Collections.emptySet(),
|
||||||
|
MetadataIndexTemplateService.resolveAliases(stateWithIndex.metadata(), templateName), stateWithIndex.metadata(),
|
||||||
|
new AliasValidator(),
|
||||||
|
// the context is only used for validation so it's fine to pass fake values for the
|
||||||
|
// shard id and the current timestamp
|
||||||
|
xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null));
|
||||||
|
|
||||||
|
// Parse mappings to ensure they are valid after being composed
|
||||||
|
List<CompressedXContent> mappings = resolveMappings(stateWithIndex, templateName);
|
||||||
|
final Map<String, Map<String, Object>> finalMappings;
|
||||||
|
try {
|
||||||
|
MapperService dummyMapperService = tempIndexService.mapperService();
|
||||||
|
for (CompressedXContent mapping : mappings) {
|
||||||
|
// TODO: Eventually change this to:
|
||||||
|
// dummyMapperService.merge(MapperService.SINGLE_MAPPING_NAME, mapping, MergeReason.INDEX_TEMPLATE);
|
||||||
|
dummyMapperService.merge(MapperService.SINGLE_MAPPING_NAME, mapping, MergeReason.MAPPING_UPDATE);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IllegalArgumentException("invalid composite mappings for [" + templateName + "]", e);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private static void validateTemplate(Settings validateSettings, String mappings,
|
private static void validateTemplate(Settings validateSettings, String mappings,
|
||||||
IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
|
IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
|
||||||
validateTemplate(validateSettings, Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, mappings),
|
validateTemplate(validateSettings, Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, mappings),
|
||||||
|
@ -88,7 +88,7 @@ public class ComponentTemplateTests extends AbstractDiffableSerializationTestCas
|
|||||||
public static Map<String, AliasMetadata> randomAliases() {
|
public static Map<String, AliasMetadata> randomAliases() {
|
||||||
String aliasName = randomAlphaOfLength(5);
|
String aliasName = randomAlphaOfLength(5);
|
||||||
AliasMetadata aliasMeta = AliasMetadata.builder(aliasName)
|
AliasMetadata aliasMeta = AliasMetadata.builder(aliasName)
|
||||||
.filter(Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)))
|
.filter("{\"term\":{\"year\":" + randomIntBetween(1, 3000) + "}}")
|
||||||
.routing(randomBoolean() ? null : randomAlphaOfLength(3))
|
.routing(randomBoolean() ? null : randomAlphaOfLength(3))
|
||||||
.isHidden(randomBoolean() ? null : randomBoolean())
|
.isHidden(randomBoolean() ? null : randomBoolean())
|
||||||
.writeIndex(randomBoolean() ? null : randomBoolean())
|
.writeIndex(randomBoolean() ? null : randomBoolean())
|
||||||
|
@ -103,7 +103,7 @@ public class ComposableIndexTemplateTests extends AbstractDiffableSerializationT
|
|||||||
private static Map<String, AliasMetadata> randomAliases() {
|
private static Map<String, AliasMetadata> randomAliases() {
|
||||||
String aliasName = randomAlphaOfLength(5);
|
String aliasName = randomAlphaOfLength(5);
|
||||||
AliasMetadata aliasMeta = AliasMetadata.builder(aliasName)
|
AliasMetadata aliasMeta = AliasMetadata.builder(aliasName)
|
||||||
.filter(Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)))
|
.filter("{\"term\":{\"year\":" + randomIntBetween(1, 3000) + "}}")
|
||||||
.routing(randomBoolean() ? null : randomAlphaOfLength(3))
|
.routing(randomBoolean() ? null : randomAlphaOfLength(3))
|
||||||
.isHidden(randomBoolean() ? null : randomBoolean())
|
.isHidden(randomBoolean() ? null : randomBoolean())
|
||||||
.writeIndex(randomBoolean() ? null : randomBoolean())
|
.writeIndex(randomBoolean() ? null : randomBoolean())
|
||||||
|
@ -964,6 +964,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
|
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/57393")
|
||||||
public void testMappingsMergingIsSmart() throws Exception {
|
public void testMappingsMergingIsSmart() throws Exception {
|
||||||
Template ctt1 = new Template(null,
|
Template ctt1 = new Template(null,
|
||||||
new CompressedXContent("{\"_doc\":{\"_source\":{\"enabled\": false},\"_meta\":{\"ct1\":{\"ver\": \"text\"}}," +
|
new CompressedXContent("{\"_doc\":{\"_source\":{\"enabled\": false},\"_meta\":{\"ct1\":{\"ver\": \"text\"}}," +
|
||||||
@ -1028,6 +1029,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
|
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/57393")
|
||||||
public void testMappingsMergingHandlesDots() throws Exception {
|
public void testMappingsMergingHandlesDots() throws Exception {
|
||||||
Template ctt1 = new Template(null,
|
Template ctt1 = new Template(null,
|
||||||
new CompressedXContent("{\"_doc\":{\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"type\": \"long\"}}}}}}"), null);
|
new CompressedXContent("{\"_doc\":{\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"type\": \"long\"}}}}}}"), null);
|
||||||
@ -1062,33 +1064,31 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
|
|||||||
equalTo(Collections.singletonMap("properties", Collections.singletonMap("bar", Collections.singletonMap("type", "long")))));
|
equalTo(Collections.singletonMap("properties", Collections.singletonMap("bar", Collections.singletonMap("type", "long")))));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testMergeIgnoringDots() throws Exception {
|
public void testMappingsMergingThrowsOnConflictDots() throws Exception {
|
||||||
Map<String, Object> first = new HashMap<>();
|
Template ctt1 = new Template(null,
|
||||||
first.put("foo", Collections.singletonMap("type", "long"));
|
new CompressedXContent("{\"_doc\":{\"properties\":{\"foo\":{\"properties\":{\"bar\":{\"type\": \"long\"}}}}}}"), null);
|
||||||
Map<String, Object> second = new HashMap<>();
|
Template ctt2 = new Template(null,
|
||||||
second.put("foo.bar", Collections.singletonMap("type", "long"));
|
new CompressedXContent("{\"_doc\":{\"properties\":{\"foo.bar\":{\"type\": \"text\",\"analyzer\":\"english\"}}}}"), null);
|
||||||
Map<String, Object> results = MetadataCreateIndexService.mergeIgnoringDots(first, second);
|
|
||||||
assertThat(results, equalTo(second));
|
|
||||||
|
|
||||||
results = MetadataCreateIndexService.mergeIgnoringDots(second, first);
|
ComponentTemplate ct1 = new ComponentTemplate(ctt1, null, null);
|
||||||
assertThat(results, equalTo(first));
|
ComponentTemplate ct2 = new ComponentTemplate(ctt2, null, null);
|
||||||
|
|
||||||
second.clear();
|
ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("index"),
|
||||||
Map<String, Object> inner = new HashMap<>();
|
null, Arrays.asList("ct2", "ct1"), null, null, null, null);
|
||||||
inner.put("type", "text");
|
|
||||||
inner.put("analyzer", "english");
|
|
||||||
second.put("foo", inner);
|
|
||||||
|
|
||||||
results = MetadataCreateIndexService.mergeIgnoringDots(first, second);
|
ClusterState state = ClusterState.builder(ClusterState.EMPTY_STATE)
|
||||||
assertThat(results, equalTo(second));
|
.metadata(Metadata.builder(Metadata.EMPTY_METADATA)
|
||||||
|
.put("ct1", ct1)
|
||||||
|
.put("ct2", ct2)
|
||||||
|
.put("index-template", template)
|
||||||
|
.build())
|
||||||
|
.build();
|
||||||
|
|
||||||
first.put("baz", 3);
|
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
|
||||||
second.put("egg", 7);
|
() -> MetadataCreateIndexService.resolveV2Mappings("{}", state,
|
||||||
|
"index-template", new NamedXContentRegistry(Collections.emptyList())));
|
||||||
|
|
||||||
results = MetadataCreateIndexService.mergeIgnoringDots(first, second);
|
assertThat(e.getMessage(), containsString("mapping fields [foo.bar] cannot be replaced during template composition"));
|
||||||
Map<String, Object> expected = new HashMap<>(second);
|
|
||||||
expected.put("baz", 3);
|
|
||||||
assertThat(results, equalTo(expected));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private IndexTemplateMetadata addMatchingTemplate(Consumer<IndexTemplateMetadata.Builder> configurator) {
|
private IndexTemplateMetadata addMatchingTemplate(Consumer<IndexTemplateMetadata.Builder> configurator) {
|
||||||
|
@ -66,10 +66,12 @@ import static org.hamcrest.CoreMatchers.containsStringIgnoringCase;
|
|||||||
import static org.hamcrest.CoreMatchers.equalTo;
|
import static org.hamcrest.CoreMatchers.equalTo;
|
||||||
import static org.hamcrest.CoreMatchers.instanceOf;
|
import static org.hamcrest.CoreMatchers.instanceOf;
|
||||||
import static org.hamcrest.CoreMatchers.not;
|
import static org.hamcrest.CoreMatchers.not;
|
||||||
|
import static org.hamcrest.Matchers.anyOf;
|
||||||
import static org.hamcrest.Matchers.contains;
|
import static org.hamcrest.Matchers.contains;
|
||||||
import static org.hamcrest.Matchers.containsInAnyOrder;
|
import static org.hamcrest.Matchers.containsInAnyOrder;
|
||||||
import static org.hamcrest.Matchers.empty;
|
import static org.hamcrest.Matchers.empty;
|
||||||
import static org.hamcrest.Matchers.is;
|
import static org.hamcrest.Matchers.is;
|
||||||
|
import static org.hamcrest.Matchers.matchesRegex;
|
||||||
|
|
||||||
public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
|
public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
|
||||||
public void testIndexTemplateInvalidNumberOfShards() {
|
public void testIndexTemplateInvalidNumberOfShards() {
|
||||||
@ -672,7 +674,8 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testResolveMappings() throws Exception {
|
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/57393")
|
||||||
|
public void testResolveConflictingMappings() throws Exception {
|
||||||
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
|
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
|
||||||
ClusterState state = ClusterState.EMPTY_STATE;
|
ClusterState state = ClusterState.EMPTY_STATE;
|
||||||
|
|
||||||
@ -738,6 +741,67 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
|
|||||||
Collections.singletonMap("field", Collections.singletonMap("type", "keyword"))))));
|
Collections.singletonMap("field", Collections.singletonMap("type", "keyword"))))));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testResolveMappings() throws Exception {
|
||||||
|
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
|
||||||
|
ClusterState state = ClusterState.EMPTY_STATE;
|
||||||
|
|
||||||
|
ComponentTemplate ct1 = new ComponentTemplate(new Template(null,
|
||||||
|
new CompressedXContent("{\n" +
|
||||||
|
" \"properties\": {\n" +
|
||||||
|
" \"field1\": {\n" +
|
||||||
|
" \"type\": \"keyword\"\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }"), null), null, null);
|
||||||
|
ComponentTemplate ct2 = new ComponentTemplate(new Template(null,
|
||||||
|
new CompressedXContent("{\n" +
|
||||||
|
" \"properties\": {\n" +
|
||||||
|
" \"field2\": {\n" +
|
||||||
|
" \"type\": \"text\"\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }"), null), null, null);
|
||||||
|
state = service.addComponentTemplate(state, true, "ct_high", ct1);
|
||||||
|
state = service.addComponentTemplate(state, true, "ct_low", ct2);
|
||||||
|
ComposableIndexTemplate it = new ComposableIndexTemplate(Arrays.asList("i*"),
|
||||||
|
new Template(null,
|
||||||
|
new CompressedXContent("{\n" +
|
||||||
|
" \"properties\": {\n" +
|
||||||
|
" \"field3\": {\n" +
|
||||||
|
" \"type\": \"integer\"\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }"), null),
|
||||||
|
Arrays.asList("ct_low", "ct_high"), 0L, 1L, null, null);
|
||||||
|
state = service.addIndexTemplateV2(state, true, "my-template", it);
|
||||||
|
|
||||||
|
List<CompressedXContent> mappings = MetadataIndexTemplateService.resolveMappings(state, "my-template");
|
||||||
|
|
||||||
|
assertNotNull(mappings);
|
||||||
|
assertThat(mappings.size(), equalTo(3));
|
||||||
|
List<Map<String, Object>> parsedMappings = mappings.stream()
|
||||||
|
.map(m -> {
|
||||||
|
try {
|
||||||
|
return MapperService.parseMapping(new NamedXContentRegistry(Collections.emptyList()), m.string());
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e);
|
||||||
|
fail("failed to parse mappings: " + m.string());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
assertThat(parsedMappings.get(0),
|
||||||
|
equalTo(Collections.singletonMap("_doc",
|
||||||
|
Collections.singletonMap("properties", Collections.singletonMap("field2", Collections.singletonMap("type", "text"))))));
|
||||||
|
assertThat(parsedMappings.get(1),
|
||||||
|
equalTo(Collections.singletonMap("_doc",
|
||||||
|
Collections.singletonMap("properties", Collections.singletonMap("field1", Collections.singletonMap("type", "keyword"))))));
|
||||||
|
assertThat(parsedMappings.get(2),
|
||||||
|
equalTo(Collections.singletonMap("_doc",
|
||||||
|
Collections.singletonMap("properties", Collections.singletonMap("field3", Collections.singletonMap("type", "integer"))))));
|
||||||
|
}
|
||||||
|
|
||||||
public void testResolveSettings() throws Exception {
|
public void testResolveSettings() throws Exception {
|
||||||
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
|
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
|
||||||
ClusterState state = ClusterState.EMPTY_STATE;
|
ClusterState state = ClusterState.EMPTY_STATE;
|
||||||
@ -864,6 +928,130 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
|
|||||||
containsString("component templates [ct] cannot be removed as they are still in use by index templates [template]"));
|
containsString("component templates [ct] cannot be removed as they are still in use by index templates [template]"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests that we check that settings/mappings/etc are valid even after template composition,
|
||||||
|
* when adding/updating a composable index template
|
||||||
|
*/
|
||||||
|
public void testIndexTemplateFailsToOverrideComponentTemplateMappingField() throws Exception {
|
||||||
|
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
|
||||||
|
ClusterState state = ClusterState.EMPTY_STATE;
|
||||||
|
|
||||||
|
ComponentTemplate ct1 = new ComponentTemplate(new Template(null,
|
||||||
|
new CompressedXContent("{\n" +
|
||||||
|
" \"properties\": {\n" +
|
||||||
|
" \"field2\": {\n" +
|
||||||
|
" \"type\": \"object\",\n" +
|
||||||
|
" \"properties\": {\n" +
|
||||||
|
" \"foo\": {\n" +
|
||||||
|
" \"type\": \"integer\"\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }"), null), null, null);
|
||||||
|
ComponentTemplate ct2 = new ComponentTemplate(new Template(null,
|
||||||
|
new CompressedXContent("{\n" +
|
||||||
|
" \"properties\": {\n" +
|
||||||
|
" \"field1\": {\n" +
|
||||||
|
" \"type\": \"text\"\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }"), null), null, null);
|
||||||
|
state = service.addComponentTemplate(state, true, "c1", ct1);
|
||||||
|
state = service.addComponentTemplate(state, true, "c2", ct2);
|
||||||
|
ComposableIndexTemplate it = new ComposableIndexTemplate(Arrays.asList("i*"),
|
||||||
|
new Template(null, new CompressedXContent("{\n" +
|
||||||
|
" \"properties\": {\n" +
|
||||||
|
" \"field2\": {\n" +
|
||||||
|
" \"type\": \"text\"\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }"), null),
|
||||||
|
randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null);
|
||||||
|
|
||||||
|
final ClusterState finalState = state;
|
||||||
|
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
|
||||||
|
() -> service.addIndexTemplateV2(finalState, randomBoolean(), "my-template", it));
|
||||||
|
|
||||||
|
assertThat(e.getMessage(),
|
||||||
|
matchesRegex("composable template \\[my-template\\] template after composition with component templates .+ is invalid"));
|
||||||
|
|
||||||
|
assertNotNull(e.getCause());
|
||||||
|
assertThat(e.getCause().getMessage(),
|
||||||
|
containsString("invalid composite mappings for [my-template]"));
|
||||||
|
|
||||||
|
assertNotNull(e.getCause().getCause());
|
||||||
|
assertThat(e.getCause().getCause().getMessage(),
|
||||||
|
anyOf(containsString("mapping fields [field2] cannot be replaced during template composition"),
|
||||||
|
containsString("Can't merge a non object mapping [field2] with an object mapping [field2]")));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests that we check that settings/mappings/etc are valid even after template composition,
|
||||||
|
* when updating a component template
|
||||||
|
*/
|
||||||
|
public void testUpdateComponentTemplateFailsIfResolvedIndexTemplatesWouldBeInvalid() throws Exception {
|
||||||
|
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
|
||||||
|
ClusterState state = ClusterState.EMPTY_STATE;
|
||||||
|
|
||||||
|
ComponentTemplate ct1 = new ComponentTemplate(new Template(null,
|
||||||
|
new CompressedXContent("{\n" +
|
||||||
|
" \"properties\": {\n" +
|
||||||
|
" \"field2\": {\n" +
|
||||||
|
" \"type\": \"object\",\n" +
|
||||||
|
" \"properties\": {\n" +
|
||||||
|
" \"foo\": {\n" +
|
||||||
|
" \"type\": \"integer\"\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }"), null), null, null);
|
||||||
|
ComponentTemplate ct2 = new ComponentTemplate(new Template(null,
|
||||||
|
new CompressedXContent("{\n" +
|
||||||
|
" \"properties\": {\n" +
|
||||||
|
" \"field1\": {\n" +
|
||||||
|
" \"type\": \"text\"\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }"), null), null, null);
|
||||||
|
state = service.addComponentTemplate(state, true, "c1", ct1);
|
||||||
|
state = service.addComponentTemplate(state, true, "c2", ct2);
|
||||||
|
ComposableIndexTemplate it = new ComposableIndexTemplate(Arrays.asList("i*"),
|
||||||
|
new Template(null, null, null),
|
||||||
|
randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null);
|
||||||
|
|
||||||
|
// Great, the templates aren't invalid
|
||||||
|
state = service.addIndexTemplateV2(state, randomBoolean(), "my-template", it);
|
||||||
|
|
||||||
|
ComponentTemplate changedCt2 = new ComponentTemplate(new Template(null,
|
||||||
|
new CompressedXContent("{\n" +
|
||||||
|
" \"properties\": {\n" +
|
||||||
|
" \"field2\": {\n" +
|
||||||
|
" \"type\": \"text\"\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }\n" +
|
||||||
|
" }"), null), null, null);
|
||||||
|
|
||||||
|
final ClusterState finalState = state;
|
||||||
|
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
|
||||||
|
() -> service.addComponentTemplate(finalState, false, "c2", changedCt2));
|
||||||
|
|
||||||
|
assertThat(e.getMessage(),
|
||||||
|
containsString("updating component template [c2] results in invalid " +
|
||||||
|
"composable template [my-template] after templates are merged"));
|
||||||
|
|
||||||
|
assertNotNull(e.getCause());
|
||||||
|
assertThat(e.getCause().getMessage(),
|
||||||
|
containsString("invalid composite mappings for [my-template]"));
|
||||||
|
|
||||||
|
assertNotNull(e.getCause().getCause());
|
||||||
|
assertThat(e.getCause().getCause().getMessage(),
|
||||||
|
anyOf(containsString("mapping fields [field2] cannot be replaced during template composition"),
|
||||||
|
containsString("Can't merge a non object mapping [field2] with an object mapping [field2]"),
|
||||||
|
containsString("mapper [field2] cannot be changed from type [text] to [ObjectMapper]")));
|
||||||
|
}
|
||||||
|
|
||||||
private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
|
private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
|
||||||
MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(
|
MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(
|
||||||
Settings.EMPTY,
|
Settings.EMPTY,
|
||||||
|
Loading…
x
Reference in New Issue
Block a user