honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-25 09:28:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[7.x] Use V2 index templates during index creation (#54669) (#54750)

Browse Source 
* Use V2 index templates during index creation

This commit changes our index creation code to use (and favor!) V2 index templates during index
creation. The creation precedence goes like so, in order of precedence:

- Existing source `IndexMetadata` - for example, when recovering from a peer or a shrink/split/clone
  where index templates should not be applied
- A matching V2 index template, if one is found
  - When a V2 template is found, all component templates (in the `composed_of` field) are applied
    in the order that they appear, with the index template having the 2nd highest precedence (the
    create index request always has the top priority when it comes to index settings)
- All matching V1 templates (the old style)

This also adds index template validation when `PUT`-ing a new v2 index template (because this was
required) and ensures that all index and component templates specify *no* top-level mapping type (it
is automatically added when the template is added to the cluster state).

This does not yet implement fine-grained component template merging of mappings, where we favor
merging only a single field's configuration, that will be done in subsequent work.

This also keeps the existing hidden index behavior present for v1 templates, where a hidden index
will match v2 index templates unless they are global (`*`) templates.

Relates to #53101
This commit is contained in:
Lee Hinman 2020-04-03 14:46:15 -06:00 committed by GitHub
parent 548145f4a3
commit 814c248819
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 953 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml Normal file
View File

@ -0,0 +1,188 @@
---
"Component and index template composition":
- skip:
version: " - 7.7.99"
reason: "itv2 is available in 7.8+"
- do:
cluster.put_component_template:
name: ct_low
body:
template:
settings:
number_of_replicas: 1
mappings:
properties:
field2:
type: text
aliases:
aliasname:
is_write_index: false
- do:
cluster.put_component_template:
name: ct_high
body:
template:
settings:
index.number_of_replicas: 0
mappings:
properties:
field2:
type: keyword
aliases:
aliasname:
is_write_index: true
- do:
indices.put_index_template:
name: my-template
body:
index_patterns: ["foo", "bar-*"]
template:
settings:
index.number_of_shards: 2
mappings:
properties:
field:
type: keyword
ignore_above: 255
aliases:
my_alias: {}
aliasname:
filter:
match_all: {}
composed_of: ["ct_low", "ct_high"]
priority: 400
- do:
indices.create:
index: bar-baz
body:
settings:
index.priority: 17
mappings:
properties:
foo:
type: keyword
aliases:
other: {}
- do:
indices.get:
index: bar-baz
- match: {bar-baz.settings.index.number_of_shards: "2"}
- match: {bar-baz.settings.index.number_of_replicas: "0"}
- match: {bar-baz.settings.index.priority: "17"}
- match: {bar-baz.mappings.properties.field: {type: keyword, ignore_above: 255}}
- match: {bar-baz.mappings.properties.field2: {type: keyword}}
- match: {bar-baz.mappings.properties.foo: {type: keyword}}
- match: {bar-baz.aliases.aliasname: {filter: {match_all: {}}}}
- match: {bar-baz.aliases.my_alias: {}}
- match: {bar-baz.aliases.other: {}}
---
"Index template priority":
- skip:
version: " - 7.7.99"
reason: "itv2 is available in 7.8+"
- do:
indices.put_index_template:
name: my-template
body:
index_patterns: ["foo", "bar-*"]
template:
settings:
index.number_of_shards: 2
composed_of: []
priority: 400
- do:
indices.put_index_template:
name: another-template
body:
index_patterns: ["bar-*"]
template:
settings:
index.number_of_shards: 3
composed_of: []
priority: 405
- do:
indices.create:
index: bar-baz
- do:
indices.get:
index: bar-baz
- match: {bar-baz.settings.index.number_of_shards: "3"}
---
"Component template only composition":
- skip:
version: " - 7.7.99"
reason: "itv2 is available in 7.8+"
- do:
cluster.put_component_template:
name: ct_low
body:
template:
aliases:
alias1: {}
- do:
cluster.put_component_template:
name: ct_high
body:
template:
mappings:
properties:
field:
type: keyword
- do:
indices.put_index_template:
name: my-template
body:
index_patterns: ["baz*"]
composed_of: ["ct_low", "ct_high"]
- do:
indices.create:
index: bazfoo
- do:
indices.get:
index: bazfoo
- match: {bazfoo.mappings.properties.field: {type: keyword}}
- match: {bazfoo.aliases.alias1: {}}
---
"Index template without component templates":
- skip:
version: " - 7.7.99"
reason: "itv2 is available in 7.8+"
- do:
indices.put_index_template:
name: my-template
body:
index_patterns: ["eggplant"]
template:
settings:
number_of_shards: 3
- do:
indices.create:
index: eggplant
- do:
indices.get:
index: eggplant
- match: {eggplant.settings.index.number_of_shards: "3"}

4
server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java
View File

@ -42,7 +42,7 @@ import java.util.List;
import java.util.Locale;
import java.util.regex.Pattern;
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findTemplates;
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates;
public class MetadataRolloverService {
private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-\\d+$");
@ -165,7 +165,7 @@ public class MetadataRolloverService {
*/
static void checkNoDuplicatedAliasInIndexTemplate(Metadata metadata, String rolloverIndexName, String rolloverRequestAlias,
@Nullable Boolean isHidden) {
final List<IndexTemplateMetadata> matchedTemplates = findTemplates(metadata, rolloverIndexName, isHidden);
final List<IndexTemplateMetadata> matchedTemplates = findV1Templates(metadata, rolloverIndexName, isHidden);
for (IndexTemplateMetadata template : matchedTemplates) {
if (template.aliases().containsKey(rolloverRequestAlias)) {
throw new IllegalArgumentException(String.format(Locale.ROOT,

13
server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateV2Action.java
View File

@ -26,15 +26,12 @@ import org.elasticsearch.action.support.master.TransportMasterNodeAction;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
@ -74,14 +71,6 @@ public class TransportPutIndexTemplateV2Action
protected void masterOperation(final PutIndexTemplateV2Action.Request request, final ClusterState state,
final ActionListener<AcknowledgedResponse> listener) {
IndexTemplateV2 indexTemplate = request.indexTemplate();
Template template = indexTemplate.template();
// Normalize the index settings if necessary
if (template.settings() != null) {
Settings.Builder settings = Settings.builder().put(template.settings()).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
template = new Template(settings.build(), template.mappings(), template.aliases());
indexTemplate = new IndexTemplateV2(indexTemplate.indexPatterns(), template, indexTemplate.composedOf(),
indexTemplate.priority(), indexTemplate.version(), indexTemplate.metadata());
}
indexTemplateService.putIndexTemplateV2(request.cause(), request.create(), request.name(), request.masterNodeTimeout(),
indexTemplate, listener);
}

2
server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
View File

@ -311,7 +311,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
}
} else if (indexRequest.index() != null) {
// the index does not exist yet (and this is a valid request), so match index templates to look for pipelines
List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findTemplates(metadata, indexRequest.index(), null);
List<IndexTemplateMetadata> templates = MetadataIndexTemplateService.findV1Templates(metadata, indexRequest.index(), null);
assert (templates != null);
// order of templates are highest order first
for (final IndexTemplateMetadata template : templates) {

5
server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateV2.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -114,11 +115,15 @@ public class IndexTemplateV2 extends AbstractDiffable<IndexTemplateV2> implement
return indexPatterns;
}
@Nullable
public Template template() {
return template;
}
public List<String> composedOf() {
if (componentTemplates == null) {
return Collections.emptyList();
}
return componentTemplates;
}

278
server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
View File

@ -92,6 +92,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@ -309,22 +310,97 @@ public class MetadataCreateIndexService {
final Index recoverFromIndex = request.recoverFrom();
final IndexMetadata sourceMetadata = recoverFromIndex == null ? null : currentState.metadata().getIndexSafe(recoverFromIndex);
// we only find a template when its an API call (a new index)
// find templates, highest order are better matching
final Boolean isHiddenFromRequest = IndexMetadata.INDEX_HIDDEN_SETTING.exists(request.settings()) ?
IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings()) : null;
final List<IndexTemplateMetadata> templates = sourceMetadata == null ?
Collections.unmodifiableList(MetadataIndexTemplateService.findTemplates(currentState.metadata(),
request.index(),
isHiddenFromRequest)) :
Collections.emptyList();
if (sourceMetadata != null) {
// If source metadata was provided, it means we're recovering from an existing index,
// in which case templates don't apply, so create the index from the source metadata
return applyCreateIndexRequestWithExistingMetadata(currentState, request, silent, sourceMetadata);
} else {
// Hidden indices apply templates slightly differently (ignoring wildcard '*'
// templates), so we need to check to see if the request is creating a hidden index
// prior to resolving which templates it matches
final Boolean isHiddenFromRequest = IndexMetadata.INDEX_HIDDEN_SETTING.exists(request.settings()) ?
IndexMetadata.INDEX_HIDDEN_SETTING.get(request.settings()) : null;
final Map<String, Map<String, Object>> mappings = Collections.unmodifiableMap(parseMappings(request.mappings(), templates,
xContentRegistry));
// Check to see if a v2 template matched
final String v2Template = MetadataIndexTemplateService.findV2Template(currentState.metadata(),
request.index(), isHiddenFromRequest);
final Settings aggregatedIndexSettings =
aggregateIndexSettings(currentState, request, templates, mappings, sourceMetadata, settings, indexScopedSettings);
int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata);
if (v2Template != null) {
// If a v2 template was found, it takes precedence over all v1 templates, so create
// the index using that template and the request's specified settings
return applyCreateIndexRequestWithV2Template(currentState, request, silent, v2Template);
} else {
// A v2 template wasn't found, check the v1 templates, in the event no templates are
// found creation still works using the request's specified index settings
final List<IndexTemplateMetadata> v1Templates = MetadataIndexTemplateService.findV1Templates(currentState.metadata(),
request.index(), isHiddenFromRequest);
return applyCreateIndexRequestWithV1Templates(currentState, request, silent, v1Templates);
}
}
}
/**
* Given the state and a request as well as the metadata necessary to build a new index,
* validate the configuration with an actual index service as return a new cluster state with
* the index added (and rerouted)
* @param currentState the current state to base the new state off of
* @param request the create index request
* @param silent a boolean for whether logging should be at a lower or higher level
* @param sourceMetadata when recovering from an existing index, metadata that should be copied to the new index
* @param temporaryIndexMeta metadata for the new index built from templates, source metadata, and request settings
* @param mappings a map of mappings for the new index
* @param aliasSupplier a function that takes the real {@link IndexService} and returns a list of {@link AliasMetadata} aliases
* @param templatesApplied a list of the names of the templates applied, for logging
* @return a new cluster state with the index added
*/
private ClusterState applyCreateIndexWithTemporaryService(final ClusterState currentState,
final CreateIndexClusterStateUpdateRequest request,
final boolean silent,
final IndexMetadata sourceMetadata,
final IndexMetadata temporaryIndexMeta,
final Map<String, Map<String, Object>> mappings,
final Function<IndexService, List<AliasMetadata>> aliasSupplier,
final List<String> templatesApplied) throws Exception {
// create the index here (on the master) to validate it can be created, as well as adding the mapping
return indicesService.<ClusterState, Exception>withTempIndexService(temporaryIndexMeta, indexService -> {
try {
updateIndexMappingsAndBuildSortOrder(indexService, mappings, sourceMetadata);
} catch (Exception e) {
logger.debug("failed on parsing mappings on index creation [{}]", request.index());
throw e;
}
final List<AliasMetadata> aliases = aliasSupplier.apply(indexService);
final IndexMetadata indexMetadata;
try {
indexMetadata = buildIndexMetadata(request.index(), aliases, indexService.mapperService()::documentMapper,
() -> indexService.mapperService().documentMapper(MapperService.DEFAULT_MAPPING), temporaryIndexMeta.getSettings(),
temporaryIndexMeta.getRoutingNumShards(), sourceMetadata);
} catch (Exception e) {
logger.info("failed to build index metadata [{}]", request.index());
throw e;
}
logger.log(silent ? Level.DEBUG : Level.INFO, "[{}] creating index, cause [{}], templates {}, shards [{}]/[{}], mappings {}",
request.index(), request.cause(), templatesApplied, indexMetadata.getNumberOfShards(),
indexMetadata.getNumberOfReplicas(), mappings.keySet());
indexService.getIndexEventListener().beforeIndexAddedToCluster(indexMetadata.getIndex(),
indexMetadata.getSettings());
return clusterStateCreateIndex(currentState, request.blocks(), indexMetadata, allocationService::reroute);
});
}
/**
* Given a state and index settings calculated after applying templates, validate metadata for
* the new index, returning an {@link IndexMetadata} for the new index
*/
private IndexMetadata buildAndValidateTemporaryIndexMetadata(final ClusterState currentState,
final Settings aggregatedIndexSettings,
final CreateIndexClusterStateUpdateRequest request,
final int routingNumShards) {
final boolean isHiddenAfterTemplates = IndexMetadata.INDEX_HIDDEN_SETTING.get(aggregatedIndexSettings);
validateDotIndex(request.index(), currentState, isHiddenAfterTemplates);
@ -339,42 +415,101 @@ public class MetadataCreateIndexService {
tmpImdBuilder.settings(indexSettings);
// Set up everything, now locally create the index to see that things are ok, and apply
IndexMetadata tmpImd = tmpImdBuilder.build();
validateActiveShardCount(request.waitForActiveShards(), tmpImd);
// create the index here (on the master) to validate it can be created, as well as adding the mapping
return indicesService.<ClusterState, Exception>withTempIndexService(tmpImd, indexService -> {
try {
updateIndexMappingsAndBuildSortOrder(indexService, mappings, sourceMetadata);
} catch (Exception e) {
logger.debug("failed on parsing mappings on index creation [{}]", request.index());
throw e;
}
IndexMetadata tempMetadata = tmpImdBuilder.build();
validateActiveShardCount(request.waitForActiveShards(), tempMetadata);
// the context is only used for validation so it's fine to pass fake values for the shard id and the current
// timestamp
final List<AliasMetadata> aliases = Collections.unmodifiableList(
resolveAndValidateAliases(request.index(), request.aliases(), templates, currentState.metadata(), aliasValidator,
xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null))
);
return tempMetadata;
}
final IndexMetadata indexMetadata;
try {
MapperService mapperService = indexService.mapperService();
indexMetadata = buildIndexMetadata(request.index(), aliases, mapperService::documentMapper,
() -> mapperService.documentMapper(MapperService.DEFAULT_MAPPING), indexSettings, routingNumShards, sourceMetadata);
} catch (Exception e) {
logger.info("failed to build index metadata [{}]", request.index());
throw e;
}
private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState currentState,
final CreateIndexClusterStateUpdateRequest request,
final boolean silent,
final List<IndexTemplateMetadata> templates) throws Exception {
logger.info("applying create index request using v1 templates {}", templates);
logger.log(silent ? Level.DEBUG : Level.INFO, "[{}] creating index, cause [{}], templates {}, shards [{}]/[{}], mappings {}",
request.index(), request.cause(), templates.stream().map(IndexTemplateMetadata::getName).collect(toList()),
indexMetadata.getNumberOfShards(), indexMetadata.getNumberOfReplicas(), mappings.keySet());
final Map<String, Map<String, Object>> mappings = Collections.unmodifiableMap(parseMappings(request.mappings(),
templates.stream().map(IndexTemplateMetadata::getMappings)
// Converts the ImmutableOpenMap into a non-terrible HashMap
.map(iom -> {
Map<String, CompressedXContent> converted = new HashMap<>(iom.size());
for (ObjectObjectCursor<String, CompressedXContent> cursor : iom) {
converted.put(cursor.key, cursor.value);
}
return converted;
})
.collect(toList()),
xContentRegistry));
indexService.getIndexEventListener().beforeIndexAddedToCluster(indexMetadata.getIndex(),
indexMetadata.getSettings());
return clusterStateCreateIndex(currentState, request.blocks(), indexMetadata, allocationService::reroute);
});
final Settings aggregatedIndexSettings =
aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(templates), mappings,
null, settings, indexScopedSettings);
int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards);
return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings,
indexService -> resolveAndValidateAliases(request.index(), request.aliases(),
MetadataIndexTemplateService.resolveAliases(templates), currentState.metadata(), aliasValidator,
// the context is only used for validation so it's fine to pass fake values for the
// shard id and the current timestamp
xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)),
templates.stream().map(IndexTemplateMetadata::getName).collect(toList()));
}
private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState currentState,
final CreateIndexClusterStateUpdateRequest request,
final boolean silent,
final String templateName) throws Exception {
logger.info("applying create index request using v2 template [{}]", templateName);
final Map<String, Map<String, Object>> mappings = Collections.unmodifiableMap(parseMappings(request.mappings(),
MetadataIndexTemplateService.resolveMappings(currentState, templateName).stream()
.map(compressedMapping -> Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, compressedMapping))
.collect(toList()),
xContentRegistry));
final Settings aggregatedIndexSettings =
aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(currentState, templateName),
mappings, null, settings, indexScopedSettings);
int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards);
return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings,
indexService -> resolveAndValidateAliases(request.index(), request.aliases(),
MetadataIndexTemplateService.resolveAliases(currentState, templateName), currentState.metadata(), aliasValidator,
// the context is only used for validation so it's fine to pass fake values for the
// shard id and the current timestamp
xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)),
Collections.singletonList(templateName));
}
private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterState currentState,
final CreateIndexClusterStateUpdateRequest request,
final boolean silent,
final IndexMetadata sourceMetadata) throws Exception {
logger.info("applying create index request using existing index [{}] metadata", sourceMetadata.getIndex().getName());
final Map<String, Map<String, Object>> mappings;
if (request.mappings().size() == 0) {
mappings = Collections.emptyMap();
} else {
assert request.mappings().size() == 1 : "expected source metadata mappings to have 1 type but it had: " + request.mappings();
String sourceMappings = request.mappings().values().iterator().next();
mappings = Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME,
Collections.unmodifiableMap(MapperService.parseMapping(xContentRegistry, sourceMappings)));
}
final Settings aggregatedIndexSettings =
aggregateIndexSettings(currentState, request, Settings.EMPTY, mappings, sourceMetadata, settings, indexScopedSettings);
final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata);
IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards);
return applyCreateIndexWithTemporaryService(currentState, request, silent, sourceMetadata, tmpImd, mappings,
indexService -> resolveAndValidateAliases(request.index(), request.aliases(), Collections.emptyList(),
currentState.metadata(), aliasValidator, xContentRegistry,
// the context is only used for validation so it's fine to pass fake values for the
// shard id and the current timestamp
indexService.newQueryShardContext(0, null, () -> 0L, null)),
Collections.emptyList());
}
/**
@ -383,48 +518,54 @@ public class MetadataCreateIndexService {
* The template mappings are applied in the order they are encountered in the list (clients should make sure the lower index, closer
* to the head of the list, templates have the highest {@link IndexTemplateMetadata#order()})
*/
static Map<String, Map<String, Object>> parseMappings(Map<String, String> requestMappings, List<IndexTemplateMetadata> templates,
static Map<String, Map<String, Object>> parseMappings(Map<String, String> requestMappings,
List<Map<String, CompressedXContent>> templateMappings,
NamedXContentRegistry xContentRegistry) throws Exception {
Map<String, Map<String, Object>> mappings = new HashMap<>();
for (Map.Entry<String, String> entry : requestMappings.entrySet()) {
Map<String, Object> mapping = MapperService.parseMapping(xContentRegistry, entry.getValue());
if (mapping.isEmpty()) {
// Someone provided an empty '{}' for mappings, which is okay, but to avoid
// tripping the below assertion, we can safely ignore it
continue;
}
assert mapping.size() == 1 : mapping;
assert entry.getKey().equals(mapping.keySet().iterator().next()) : entry.getKey() + " != " + mapping;
mappings.put(entry.getKey(), mapping);
}
// apply templates, merging the mappings into the request mapping if exists
for (IndexTemplateMetadata template : templates) {
for (ObjectObjectCursor<String, CompressedXContent> cursor : template.mappings()) {
String mappingString = cursor.value.string();
if (mappings.containsKey(cursor.key)) {
XContentHelper.mergeDefaults(mappings.get(cursor.key),
for (Map<String, CompressedXContent> tMapping : templateMappings) {
for (Map.Entry<String, CompressedXContent> cursor : tMapping.entrySet()) {
String mappingString = cursor.getValue().string();
String type = cursor.getKey();
if (mappings.containsKey(type)) {
XContentHelper.mergeDefaults(mappings.get(type),
MapperService.parseMapping(xContentRegistry, mappingString));
} else if (mappings.size() == 1 && cursor.key.equals(MapperService.SINGLE_MAPPING_NAME)) {
} else if (mappings.size() == 1 && type.equals(MapperService.SINGLE_MAPPING_NAME)) {
// Typeless template with typed mapping
Map<String, Object> templateMapping = MapperService.parseMapping(xContentRegistry, mappingString);
assert templateMapping.size() == 1 : templateMapping;
assert cursor.key.equals(templateMapping.keySet().iterator().next()) :
cursor.key + " != " + templateMapping;
assert type.equals(templateMapping.keySet().iterator().next()) :
type + " != " + templateMapping;
Map.Entry<String, Map<String, Object>> mappingEntry = mappings.entrySet().iterator().next();
templateMapping = Collections.singletonMap(
mappingEntry.getKey(), // reuse type name from the mapping
templateMapping.values().iterator().next()); // but actual mappings from the template
XContentHelper.mergeDefaults(mappingEntry.getValue(), templateMapping);
} else if (template.mappings().size() == 1 && mappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) {
} else if (tMapping.size() == 1 && mappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) {
// Typed template with typeless mapping
Map<String, Object> templateMapping = MapperService.parseMapping(xContentRegistry, mappingString);
assert templateMapping.size() == 1 : templateMapping;
assert cursor.key.equals(templateMapping.keySet().iterator().next()) :
cursor.key + " != " + templateMapping;
assert type.equals(templateMapping.keySet().iterator().next()) :
type + " != " + templateMapping;
Map<String, Object> mapping = mappings.get(MapperService.SINGLE_MAPPING_NAME);
templateMapping = Collections.singletonMap(
MapperService.SINGLE_MAPPING_NAME, // make template mapping typeless
templateMapping.values().iterator().next());
XContentHelper.mergeDefaults(mapping, templateMapping);
} else {
mappings.put(cursor.key,
MapperService.parseMapping(xContentRegistry, mappingString));
mappings.put(type, MapperService.parseMapping(xContentRegistry, mappingString));
}
}
}
@ -442,15 +583,12 @@ public class MetadataCreateIndexService {
* @return the aggregated settings for the new index
*/
static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClusterStateUpdateRequest request,
List<IndexTemplateMetadata> templates, Map<String, Map<String, Object>> mappings,
Settings templateSettings, Map<String, Map<String, Object>> mappings,
@Nullable IndexMetadata sourceMetadata, Settings settings,
IndexScopedSettings indexScopedSettings) {
Settings.Builder indexSettingsBuilder = Settings.builder();
if (sourceMetadata == null) {
// apply templates, here, in reverse order, since first ones are better matching
for (int i = templates.size() - 1; i >= 0; i--) {
indexSettingsBuilder.put(templates.get(i).settings());
}
indexSettingsBuilder.put(templateSettings);
}
// now, put the request settings, so they override templates
indexSettingsBuilder.put(request.settings());
@ -553,7 +691,7 @@ public class MetadataCreateIndexService {
* @return the list of resolved aliases, with the explicitly provided aliases occurring first (having a higher priority) followed by
* the ones inherited from the templates
*/
static List<AliasMetadata> resolveAndValidateAliases(String index, Set<Alias> aliases, List<IndexTemplateMetadata> templates,
static List<AliasMetadata> resolveAndValidateAliases(String index, Set<Alias> aliases, List<Map<String, AliasMetadata>> templateAliases,
Metadata metadata, AliasValidator aliasValidator,
NamedXContentRegistry xContentRegistry, QueryShardContext queryShardContext) {
List<AliasMetadata> resolvedAliases = new ArrayList<>();
@ -569,17 +707,17 @@ public class MetadataCreateIndexService {
}
Map<String, AliasMetadata> templatesAliases = new HashMap<>();
for (IndexTemplateMetadata template : templates) {
for (Map<String, AliasMetadata> templateAliasConfig : templateAliases) {
// handle aliases
for (ObjectObjectCursor<String, AliasMetadata> cursor : template.aliases()) {
AliasMetadata aliasMetadata = cursor.value;
for (Map.Entry<String, AliasMetadata> entry : templateAliasConfig.entrySet()) {
AliasMetadata aliasMetadata = entry.getValue();
// if an alias with same name came with the create index request itself,
// ignore this one taken from the index template
if (aliases.contains(new Alias(aliasMetadata.alias()))) {
continue;
}
// if an alias with same name was already processed, ignore this one
if (templatesAliases.containsKey(cursor.key)) {
if (templatesAliases.containsKey(entry.getKey())) {
continue;
}

248
server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
View File

@ -46,6 +46,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.MapperParsingException;
@ -63,6 +64,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
@ -182,12 +184,35 @@ public class MetadataIndexTemplateService {
}
CompressedXContent mappings = template.template().mappings();
validateTemplate(template.template().settings(), Collections.singletonMap("_doc", mappings == null ? null : mappings.string()),
indicesService, xContentRegistry);
String stringMappings = mappings == null ? null : mappings.string();
// We may need to normalize index settings, so do that also
Settings finalSettings = template.template().settings();
if (finalSettings != null) {
finalSettings = Settings.builder()
.put(finalSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX)
.build();
}
validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry);
// Mappings in component templates don't include _doc, so update the mappings to include this single type
if (stringMappings != null) {
Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
if (parsedMappings.size() > 0) {
stringMappings = Strings.toString(XContentFactory.jsonBuilder()
.startObject()
.field(MapperService.SINGLE_MAPPING_NAME, parsedMappings)
.endObject());
}
}
final Template finalTemplate = new Template(finalSettings,
stringMappings == null ? null : new CompressedXContent(stringMappings), template.template().aliases());
final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata());
logger.info("adding component template [{}]", name);
return ClusterState.builder(currentState)
.metadata(Metadata.builder(currentState.metadata()).put(name, template))
.metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate))
.build();
}
@ -262,7 +287,7 @@ public class MetadataIndexTemplateService {
}
@Override
public ClusterState execute(ClusterState currentState) {
public ClusterState execute(ClusterState currentState) throws Exception {
return addIndexTemplateV2(currentState, create, name, template);
}
@ -274,8 +299,8 @@ public class MetadataIndexTemplateService {
}
// Package visible for testing
static ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean create,
final String name, final IndexTemplateV2 template) {
ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean create,
final String name, final IndexTemplateV2 template) throws Exception {
if (create && currentState.metadata().templatesV2().containsKey(name)) {
throw new IllegalArgumentException("index template [" + name + "] already exists");
}
@ -295,12 +320,41 @@ public class MetadataIndexTemplateService {
deprecationLogger.deprecated(warning);
}
// TODO: validation of index template
// validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry);
IndexTemplateV2 finalIndexTemplate = template;
Template innerTemplate = template.template();
if (innerTemplate != null) {
// We may need to normalize index settings, so do that also
Settings finalSettings = innerTemplate.settings();
if (finalSettings != null) {
finalSettings = Settings.builder()
.put(finalSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX)
.build();
}
// If an inner template was specified, its mappings may need to be
// adjusted (to add _doc) and it should be validated
CompressedXContent mappings = innerTemplate.mappings();
String stringMappings = mappings == null ? null : mappings.string();
validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry);
// Mappings in index templates don't include _doc, so update the mappings to include this single type
if (stringMappings != null) {
Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
if (parsedMappings.size() > 0) {
stringMappings = Strings.toString(XContentFactory.jsonBuilder()
.startObject()
.field(MapperService.SINGLE_MAPPING_NAME, parsedMappings)
.endObject());
}
}
final Template finalTemplate = new Template(finalSettings,
stringMappings == null ? null : new CompressedXContent(stringMappings), innerTemplate.aliases());
finalIndexTemplate = new IndexTemplateV2(template.indexPatterns(), finalTemplate, template.composedOf(),
template.priority(), template.version(), template.metadata());
}
logger.info("adding index template [{}]", name);
return ClusterState.builder(currentState)
.metadata(Metadata.builder(currentState.metadata()).put(name, template))
.metadata(Metadata.builder(currentState.metadata()).put(name, finalIndexTemplate))
.build();
}
@ -532,7 +586,7 @@ public class MetadataIndexTemplateService {
* @return a list of templates sorted by {@link IndexTemplateMetadata#order()} descending.
*
*/
public static List<IndexTemplateMetadata> findTemplates(Metadata metadata, String indexName, @Nullable Boolean isHidden) {
public static List<IndexTemplateMetadata> findV1Templates(Metadata metadata, String indexName, @Nullable Boolean isHidden) {
final Predicate<String> patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, indexName);
final List<IndexTemplateMetadata> matchedTemplates = new ArrayList<>();
for (ObjectCursor<IndexTemplateMetadata> cursor : metadata.templates().values()) {
@ -575,19 +629,179 @@ public class MetadataIndexTemplateService {
}
}
}
return matchedTemplates;
return Collections.unmodifiableList(matchedTemplates);
}
private static void validateTemplate(Settings settings, Map<String, String> mappings,
/**
* Return the name (id) of the highest matching index template for the given index name. In
* the event that no templates are matched, {@code null} is returned.
*/
@Nullable
public static String findV2Template(Metadata metadata, String indexName, @Nullable Boolean isHidden) {
final Predicate<String> patternMatchPredicate = pattern -> Regex.simpleMatch(pattern, indexName);
final Map<IndexTemplateV2, String> matchedTemplates = new HashMap<>();
for (Map.Entry<String, IndexTemplateV2> entry : metadata.templatesV2().entrySet()) {
final String name = entry.getKey();
final IndexTemplateV2 template = entry.getValue();
if (isHidden == null || isHidden == Boolean.FALSE) {
final boolean matched = template.indexPatterns().stream().anyMatch(patternMatchPredicate);
if (matched) {
matchedTemplates.put(template, name);
}
} else {
assert isHidden == Boolean.TRUE;
final boolean isNotMatchAllTemplate = template.indexPatterns().stream().noneMatch(Regex::isMatchAllPattern);
if (isNotMatchAllTemplate) {
if (template.indexPatterns().stream().anyMatch(patternMatchPredicate)) {
matchedTemplates.put(template, name);
}
}
}
}
if (matchedTemplates.size() == 0) {
return null;
}
final List<IndexTemplateV2> candidates = new ArrayList<>(matchedTemplates.keySet());
CollectionUtil.timSort(candidates, Comparator.comparingLong(IndexTemplateV2::priority).reversed());
assert candidates.size() > 0 : "we should have returned early with no candidates";
IndexTemplateV2 winner = candidates.get(0);
return matchedTemplates.get(winner);
}
/**
* Resolve the given v2 template into an ordered list of mappings
*/
public static List<CompressedXContent> resolveMappings(final ClusterState state, final String templateName) {
final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName);
assert template != null : "attempted to resolve mappings for a template [" + templateName +
"] that did not exist in the cluster state";
if (template == null) {
return Collections.emptyList();
}
final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates();
// TODO: more fine-grained merging of component template mappings, ie, merge fields as distint entities
List<CompressedXContent> mappings = template.composedOf().stream()
.map(componentTemplates::get)
.filter(Objects::nonNull)
.map(ComponentTemplate::template)
.map(Template::mappings)
.filter(Objects::nonNull)
.collect(Collectors.toList());
// Add the actual index template's mappings, since it takes the highest precedence
Optional.ofNullable(template.template())
.map(Template::mappings)
.ifPresent(mappings::add);
// When actually merging mappings, the highest precedence ones should go first, so reverse the list
Collections.reverse(mappings);
return Collections.unmodifiableList(mappings);
}
/**
* Resolve index settings for the given list of v1 templates, templates are apply in reverse
* order since they should be provided in order of priority/order
*/
public static Settings resolveSettings(final List<IndexTemplateMetadata> templates) {
Settings.Builder templateSettings = Settings.builder();
// apply templates, here, in reverse order, since first ones are better matching
for (int i = templates.size() - 1; i >= 0; i--) {
templateSettings.put(templates.get(i).settings());
}
return templateSettings.build();
}
/**
* Resolve the given v2 template into a collected {@link Settings} object
*/
public static Settings resolveSettings(final ClusterState state, final String templateName) {
final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName);
assert template != null : "attempted to resolve settings for a template [" + templateName +
"] that did not exist in the cluster state";
if (template == null) {
return Settings.EMPTY;
}
final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates();
List<Settings> componentSettings = template.composedOf().stream()
.map(componentTemplates::get)
.filter(Objects::nonNull)
.map(ComponentTemplate::template)
.map(Template::settings)
.filter(Objects::nonNull)
.collect(Collectors.toList());
Settings.Builder templateSettings = Settings.builder();
componentSettings.forEach(templateSettings::put);
// Add the actual index template's settings to the end, since it takes the highest precedence.
Optional.ofNullable(template.template())
.map(Template::settings)
.ifPresent(templateSettings::put);
return templateSettings.build();
}
/**
* Resolve the given v1 templates into an ordered list of aliases
*/
public static List<Map<String, AliasMetadata>> resolveAliases(final List<IndexTemplateMetadata> templates) {
final List<Map<String, AliasMetadata>> resolvedAliases = new ArrayList<>();
templates.forEach(template -> {
if (template.aliases() != null) {
Map<String, AliasMetadata> aliasMeta = new HashMap<>();
for (ObjectObjectCursor<String, AliasMetadata> cursor : template.aliases()) {
aliasMeta.put(cursor.key, cursor.value);
}
resolvedAliases.add(aliasMeta);
}
});
return Collections.unmodifiableList(resolvedAliases);
}
/**
* Resolve the given v2 template into an ordered list of aliases
*/
public static List<Map<String, AliasMetadata>> resolveAliases(final ClusterState state, final String templateName) {
final IndexTemplateV2 template = state.metadata().templatesV2().get(templateName);
assert template != null : "attempted to resolve aliases for a template [" + templateName +
"] that did not exist in the cluster state";
if (template == null) {
return Collections.emptyList();
}
final Map<String, ComponentTemplate> componentTemplates = state.metadata().componentTemplates();
List<Map<String, AliasMetadata>> aliases = template.composedOf().stream()
.map(componentTemplates::get)
.filter(Objects::nonNull)
.map(ComponentTemplate::template)
.map(Template::aliases)
.filter(Objects::nonNull)
.collect(Collectors.toList());
// Add the actual index template's aliases to the end if they exist
Optional.ofNullable(template.template())
.map(Template::aliases)
.ifPresent(aliases::add);
// Aliases are applied in order, but subsequent alias configuration from the same name is
// ignored, so in order for the order to be correct, alias configuration should be in order
// of precedence (with the index template first)
Collections.reverse(aliases);
return Collections.unmodifiableList(aliases);
}
private static void validateTemplate(Settings validateSettings, String mappings,
IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
validateTemplate(validateSettings, Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, mappings),
indicesService, xContentRegistry);
}
private static void validateTemplate(Settings validateSettings, Map<String, String> mappings,
IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
// First check to see if mappings are valid XContent
Map<String, Map<String, Object>> mappingsForValidation = new HashMap<>();
if (mappings != null) {
IndexTemplateMetadata.Builder templateBuilder = IndexTemplateMetadata.builder("validation");
for (Map.Entry<String, String> entry : mappings.entrySet()) {
if (entry.getValue() != null) {
try {
templateBuilder.putMapping(entry.getKey(), entry.getValue());
new CompressedXContent(entry.getValue());
mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue()));
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
@ -596,6 +810,12 @@ public class MetadataIndexTemplateService {
}
}
// Hard to validate settings if they're non-existent, so used empty ones if none were provided
Settings settings = validateSettings;
if (settings == null) {
settings = Settings.EMPTY;
}
Index createdIndex = null;
final String temporaryIndexName = UUIDs.randomBase64UUID();
try {

12
server/src/main/java/org/elasticsearch/cluster/metadata/Template.java
View File

@ -35,6 +35,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.mapper.MapperService;
import java.io.IOException;
import java.util.HashMap;
@ -171,7 +172,7 @@ public class Template extends AbstractDiffable<Template> implements ToXContentOb
XContentHelper.convertToMap(new BytesArray(this.mappings.uncompressed()), true, XContentType.JSON).v2();
if (uncompressedMapping.size() > 0) {
builder.field(MAPPINGS.getPreferredName());
builder.map(uncompressedMapping);
builder.map(reduceMapping(uncompressedMapping));
}
}
if (this.aliases != null) {
@ -184,4 +185,13 @@ public class Template extends AbstractDiffable<Template> implements ToXContentOb
builder.endObject();
return builder;
}
@SuppressWarnings("unchecked")
private static Map<String, Object> reduceMapping(Map<String, Object> mapping) {
if (mapping.size() == 1 && MapperService.SINGLE_MAPPING_NAME.equals(mapping.keySet().iterator().next())) {
return (Map<String, Object>) mapping.values().iterator().next();
} else {
return mapping;
}
}
}

9
server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
View File

@ -98,7 +98,7 @@ public class ComponentTemplateTests extends AbstractDiffableSerializationTestCas
private static CompressedXContent randomMappings() {
try {
return new CompressedXContent("{\"" + randomAlphaOfLength(3) + "\":\"" + randomAlphaOfLength(7) + "\"}");
return new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}");
} catch (IOException e) {
fail("got an IO exception creating fake mappings: " + e);
return null;
@ -107,7 +107,12 @@ public class ComponentTemplateTests extends AbstractDiffableSerializationTestCas
private static Settings randomSettings() {
return Settings.builder()
.put(randomAlphaOfLength(4), randomAlphaOfLength(10))
.put(IndexMetadata.SETTING_BLOCKS_READ, randomBoolean())
.put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean())
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 5))
.put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean())
.put(IndexMetadata.SETTING_PRIORITY, randomIntBetween(0, 100000))
.build();
}

9
server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateV2Tests.java
View File

@ -110,7 +110,7 @@ public class IndexTemplateV2Tests extends AbstractDiffableSerializationTestCase<
private static CompressedXContent randomMappings() {
try {
return new CompressedXContent("{\"" + randomAlphaOfLength(3) + "\":\"" + randomAlphaOfLength(7) + "\"}");
return new CompressedXContent("{\"properties\":{\"" + randomAlphaOfLength(5) + "\":{\"type\":\"keyword\"}}}");
} catch (IOException e) {
fail("got an IO exception creating fake mappings: " + e);
return null;
@ -119,7 +119,12 @@ public class IndexTemplateV2Tests extends AbstractDiffableSerializationTestCase<
private static Settings randomSettings() {
return Settings.builder()
.put(randomAlphaOfLength(4), randomAlphaOfLength(10))
.put(IndexMetadata.SETTING_BLOCKS_READ, randomBoolean())
.put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean())
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10))
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 5))
.put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean())
.put(IndexMetadata.SETTING_PRIORITY, randomIntBetween(0, 100000))
.build();
}

50
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.cluster.metadata;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.Version;
@ -73,6 +74,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
@ -645,8 +647,8 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
});
request.mappings(singletonMap("type", createMapping("mapping_from_request", "text").string()));
Map<String, Map<String, Object>> parsedMappings = parseMappings(request.mappings(), singletonList(templateMetadata),
NamedXContentRegistry.EMPTY);
Map<String, Map<String, Object>> parsedMappings = MetadataCreateIndexService.parseMappings(request.mappings(),
Collections.singletonList(convertMappings(templateMetadata.getMappings())), NamedXContentRegistry.EMPTY);
assertThat(parsedMappings, hasKey("type"));
Map<String, Object> mappingType = parsedMappings.get("type");
@ -671,7 +673,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
.build();
request.settings(Settings.builder().put("request_setting", "value2").build());
Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, singletonList(templateMetadata), emptyMap(),
Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateMetadata.settings(), emptyMap(),
null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("value1"));
@ -703,11 +705,12 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
request.aliases(singleton(new Alias("alias").searchRouting("fromRequest")));
request.settings(Settings.builder().put("key1", "requestValue").build());
Map<String, Map<String, Object>> parsedMappings = parseMappings(request.mappings(), singletonList(templateMetadata),
xContentRegistry());
Map<String, Map<String, Object>> parsedMappings = MetadataCreateIndexService.parseMappings(request.mappings(),
Collections.singletonList(convertMappings(templateMetadata.mappings())), xContentRegistry());
List<AliasMetadata> resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(),
singletonList(templateMetadata), Metadata.builder().build(), aliasValidator, xContentRegistry(), queryShardContext);
Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, singletonList(templateMetadata),
MetadataIndexTemplateService.resolveAliases(Collections.singletonList(templateMetadata)),
Metadata.builder().build(), aliasValidator, xContentRegistry(), queryShardContext);
Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, templateMetadata.settings(),
emptyMap(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
assertThat(resolvedAliases.get(0).getSearchRouting(), equalTo("fromRequest"));
@ -723,14 +726,14 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
}
public void testDefaultSettings() {
Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, emptyList(), emptyMap(),
Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, emptyMap(),
null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("1"));
}
public void testSettingsFromClusterState() {
Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, emptyList(), emptyMap(),
Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, emptyMap(),
null, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 15).build(), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("15"));
@ -753,9 +756,11 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
.settings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 10))
.putAlias(AliasMetadata.builder("alias1").searchRouting("1").build())
));
Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, templates, emptyMap(),
Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request,
MetadataIndexTemplateService.resolveSettings(templates), emptyMap(),
null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
List<AliasMetadata> resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(), templates,
List<AliasMetadata> resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(),
MetadataIndexTemplateService.resolveAliases(templates),
Metadata.builder().build(), aliasValidator, xContentRegistry(), queryShardContext);
assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("12"));
AliasMetadata alias = resolvedAliases.get(0);
@ -779,7 +784,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
createClusterState("sourceIndex", 1, 0,
Settings.builder().put("index.blocks.write", true).build());
Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, singletonList(templateMetadata), emptyMap(),
Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateMetadata.settings(), emptyMap(),
clusterState.metadata().index("sourceIndex"), Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
assertThat(aggregatedIndexSettings.get("templateSetting"), is(nullValue()));
@ -842,7 +847,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
});
Map<String, Map<String, Object>> mappings = parseMappings(singletonMap(MapperService.SINGLE_MAPPING_NAME, "{\"_doc\":{}}"),
singletonList(templateMetadata), xContentRegistry());
Collections.singletonList(convertMappings(templateMetadata.mappings())), xContentRegistry());
assertThat(mappings, Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME));
}
@ -855,7 +860,8 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
ExceptionsHelper.reThrowIfNotNull(e);
}
});
Map<String, Map<String, Object>> mappings = parseMappings(emptyMap(), singletonList(templateMetadata), xContentRegistry());
Map<String, Map<String, Object>> mappings = parseMappings(emptyMap(),
Collections.singletonList(convertMappings(templateMetadata.mappings())), xContentRegistry());
assertThat(mappings, Matchers.hasKey("type"));
}
@ -867,7 +873,8 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
ExceptionsHelper.reThrowIfNotNull(e);
}
});
Map<String, Map<String, Object>> mappings = parseMappings(emptyMap(), singletonList(templateMetadata), xContentRegistry());
Map<String, Map<String, Object>> mappings = parseMappings(emptyMap(),
Collections.singletonList(convertMappings(templateMetadata.mappings())), xContentRegistry());
assertThat(mappings, Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME));
}
@ -935,7 +942,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
public void testSoftDeletesDisabledDeprecation() {
request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build());
aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Collections.emptyList(), Collections.emptyMap(),
aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Collections.emptyMap(),
null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
assertWarnings("Creating indices with soft-deletes disabled is deprecated and will be removed in future Elasticsearch versions. "
+ "Please do not specify value for setting [index.soft_deletes.enabled] of index [test].");
@ -943,7 +950,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
if (randomBoolean()) {
request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true).build());
}
aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Collections.emptyList(), Collections.emptyMap(),
aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Collections.emptyMap(),
null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
}
@ -956,7 +963,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), between(1, 128) + "mb");
}
request.settings(settings.build());
aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Collections.emptyList(), Collections.emptyMap(),
aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, Collections.emptyMap(),
null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
assertWarnings("Translog retention settings [index.translog.retention.age] "
+ "and [index.translog.retention.size] are deprecated and effectively ignored. They will be removed in a future version.");
@ -993,4 +1000,11 @@ public class MetadataCreateIndexServiceTests extends ESTestCase {
}
}
private static Map<String, CompressedXContent> convertMappings(ImmutableOpenMap<String, CompressedXContent> mappings) {
Map<String, CompressedXContent> converted = new HashMap<>(mappings.size());
for (ObjectObjectCursor<String, CompressedXContent> cursor : mappings) {
converted.put(cursor.key, cursor.value);
}
return converted;
}
}

290
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
View File

@ -27,21 +27,28 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.indices.IndexTemplateMissingException;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.InvalidIndexTemplateException;
import org.elasticsearch.test.ESSingleNodeTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.stream.Collectors;
@ -192,11 +199,11 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
putTemplateDetail(new PutRequest("test", "foo-2").patterns(singletonList("foo-*")).order(2));
putTemplateDetail(new PutRequest("test", "bar").patterns(singletonList("bar-*")).order(between(0, 100)));
final ClusterState state = client().admin().cluster().prepareState().get().getState();
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "foo-1234", randomBoolean()).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "foo-1234", randomBoolean()).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("foo-2", "foo-1"));
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "bar-xyz", randomBoolean()).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "bar-xyz", randomBoolean()).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("bar"));
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "baz", randomBoolean()), empty());
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "baz", randomBoolean()), empty());
}
public void testFindTemplatesWithHiddenIndices() throws Exception {
@ -211,32 +218,32 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
final ClusterState state = client().admin().cluster().prepareState().get().getState();
// hidden
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "foo-1234", true).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "foo-1234", true).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("foo-2", "foo-1"));
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "bar-xyz", true).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "bar-xyz", true).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("bar"));
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "baz", true), empty());
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "sneaky1", true).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "baz", true), empty());
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "sneaky1", true).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("sneaky-hidden"));
// not hidden
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "foo-1234", false).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "foo-1234", false).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("foo-2", "foo-1", "global"));
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "bar-xyz", false).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "bar-xyz", false).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("bar", "global"));
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "baz", false).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "baz", false).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("global"));
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "sneaky1", false).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "sneaky1", false).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("global", "sneaky-hidden"));
// unknown
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "foo-1234", null).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "foo-1234", null).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("foo-2", "foo-1", "global"));
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "bar-xyz", null).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "bar-xyz", null).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), containsInAnyOrder("bar", "global"));
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "baz", null).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "baz", null).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("global"));
assertThat(MetadataIndexTemplateService.findTemplates(state.metadata(), "sneaky1", null).stream()
assertThat(MetadataIndexTemplateService.findV1Templates(state.metadata(), "sneaky1", null).stream()
.map(IndexTemplateMetadata::name).collect(Collectors.toList()), contains("sneaky-hidden"));
}
@ -284,32 +291,34 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
() -> metadataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate4));
}
public void testAddIndexTemplateV2() {
public void testAddIndexTemplateV2() throws Exception {
ClusterState state = ClusterState.EMPTY_STATE;
final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
IndexTemplateV2 template = IndexTemplateV2Tests.randomInstance();
state = MetadataIndexTemplateService.addIndexTemplateV2(state, false, "foo", template);
state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo", template);
assertNotNull(state.metadata().templatesV2().get("foo"));
assertThat(state.metadata().templatesV2().get("foo"), equalTo(template));
assertTemplatesEqual(state.metadata().templatesV2().get("foo"), template);
final ClusterState throwState = ClusterState.builder(state).build();
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> MetadataIndexTemplateService.addIndexTemplateV2(throwState, true, "foo", template));
() -> metadataIndexTemplateService.addIndexTemplateV2(throwState, true, "foo", template));
assertThat(e.getMessage(), containsString("index template [foo] already exists"));
state = MetadataIndexTemplateService.addIndexTemplateV2(state, randomBoolean(), "bar", template);
state = metadataIndexTemplateService.addIndexTemplateV2(state, randomBoolean(), "bar", template);
assertNotNull(state.metadata().templatesV2().get("bar"));
}
public void testRemoveIndexTemplateV2() {
public void testRemoveIndexTemplateV2() throws Exception {
IndexTemplateV2 template = IndexTemplateV2Tests.randomInstance();
final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
IndexTemplateMissingException e = expectThrows(IndexTemplateMissingException.class,
() -> MetadataIndexTemplateService.innerRemoveIndexTemplateV2(ClusterState.EMPTY_STATE, "foo"));
assertThat(e.getMessage(), equalTo("index_template [foo] missing"));
final ClusterState state = MetadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template);
final ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template);
assertNotNull(state.metadata().templatesV2().get("foo"));
assertThat(state.metadata().templatesV2().get("foo"), equalTo(template));
assertTemplatesEqual(state.metadata().templatesV2().get("foo"), template);
ClusterState updatedState = MetadataIndexTemplateService.innerRemoveIndexTemplateV2(state, "foo");
assertNull(updatedState.metadata().templatesV2().get("foo"));
@ -318,10 +327,11 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
/**
* Test that if we have a pre-existing v1 template and put a v2 template that would match the same indices, we generate a warning
*/
public void testPuttingV2TemplateGeneratesWarning() {
public void testPuttingV2TemplateGeneratesWarning() throws Exception {
IndexTemplateMetadata v1Template = IndexTemplateMetadata.builder("v1-template")
.patterns(Arrays.asList("fo*", "baz"))
.build();
final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
ClusterState state = ClusterState.builder(ClusterState.EMPTY_STATE)
.metadata(Metadata.builder(Metadata.EMPTY_METADATA)
@ -330,7 +340,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
.build();
IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null);
state = MetadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template);
state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template);
assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " +
"from existing older templates [v1-template] with patterns (v1-template => [fo*, baz]); this template [v2-template] will " +
@ -343,9 +353,10 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
/**
* Test that if we have a pre-existing v2 template and put a "*" v1 template, we generate a warning
*/
public void testPuttingV1StarTemplateGeneratesWarning() {
public void testPuttingV1StarTemplateGeneratesWarning() throws Exception {
final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null);
ClusterState state = MetadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template);
ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template);
MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template");
req.patterns(Arrays.asList("*", "baz"));
@ -362,9 +373,10 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
/**
* Test that if we have a pre-existing v2 template and put a v1 template that would match the same indices, we generate a hard error
*/
public void testPuttingV1NonStarTemplateGeneratesError() {
public void testPuttingV1NonStarTemplateGeneratesError() throws Exception {
final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null);
ClusterState state = MetadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template);
ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template);
MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template");
req.patterns(Arrays.asList("egg*", "baz"));
@ -383,7 +395,9 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
* Test that if we have a pre-existing v1 and v2 template, and we update the existing v1
* template without changing its index patterns, a warning is generated
*/
public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() {
public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() throws Exception {
final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
IndexTemplateMetadata v1Template = IndexTemplateMetadata.builder("v1-template")
.patterns(Arrays.asList("fo*", "baz"))
.build();
@ -395,7 +409,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
.build();
IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null);
state = MetadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template);
state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template);
assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " +
"from existing older templates [v1-template] with patterns (v1-template => [fo*, baz]); this template [v2-template] will " +
@ -422,7 +436,8 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
* Test that if we have a pre-existing v1 and v2 template, and we update the existing v1
* template *AND* change the index patterns that an error is generated
*/
public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() {
public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() throws Exception {
final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
IndexTemplateMetadata v1Template = IndexTemplateMetadata.builder("v1-template")
.patterns(Arrays.asList("fo*", "baz"))
.build();
@ -434,7 +449,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
.build();
IndexTemplateV2 v2Template = new IndexTemplateV2(Arrays.asList("foo-bar-*", "eggplant"), null, null, null, null, null);
state = MetadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template);
state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template);
assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " +
"from existing older templates [v1-template] with patterns (v1-template => [fo*, baz]); this template [v2-template] will " +
@ -456,6 +471,164 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
"templates (/_index_template) instead"));
}
public void testFindV2Templates() throws Exception {
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
ClusterState state = ClusterState.EMPTY_STATE;
assertNull(MetadataIndexTemplateService.findV2Template(state.metadata(), "index", randomBoolean() ? null : randomBoolean()));
ComponentTemplate ct = ComponentTemplateTests.randomInstance();
state = service.addComponentTemplate(state, true, "ct", ct);
IndexTemplateV2 it = new IndexTemplateV2(Collections.singletonList("i*"), null, Collections.singletonList("ct"), 0L, 1L, null);
state = service.addIndexTemplateV2(state, true, "my-template", it);
IndexTemplateV2 it2 = new IndexTemplateV2(Collections.singletonList("in*"), null, Collections.singletonList("ct"), 10L, 2L, null);
state = service.addIndexTemplateV2(state, true, "my-template2", it2);
String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", randomBoolean() ? null : randomBoolean());
assertThat(result, equalTo("my-template2"));
}
public void testFindV2TemplatesForHiddenIndex() throws Exception {
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
ClusterState state = ClusterState.EMPTY_STATE;
assertNull(MetadataIndexTemplateService.findV2Template(state.metadata(), "index", true));
ComponentTemplate ct = ComponentTemplateTests.randomInstance();
state = service.addComponentTemplate(state, true, "ct", ct);
IndexTemplateV2 it = new IndexTemplateV2(Collections.singletonList("i*"), null, Collections.singletonList("ct"), 0L, 1L, null);
state = service.addIndexTemplateV2(state, true, "my-template", it);
IndexTemplateV2 it2 = new IndexTemplateV2(Collections.singletonList("*"), null, Collections.singletonList("ct"), 10L, 2L, null);
state = service.addIndexTemplateV2(state, true, "my-template2", it2);
String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", true);
assertThat(result, equalTo("my-template"));
}
public void testResolveMappings() throws Exception {
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
ClusterState state = ClusterState.EMPTY_STATE;
ComponentTemplate ct1 = new ComponentTemplate(new Template(null,
new CompressedXContent("{\n" +
" \"properties\": {\n" +
" \"field2\": {\n" +
" \"type\": \"keyword\"\n" +
" }\n" +
" }\n" +
" }"), null), null, null);
ComponentTemplate ct2 = new ComponentTemplate(new Template(null,
new CompressedXContent("{\n" +
" \"properties\": {\n" +
" \"field2\": {\n" +
" \"type\": \"text\"\n" +
" }\n" +
" }\n" +
" }"), null), null, null);
state = service.addComponentTemplate(state, true, "ct_high", ct1);
state = service.addComponentTemplate(state, true, "ct_low", ct2);
IndexTemplateV2 it = new IndexTemplateV2(Collections.singletonList("i*"),
new Template(null,
new CompressedXContent("{\n" +
" \"properties\": {\n" +
" \"field\": {\n" +
" \"type\": \"keyword\"\n" +
" }\n" +
" }\n" +
" }"), null),
Arrays.asList("ct_low", "ct_high"), 0L, 1L, null);
state = service.addIndexTemplateV2(state, true, "my-template", it);
List<CompressedXContent> mappings = MetadataIndexTemplateService.resolveMappings(state, "my-template");
assertNotNull(mappings);
assertThat(mappings.size(), equalTo(3));
List<Map<String, Object>> parsedMappings = mappings.stream()
.map(m -> {
try {
return MapperService.parseMapping(new NamedXContentRegistry(Collections.emptyList()), m.string());
} catch (Exception e) {
logger.error(e);
fail("failed to parse mappings: " + m.string());
return null;
}
})
.collect(Collectors.toList());
// The order of mappings should be:
// - index template
// - ct_high
// - ct_low
// Because the first elements when merging mappings have the highest precedence
assertThat(parsedMappings.get(0),
equalTo(Collections.singletonMap("_doc", Collections.singletonMap("properties",
Collections.singletonMap("field", Collections.singletonMap("type", "keyword"))))));
assertThat(parsedMappings.get(1),
equalTo(Collections.singletonMap("_doc", Collections.singletonMap("properties",
Collections.singletonMap("field2", Collections.singletonMap("type", "keyword"))))));
assertThat(parsedMappings.get(2),
equalTo(Collections.singletonMap("_doc", Collections.singletonMap("properties",
Collections.singletonMap("field2", Collections.singletonMap("type", "text"))))));
}
public void testResolveSettings() throws Exception {
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
ClusterState state = ClusterState.EMPTY_STATE;
ComponentTemplate ct1 = new ComponentTemplate(new Template(Settings.builder()
.put("number_of_replicas", 2)
.put("index.blocks.write", true)
.build(),
null, null), null, null);
ComponentTemplate ct2 = new ComponentTemplate(new Template(Settings.builder()
.put("index.number_of_replicas", 1)
.put("index.blocks.read", true)
.build(),
null, null), null, null);
state = service.addComponentTemplate(state, true, "ct_high", ct1);
state = service.addComponentTemplate(state, true, "ct_low", ct2);
IndexTemplateV2 it = new IndexTemplateV2(Collections.singletonList("i*"),
new Template(Settings.builder()
.put("index.blocks.write", false)
.put("index.number_of_shards", 3)
.build(), null, null),
Arrays.asList("ct_low", "ct_high"), 0L, 1L, null);
state = service.addIndexTemplateV2(state, true, "my-template", it);
Settings settings = MetadataIndexTemplateService.resolveSettings(state, "my-template");
assertThat(settings.get("index.number_of_replicas"), equalTo("2"));
assertThat(settings.get("index.blocks.write"), equalTo("false"));
assertThat(settings.get("index.blocks.read"), equalTo("true"));
assertThat(settings.get("index.number_of_shards"), equalTo("3"));
}
public void testResolveAliases() throws Exception {
final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
ClusterState state = ClusterState.EMPTY_STATE;
Map<String, AliasMetadata> a1 = new HashMap<>();
a1.put("foo", AliasMetadata.newAliasMetadataBuilder("foo").build());
Map<String, AliasMetadata> a2 = new HashMap<>();
a2.put("bar", AliasMetadata.newAliasMetadataBuilder("bar").build());
Map<String, AliasMetadata> a3 = new HashMap<>();
a3.put("eggplant", AliasMetadata.newAliasMetadataBuilder("eggplant").build());
a3.put("baz", AliasMetadata.newAliasMetadataBuilder("baz").build());
ComponentTemplate ct1 = new ComponentTemplate(new Template(null, null, a1), null, null);
ComponentTemplate ct2 = new ComponentTemplate(new Template(null, null, a2), null, null);
state = service.addComponentTemplate(state, true, "ct_high", ct1);
state = service.addComponentTemplate(state, true, "ct_low", ct2);
IndexTemplateV2 it = new IndexTemplateV2(Collections.singletonList("i*"),
new Template(null, null, a3),
Arrays.asList("ct_low", "ct_high"), 0L, 1L, null);
state = service.addIndexTemplateV2(state, true, "my-template", it);
List<Map<String, AliasMetadata>> resolvedAliases = MetadataIndexTemplateService.resolveAliases(state, "my-template");
// These should be order of precedence, so the index template (a3), then ct_high (a1), then ct_low (a2)
assertThat(resolvedAliases, equalTo(Arrays.asList(a3, a1, a2)));
}
private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(
Settings.EMPTY,
@ -528,4 +701,55 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
clusterService, createIndexService, new AliasValidator(), indicesService,
new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry());
}
@SuppressWarnings("unchecked")
public static void assertTemplatesEqual(IndexTemplateV2 actual, IndexTemplateV2 expected) {
IndexTemplateV2 actualNoTemplate = new IndexTemplateV2(actual.indexPatterns(), null,
actual.composedOf(), actual.priority(), actual.version(), actual.metadata());
IndexTemplateV2 expectedNoTemplate = new IndexTemplateV2(expected.indexPatterns(), null,
expected.composedOf(), expected.priority(), expected.version(), expected.metadata());
assertThat(actualNoTemplate, equalTo(expectedNoTemplate));
Template actualTemplate = actual.template();
Template expectedTemplate = expected.template();
assertThat("expected both templates to have either a template or no template",
Objects.nonNull(actualTemplate), equalTo(Objects.nonNull(expectedTemplate)));
if (actualTemplate != null) {
assertThat(actualTemplate.settings(), equalTo(expectedTemplate.settings()));
assertThat(actualTemplate.aliases(), equalTo(expectedTemplate.aliases()));
assertThat("expected both templates to have either mappings or no mappings",
Objects.nonNull(actualTemplate.mappings()), equalTo(Objects.nonNull(expectedTemplate.mappings())));
if (actualTemplate.mappings() != null) {
Map<String, Object> actualMappings;
Map<String, Object> expectedMappings;
try (XContentParser parser = XContentType.JSON.xContent()
.createParser(new NamedXContentRegistry(Collections.emptyList()), LoggingDeprecationHandler.INSTANCE,
actualTemplate.mappings().string())) {
actualMappings = parser.map();
} catch (IOException e) {
throw new AssertionError(e);
}
try (XContentParser parser = XContentType.JSON.xContent()
.createParser(new NamedXContentRegistry(Collections.emptyList()), LoggingDeprecationHandler.INSTANCE,
expectedTemplate.mappings().string())) {
expectedMappings = parser.map();
} catch (IOException e) {
throw new AssertionError(e);
}
if (actualMappings.size() == 1 && actualMappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) {
actualMappings = (Map<String, Object>) actualMappings.get(MapperService.SINGLE_MAPPING_NAME);
}
if (expectedMappings.size() == 1 && expectedMappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) {
expectedMappings = (Map<String, Object>) expectedMappings.get(MapperService.SINGLE_MAPPING_NAME);
}
assertThat(actualMappings, equalTo(expectedMappings));
}
}
}
}