honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x] Add validation for component templates (#54023) (#54118) 

* Add validation for component templates (#54023)

* Add validation for component templates

This change adds validation to make sure that settings and mappings are correct in
component template. It's done the same way as in index templates - code is reused.

Reletes to #53101

* Fix checkstyle violation

* Update server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java

Co-Authored-By: Lee Hinman <dakrone@users.noreply.github.com>

* Update server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java

Co-Authored-By: Lee Hinman <dakrone@users.noreply.github.com>

* Update server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java

Co-Authored-By: Lee Hinman <dakrone@users.noreply.github.com>

* Update server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java

Co-Authored-By: Lee Hinman <dakrone@users.noreply.github.com>

* Update server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java

Co-Authored-By: Lee Hinman <dakrone@users.noreply.github.com>

Co-authored-by: Lee Hinman <dakrone@users.noreply.github.com>

* Adjusted to 7.7

* unused import fixed

* npe fixeD

* change exception type

Co-authored-by: Lee Hinman <dakrone@users.noreply.github.com>
This commit is contained in:
Przemko Robakowski 2020-03-24 22:20:34 +01:00 committed by GitHub
parent 025857d949
commit fc498f625a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 94 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
@ -44,14 +45,17 @@ public class TransportPutComponentTemplateAction
extends TransportMasterNodeAction<PutComponentTemplateAction.Request, AcknowledgedResponse> {
private final MetaDataIndexTemplateService indexTemplateService;
private final IndexScopedSettings indexScopedSettings;
@Inject
public TransportPutComponentTemplateAction(TransportService transportService, ClusterService clusterService,
ThreadPool threadPool, MetaDataIndexTemplateService indexTemplateService,
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
IndexScopedSettings indexScopedSettings) {
super(PutComponentTemplateAction.NAME, transportService, clusterService, threadPool, actionFilters,
PutComponentTemplateAction.Request::new, indexNameExpressionResolver);
this.indexTemplateService = indexTemplateService;
this.indexScopedSettings = indexScopedSettings;
}
@Override
@ -77,8 +81,10 @@ public class TransportPutComponentTemplateAction
Template template = componentTemplate.template();
// Normalize the index settings if necessary
if (template.settings() != null) {
Settings.Builder settings = Settings.builder().put(template.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
template = new Template(settings.build(), template.mappings(), template.aliases());
Settings.Builder builder = Settings.builder().put(template.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
Settings settings = builder.build();
indexScopedSettings.validate(settings, true);
template = new Template(settings, template.mappings(), template.aliases());
componentTemplate = new ComponentTemplate(template, componentTemplate.version(), componentTemplate.metadata());
}
indexTemplateService.putComponentTemplate(request.cause(), request.create(), request.name(), request.masterNodeTimeout(),

61
server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
View File

@ -35,12 +35,15 @@ import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.ValidationException;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.MapperParsingException;
@ -156,7 +159,7 @@ public class MetaDataIndexTemplateService {
}
@Override
public ClusterState execute(ClusterState currentState) {
public ClusterState execute(ClusterState currentState) throws Exception {
return addComponentTemplate(currentState, create, name, template);
}
@ -168,14 +171,18 @@ public class MetaDataIndexTemplateService {
}
// Package visible for testing
static ClusterState addComponentTemplate(final ClusterState currentState, final boolean create,
final String name, final ComponentTemplate template) {
ClusterState addComponentTemplate(final ClusterState currentState, final boolean create,
final String name, final ComponentTemplate template) throws Exception {
if (create && currentState.metaData().componentTemplates().containsKey(name)) {
throw new IllegalArgumentException("component template [" + name + "] already exists");
}
// TODO: validation of component template
// validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry);
CompressedXContent mappings = template.template().mappings();
Map<String, Object> mappingsArray = Collections.emptyMap();
if(mappings != null) {
mappingsArray = XContentHelper.convertToMap(XContentType.JSON.xContent(), mappings.string(), true);
}
validateTemplate(template.template().settings(), Collections.singletonMap("_doc", mappingsArray), indicesService);
logger.info("adding component template [{}]", name);
return ClusterState.builder(currentState)
@ -276,7 +283,22 @@ public class MetaDataIndexTemplateService {
throw new IllegalArgumentException("index_template [" + request.name + "] already exists");
}
validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry);
templateBuilder.order(request.order);
templateBuilder.version(request.version);
templateBuilder.patterns(request.indexPatterns);
templateBuilder.settings(request.settings);
Map<String, Map<String, Object>> mappingsForValidation = new HashMap<>();
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
try {
templateBuilder.putMapping(entry.getKey(), entry.getValue());
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
}
mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue()));
}
validateTemplate(request.settings, mappingsForValidation, indicesService);
for (Alias alias : request.aliases) {
AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
@ -357,20 +379,20 @@ public class MetaDataIndexTemplateService {
return matchedTemplates;
}
private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder,
IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
private static void validateTemplate(Settings settings, Map<String, Map<String, Object>> mappings,
IndicesService indicesService) throws Exception {
Index createdIndex = null;
final String temporaryIndexName = UUIDs.randomBase64UUID();
try {
// use the provided values, otherwise just pick valid dummy values
int dummyPartitionSize = IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(request.settings);
int dummyShards = request.settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS,
int dummyPartitionSize = IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(settings);
int dummyShards = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS,
dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1);
//create index service for parsing and validating "mappings"
Settings dummySettings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(request.settings)
.put(settings)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, dummyShards)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
@ -380,22 +402,7 @@ public class MetaDataIndexTemplateService {
IndexService dummyIndexService = indicesService.createIndex(tmpIndexMetadata, Collections.emptyList(), false);
createdIndex = dummyIndexService.index();
templateBuilder.order(request.order);
templateBuilder.version(request.version);
templateBuilder.patterns(request.indexPatterns);
templateBuilder.settings(request.settings);
Map<String, Map<String, Object>> mappingsForValidation = new HashMap<>();
for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
try {
templateBuilder.putMapping(entry.getKey(), entry.getValue());
} catch (Exception e) {
throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
}
mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue()));
}
dummyIndexService.mapperService().merge(mappingsForValidation, MergeReason.MAPPING_UPDATE);
dummyIndexService.mapperService().merge(mappings, MergeReason.MAPPING_UPDATE);
} finally {
if (createdIndex != null) {

2
server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java
View File

@ -85,7 +85,7 @@ public class ComponentTemplateTests extends AbstractDiffableSerializationTestCas
return new ComponentTemplate(template, randomBoolean() ? null : randomNonNegativeLong(), meta);
}
private static Map<String, AliasMetaData> randomAliases() {
public static Map<String, AliasMetaData> randomAliases() {
String aliasName = randomAlphaOfLength(5);
AliasMetaData aliasMeta = AliasMetaData.builder(aliasName)
.filter(Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)))

73
server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateServiceTests.java
View File

@ -19,11 +19,13 @@
package org.elasticsearch.cluster.metadata;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.PutRequest;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@ -37,6 +39,7 @@ import org.elasticsearch.test.ESSingleNodeTestCase;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@ -197,21 +200,41 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
assertThat(errors.get(0).getMessage(), containsString("global templates may not specify the setting index.hidden"));
}
public void testAddComponentTemplate() {
public void testAddComponentTemplate() throws Exception{
MetaDataIndexTemplateService metaDataIndexTemplateService = getMetaDataIndexTemplateService();
ClusterState state = ClusterState.EMPTY_STATE;
ComponentTemplate template = ComponentTemplateTests.randomInstance();
state = MetaDataIndexTemplateService.addComponentTemplate(state, false, "foo", template);
Template template = new Template(Settings.builder().build(), null, ComponentTemplateTests.randomAliases());
ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>());
state = metaDataIndexTemplateService.addComponentTemplate(state, false, "foo", componentTemplate);
assertNotNull(state.metaData().componentTemplates().get("foo"));
assertThat(state.metaData().componentTemplates().get("foo"), equalTo(template));
assertThat(state.metaData().componentTemplates().get("foo"), equalTo(componentTemplate));
final ClusterState throwState = ClusterState.builder(state).build();
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> MetaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo", template));
() -> metaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo", componentTemplate));
assertThat(e.getMessage(), containsString("component template [foo] already exists"));
state = MetaDataIndexTemplateService.addComponentTemplate(state, randomBoolean(), "bar", template);
state = metaDataIndexTemplateService.addComponentTemplate(state, randomBoolean(), "bar", componentTemplate);
assertNotNull(state.metaData().componentTemplates().get("bar"));
template = new Template(Settings.builder().build(), new CompressedXContent("{\"invalid\"}"),
ComponentTemplateTests.randomAliases());
ComponentTemplate componentTemplate2 = new ComponentTemplate(template, 1L, new HashMap<>());
expectThrows(ElasticsearchParseException.class,
() -> metaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate2));
template = new Template(Settings.builder().build(), new CompressedXContent("{\"invalid\":\"invalid\"}"),
ComponentTemplateTests.randomAliases());
ComponentTemplate componentTemplate3 = new ComponentTemplate(template, 1L, new HashMap<>());
expectThrows(MapperParsingException.class,
() -> metaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate3));
template = new Template(Settings.builder().put("invalid", "invalid").build(), new CompressedXContent("{}"),
ComponentTemplateTests.randomAliases());
ComponentTemplate componentTemplate4 = new ComponentTemplate(template, 1L, new HashMap<>());
expectThrows(IllegalArgumentException.class,
() -> metaDataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate4));
}
private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
@ -247,23 +270,7 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
}
private List<Throwable> putTemplateDetail(PutRequest request) throws Exception {
IndicesService indicesService = getInstanceFromNode(IndicesService.class);
ClusterService clusterService = getInstanceFromNode(ClusterService.class);
MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
Settings.EMPTY,
clusterService,
indicesService,
null,
null,
new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null),
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
null,
xContentRegistry(),
Collections.emptyList(),
true);
MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(
clusterService, createIndexService, new AliasValidator(), indicesService,
new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry());
MetaDataIndexTemplateService service = getMetaDataIndexTemplateService();
final List<Throwable> throwables = new ArrayList<>();
final CountDownLatch latch = new CountDownLatch(1);
@ -282,4 +289,24 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
latch.await();
return throwables;
}
private MetaDataIndexTemplateService getMetaDataIndexTemplateService() {
IndicesService indicesService = getInstanceFromNode(IndicesService.class);
ClusterService clusterService = getInstanceFromNode(ClusterService.class);
MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
Settings.EMPTY,
clusterService,
indicesService,
null,
null,
new Environment(builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(), null),
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
null,
xContentRegistry(),
Collections.emptyList(),
true);
return new MetaDataIndexTemplateService(
clusterService, createIndexService, new AliasValidator(), indicesService,
new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry());
}
}