[7.x] Validate global V2 templates don't set index.hidden (#55010) (#55519)

Validate adding global V2 templates don't configure the index.hidden setting.
This also prevents updating the component template to add the index.hidden
setting if that component template is referenced by a global index template.

(cherry picked from commit 2e768981809887649f49d265d039f056985f7e6a)
Signed-off-by: Andrei Dan <andrei.dan@elastic.co>
This commit is contained in:
Andrei Dan 2020-04-21 12:47:59 +01:00 committed by GitHub
parent ba39c261e8
commit f13ebc4d4b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 177 additions and 1 deletions

View File

@ -25,11 +25,13 @@ import org.elasticsearch.action.IndicesRequest;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.support.master.MasterNodeRequest;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.regex.Regex;
import java.io.IOException;
import java.util.Objects;
@ -87,7 +89,17 @@ public class PutIndexTemplateV2Action extends ActionType<AcknowledgedResponse> {
}
if (indexTemplate == null) {
validationException = addValidationError("an index template is required", validationException);
} else {
if (indexTemplate.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) {
if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(indexTemplate.template().settings())) {
validationException = addValidationError("global V2 templates may not specify the setting "
+ IndexMetadata.INDEX_HIDDEN_SETTING.getKey(),
validationException
);
}
}
}
return validationException;
}

View File

@ -178,7 +178,7 @@ public class MetadataIndexTemplateService {
// Package visible for testing
ClusterState addComponentTemplate(final ClusterState currentState, final boolean create,
final String name, final ComponentTemplate template) throws Exception {
final String name, final ComponentTemplate template) throws Exception {
if (create && currentState.metadata().componentTemplates().containsKey(name)) {
throw new IllegalArgumentException("component template [" + name + "] already exists");
}
@ -196,6 +196,29 @@ public class MetadataIndexTemplateService {
validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry);
// if we're updating a component template, let's check if it's part of any V2 template that will yield the CT update invalid
if (create == false && finalSettings != null) {
// if the CT is specifying the `index.hidden` setting it cannot be part of any global template
if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(finalSettings)) {
Map<String, IndexTemplateV2> existingTemplates = currentState.metadata().templatesV2();
List<String> globalTemplatesThatUseThisComponent = new ArrayList<>();
for (Map.Entry<String, IndexTemplateV2> entry : existingTemplates.entrySet()) {
IndexTemplateV2 templateV2 = entry.getValue();
if (templateV2.composedOf().contains(name) && templateV2.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) {
// global templates don't support configuring the `index.hidden` setting so we don't need to resolve the settings as
// no other component template can remove this setting from the resolved settings, so just invalidate this update
globalTemplatesThatUseThisComponent.add(entry.getKey());
}
}
if (globalTemplatesThatUseThisComponent.isEmpty() == false) {
throw new IllegalArgumentException("cannot update component template ["
+ name + "] because the following global templates would resolve to specifying the ["
+ IndexMetadata.SETTING_INDEX_HIDDEN + "] setting: [" + String.join(",", globalTemplatesThatUseThisComponent)
+ "]");
}
}
}
// Mappings in component templates don't include _doc, so update the mappings to include this single type
if (stringMappings != null) {
Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
@ -273,6 +296,7 @@ public class MetadataIndexTemplateService {
*/
public void putIndexTemplateV2(final String cause, final boolean create, final String name, final TimeValue masterTimeout,
final IndexTemplateV2 template, final ActionListener<AcknowledgedResponse> listener) {
validateV2TemplateRequest(clusterService.state().metadata(), name, template);
clusterService.submitStateUpdateTask("create-index-template-v2 [" + name + "], cause [" + cause + "]",
new ClusterStateUpdateTask(Priority.URGENT) {
@ -298,6 +322,16 @@ public class MetadataIndexTemplateService {
});
}
static void validateV2TemplateRequest(Metadata metadata, String name, IndexTemplateV2 template) {
if (template.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) {
Settings mergedSettings = resolveSettings(metadata, template);
if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(mergedSettings)) {
throw new InvalidIndexTemplateException(name, "global V2 templates may not specify the setting "
+ IndexMetadata.INDEX_HIDDEN_SETTING.getKey());
}
}
}
// Package visible for testing
ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean create,
final String name, final IndexTemplateV2 template) throws Exception {
@ -761,6 +795,10 @@ public class MetadataIndexTemplateService {
if (template == null) {
return Settings.EMPTY;
}
return resolveSettings(metadata, template);
}
private static Settings resolveSettings(Metadata metadata, IndexTemplateV2 template) {
final Map<String, ComponentTemplate> componentTemplates = metadata.componentTemplates();
List<Settings> componentSettings = template.composedOf().stream()
.map(componentTemplates::get)

View File

@ -19,11 +19,20 @@
package org.elasticsearch.action.admin.indices.template.put;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexTemplateV2;
import org.elasticsearch.cluster.metadata.IndexTemplateV2Tests;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.AbstractWireSerializingTestCase;
import java.io.IOException;
import java.util.List;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
public class PutIndexTemplateV2RequestTests extends AbstractWireSerializingTestCase<PutIndexTemplateV2Action.Request> {
@Override
@ -44,4 +53,31 @@ public class PutIndexTemplateV2RequestTests extends AbstractWireSerializingTestC
protected PutIndexTemplateV2Action.Request mutateInstance(PutIndexTemplateV2Action.Request instance) throws IOException {
return randomValueOtherThan(instance, this::createTestInstance);
}
public void testPutGlobalTemplatesCannotHaveHiddenIndexSetting() {
Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null);
IndexTemplateV2 globalTemplate = new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), template, null, null, null,
null);
PutIndexTemplateV2Action.Request request = new PutIndexTemplateV2Action.Request("test");
request.indexTemplate(globalTemplate);
ActionRequestValidationException validationException = request.validate();
assertThat(validationException, is(notNullValue()));
List<String> validationErrors = validationException.validationErrors();
assertThat(validationErrors.size(), is(1));
String error = validationErrors.get(0);
assertThat(error, is("global V2 templates may not specify the setting " + IndexMetadata.SETTING_INDEX_HIDDEN));
}
public void testPutIndexTemplateV2RequestMustContainTemplate() {
PutIndexTemplateV2Action.Request requestWithoutTemplate = new PutIndexTemplateV2Action.Request("test");
ActionRequestValidationException validationException = requestWithoutTemplate.validate();
assertThat(validationException, is(notNullValue()));
List<String> validationErrors = validationException.validationErrors();
assertThat(validationErrors.size(), is(1));
String error = validationErrors.get(0);
assertThat(error, is("an index template is required"));
}
}

View File

@ -19,7 +19,9 @@
package org.elasticsearch.cluster.metadata;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.PutRequest;
import org.elasticsearch.cluster.service.ClusterService;
@ -27,6 +29,7 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentFactory;
@ -51,13 +54,16 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static java.util.Collections.singletonList;
import static org.elasticsearch.common.settings.Settings.builder;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.containsStringIgnoringCase;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.empty;
@ -129,6 +135,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
assertThat(throwables.get(0), instanceOf(IllegalArgumentException.class));
assertThat(throwables.get(0).getMessage(), containsString("[2]: cannot be greater than number of shard copies [1]"));
}
public void testIndexTemplateValidationAccumulatesValidationErrors() {
PutRequest request = new PutRequest("test", "putTemplate shards");
request.patterns(singletonList("_test_shards*"));
@ -291,6 +298,44 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
() -> metadataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate4));
}
public void testUpdateComponentTemplateWithIndexHiddenSetting() throws Exception {
MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
ClusterState state = ClusterState.EMPTY_STATE;
Template template = new Template(Settings.builder().build(), null, ComponentTemplateTests.randomAliases());
ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>());
state = metadataIndexTemplateService.addComponentTemplate(state, true, "foo", componentTemplate);
assertNotNull(state.metadata().componentTemplates().get("foo"));
IndexTemplateV2 firstGlobalIndexTemplate =
new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), template, org.elasticsearch.common.collect.List.of("foo"),
1L, null, null);
state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalIndexTemplate1", firstGlobalIndexTemplate);
IndexTemplateV2 secondGlobalIndexTemplate =
new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), template, org.elasticsearch.common.collect.List.of("foo"),
2L, null, null);
state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalIndexTemplate2", secondGlobalIndexTemplate);
IndexTemplateV2 fooPatternIndexTemplate =
new IndexTemplateV2(org.elasticsearch.common.collect.List.of("foo-*"), template, org.elasticsearch.common.collect.List.of(
"foo"), 3L, null, null);
state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "fooPatternIndexTemplate", fooPatternIndexTemplate);
// update the component template to set the index.hidden setting
Template templateWithIndexHiddenSetting = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(),
null, null);
ComponentTemplate updatedComponentTemplate = new ComponentTemplate(templateWithIndexHiddenSetting, 2L, new HashMap<>());
try {
metadataIndexTemplateService.addComponentTemplate(state, false, "foo", updatedComponentTemplate);
fail("expecting an exception as updating the component template would yield the global templates to include the index.hidden " +
"setting");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), containsStringIgnoringCase("globalIndexTemplate1"));
assertThat(e.getMessage(), containsStringIgnoringCase("globalIndexTemplate2"));
assertThat(e.getMessage(), not(containsStringIgnoringCase("fooPatternIndexTemplate")));
}
}
public void testAddIndexTemplateV2() throws Exception {
ClusterState state = ClusterState.EMPTY_STATE;
final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
@ -354,6 +399,51 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
assertThat(state.metadata().templatesV2().get("v2-template"), equalTo(v2Template));
}
public void testPutGlobalV2TemplateWhichResolvesIndexHiddenSetting() throws Exception {
MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
Template templateWithIndexHiddenSetting = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(),
null, null);
ComponentTemplate componentTemplate = new ComponentTemplate(templateWithIndexHiddenSetting, 1L, new HashMap<>());
CountDownLatch waitToCreateComponentTemplate = new CountDownLatch(1);
ActionListener<AcknowledgedResponse> createComponentTemplateListener = new ActionListener<AcknowledgedResponse>() {
@Override
public void onResponse(AcknowledgedResponse response) {
waitToCreateComponentTemplate.countDown();
}
@Override
public void onFailure(Exception e) {
fail("expecting the component template PUT to succeed but got: " + e.getMessage());
}
};
metadataIndexTemplateService.putComponentTemplate("test", true, "ct-with-index-hidden-setting", TimeValue.timeValueSeconds(30L),
componentTemplate, createComponentTemplateListener);
waitToCreateComponentTemplate.await(10, TimeUnit.SECONDS);
IndexTemplateV2 globalIndexTemplate = new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), null,
org.elasticsearch.common.collect.List.of("ct-with-index-hidden-setting"), null, null, null);
expectThrows(InvalidIndexTemplateException.class, () ->
metadataIndexTemplateService.putIndexTemplateV2("testing", true, "template-referencing-ct-with-hidden-index-setting",
TimeValue.timeValueSeconds(30L), globalIndexTemplate, new ActionListener<AcknowledgedResponse>() {
@Override
public void onResponse(AcknowledgedResponse response) {
fail("the listener should not be invoked as the validation should be executed before any cluster state updates " +
"are issued");
}
@Override
public void onFailure(Exception e) {
fail("the listener should not be invoked as the validation should be executed before any cluster state updates " +
"are issued");
}
}));
}
/**
* Test that if we have a pre-existing v2 template and put a "*" v1 template, we generate a warning
*/