[7.x] Validate global V2 templates don't set index.hidden (#55010) (#55519)

Validate adding global V2 templates don't configure the index.hidden setting.
This also prevents updating the component template to add the index.hidden
setting if that component template is referenced by a global index template.

(cherry picked from commit 2e768981809887649f49d265d039f056985f7e6a)
Signed-off-by: Andrei Dan <andrei.dan@elastic.co>

server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateV2Action.java

@@ -25,11 +25,13 @@ import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexTemplateV2;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.regex.Regex;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -87,7 +89,17 @@ public class PutIndexTemplateV2Action extends ActionType<AcknowledgedResponse> {
             }
             if (indexTemplate == null) {
                 validationException = addValidationError("an index template is required", validationException);
+            } else {
+                if (indexTemplate.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) {
+                    if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(indexTemplate.template().settings())) {
+                        validationException = addValidationError("global V2 templates may not specify the setting "
+                                + IndexMetadata.INDEX_HIDDEN_SETTING.getKey(),
+                            validationException
+                        );
+                    }
+                }
             }
+
             return validationException;
         }
 

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -178,7 +178,7 @@ public class MetadataIndexTemplateService {
 
     // Package visible for testing
     ClusterState addComponentTemplate(final ClusterState currentState, final boolean create,
-                                             final String name, final ComponentTemplate template) throws Exception {
+                                      final String name, final ComponentTemplate template) throws Exception {
         if (create && currentState.metadata().componentTemplates().containsKey(name)) {
             throw new IllegalArgumentException("component template [" + name + "] already exists");
         }
@@ -196,6 +196,29 @@ public class MetadataIndexTemplateService {
 
         validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry);
 
+        // if we're updating a component template, let's check if it's part of any V2 template that will yield the CT update invalid
+        if (create == false && finalSettings != null) {
+            // if the CT is specifying the `index.hidden` setting it cannot be part of any global template
+            if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(finalSettings)) {
+                Map<String, IndexTemplateV2> existingTemplates = currentState.metadata().templatesV2();
+                List<String> globalTemplatesThatUseThisComponent = new ArrayList<>();
+                for (Map.Entry<String, IndexTemplateV2> entry : existingTemplates.entrySet()) {
+                    IndexTemplateV2 templateV2 = entry.getValue();
+                    if (templateV2.composedOf().contains(name) && templateV2.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) {
+                        // global templates don't support configuring the `index.hidden` setting so we don't need to resolve the settings as
+                        // no other component template can remove this setting from the resolved settings, so just invalidate this update
+                        globalTemplatesThatUseThisComponent.add(entry.getKey());
+                    }
+                }
+                if (globalTemplatesThatUseThisComponent.isEmpty() == false) {
+                    throw new IllegalArgumentException("cannot update component template ["
+                        + name + "] because the following global templates would resolve to specifying the ["
+                        + IndexMetadata.SETTING_INDEX_HIDDEN + "] setting: [" + String.join(",", globalTemplatesThatUseThisComponent)
+                        + "]");
+                }
+            }
+        }
+
         // Mappings in component templates don't include _doc, so update the mappings to include this single type
         if (stringMappings != null) {
             Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
@@ -273,6 +296,7 @@ public class MetadataIndexTemplateService {
      */
     public void putIndexTemplateV2(final String cause, final boolean create, final String name, final TimeValue masterTimeout,
                                    final IndexTemplateV2 template, final ActionListener<AcknowledgedResponse> listener) {
+        validateV2TemplateRequest(clusterService.state().metadata(), name, template);
         clusterService.submitStateUpdateTask("create-index-template-v2 [" + name + "], cause [" + cause + "]",
             new ClusterStateUpdateTask(Priority.URGENT) {
 
@@ -298,6 +322,16 @@ public class MetadataIndexTemplateService {
             });
     }
 
+    static void validateV2TemplateRequest(Metadata metadata, String name, IndexTemplateV2 template) {
+        if (template.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) {
+            Settings mergedSettings = resolveSettings(metadata, template);
+            if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(mergedSettings)) {
+                throw new InvalidIndexTemplateException(name, "global V2 templates may not specify the setting "
+                    + IndexMetadata.INDEX_HIDDEN_SETTING.getKey());
+            }
+        }
+    }
+
     // Package visible for testing
     ClusterState addIndexTemplateV2(final ClusterState currentState, final boolean create,
                                     final String name, final IndexTemplateV2 template) throws Exception {
@@ -761,6 +795,10 @@ public class MetadataIndexTemplateService {
         if (template == null) {
             return Settings.EMPTY;
         }
+        return resolveSettings(metadata, template);
+    }
+
+    private static Settings resolveSettings(Metadata metadata, IndexTemplateV2 template) {
         final Map<String, ComponentTemplate> componentTemplates = metadata.componentTemplates();
         List<Settings> componentSettings = template.composedOf().stream()
             .map(componentTemplates::get)

server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateV2RequestTests.java

@@ -19,11 +19,20 @@
 
 package org.elasticsearch.action.admin.indices.template.put;
 
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexTemplateV2;
 import org.elasticsearch.cluster.metadata.IndexTemplateV2Tests;
+import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
 import java.io.IOException;
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class PutIndexTemplateV2RequestTests extends AbstractWireSerializingTestCase<PutIndexTemplateV2Action.Request> {
     @Override
@@ -44,4 +53,31 @@ public class PutIndexTemplateV2RequestTests extends AbstractWireSerializingTestC
     protected PutIndexTemplateV2Action.Request mutateInstance(PutIndexTemplateV2Action.Request instance) throws IOException {
         return randomValueOtherThan(instance, this::createTestInstance);
     }
+
+    public void testPutGlobalTemplatesCannotHaveHiddenIndexSetting() {
+        Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null);
+        IndexTemplateV2 globalTemplate = new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), template, null, null, null,
+            null);
+
+        PutIndexTemplateV2Action.Request request = new PutIndexTemplateV2Action.Request("test");
+        request.indexTemplate(globalTemplate);
+
+        ActionRequestValidationException validationException = request.validate();
+        assertThat(validationException, is(notNullValue()));
+        List<String> validationErrors = validationException.validationErrors();
+        assertThat(validationErrors.size(), is(1));
+        String error = validationErrors.get(0);
+        assertThat(error, is("global V2 templates may not specify the setting " + IndexMetadata.SETTING_INDEX_HIDDEN));
+    }
+
+    public void testPutIndexTemplateV2RequestMustContainTemplate() {
+        PutIndexTemplateV2Action.Request requestWithoutTemplate = new PutIndexTemplateV2Action.Request("test");
+
+        ActionRequestValidationException validationException = requestWithoutTemplate.validate();
+        assertThat(validationException, is(notNullValue()));
+        List<String> validationErrors = validationException.validationErrors();
+        assertThat(validationErrors.size(), is(1));
+        String error = validationErrors.get(0);
+        assertThat(error, is("an index template is required"));
+    }
 }

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -19,7 +19,9 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.PutRequest;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -27,6 +29,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -51,13 +54,16 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.settings.Settings.builder;
 import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.containsStringIgnoringCase;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
@@ -129,6 +135,7 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
         assertThat(throwables.get(0), instanceOf(IllegalArgumentException.class));
         assertThat(throwables.get(0).getMessage(), containsString("[2]: cannot be greater than number of shard copies [1]"));
     }
+
     public void testIndexTemplateValidationAccumulatesValidationErrors() {
         PutRequest request = new PutRequest("test", "putTemplate shards");
         request.patterns(singletonList("_test_shards*"));
@@ -291,6 +298,44 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
             () -> metadataIndexTemplateService.addComponentTemplate(throwState, true, "foo2", componentTemplate4));
     }
 
+    public void testUpdateComponentTemplateWithIndexHiddenSetting() throws Exception {
+        MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
+        ClusterState state = ClusterState.EMPTY_STATE;
+        Template template = new Template(Settings.builder().build(), null, ComponentTemplateTests.randomAliases());
+        ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>());
+        state = metadataIndexTemplateService.addComponentTemplate(state, true, "foo", componentTemplate);
+        assertNotNull(state.metadata().componentTemplates().get("foo"));
+
+        IndexTemplateV2 firstGlobalIndexTemplate =
+            new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), template, org.elasticsearch.common.collect.List.of("foo"),
+                1L, null, null);
+        state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalIndexTemplate1", firstGlobalIndexTemplate);
+
+        IndexTemplateV2 secondGlobalIndexTemplate =
+            new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), template, org.elasticsearch.common.collect.List.of("foo"),
+                2L, null, null);
+        state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalIndexTemplate2", secondGlobalIndexTemplate);
+
+        IndexTemplateV2 fooPatternIndexTemplate =
+            new IndexTemplateV2(org.elasticsearch.common.collect.List.of("foo-*"), template, org.elasticsearch.common.collect.List.of(
+                "foo"), 3L, null, null);
+        state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "fooPatternIndexTemplate", fooPatternIndexTemplate);
+
+        // update the component template to set the index.hidden setting
+        Template templateWithIndexHiddenSetting = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(),
+            null, null);
+        ComponentTemplate updatedComponentTemplate = new ComponentTemplate(templateWithIndexHiddenSetting, 2L, new HashMap<>());
+        try {
+            metadataIndexTemplateService.addComponentTemplate(state, false, "foo", updatedComponentTemplate);
+            fail("expecting an exception as updating the component template would yield the global templates to include the index.hidden " +
+                "setting");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsStringIgnoringCase("globalIndexTemplate1"));
+            assertThat(e.getMessage(), containsStringIgnoringCase("globalIndexTemplate2"));
+            assertThat(e.getMessage(), not(containsStringIgnoringCase("fooPatternIndexTemplate")));
+        }
+    }
+
     public void testAddIndexTemplateV2() throws Exception {
         ClusterState state = ClusterState.EMPTY_STATE;
         final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
@@ -354,6 +399,51 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
         assertThat(state.metadata().templatesV2().get("v2-template"), equalTo(v2Template));
     }
 
+    public void testPutGlobalV2TemplateWhichResolvesIndexHiddenSetting() throws Exception {
+        MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
+        Template templateWithIndexHiddenSetting = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(),
+            null, null);
+        ComponentTemplate componentTemplate = new ComponentTemplate(templateWithIndexHiddenSetting, 1L, new HashMap<>());
+
+        CountDownLatch waitToCreateComponentTemplate = new CountDownLatch(1);
+        ActionListener<AcknowledgedResponse> createComponentTemplateListener = new ActionListener<AcknowledgedResponse>() {
+
+            @Override
+            public void onResponse(AcknowledgedResponse response) {
+                waitToCreateComponentTemplate.countDown();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                fail("expecting the component template PUT to succeed but got: " + e.getMessage());
+            }
+        };
+
+        metadataIndexTemplateService.putComponentTemplate("test", true, "ct-with-index-hidden-setting", TimeValue.timeValueSeconds(30L),
+            componentTemplate, createComponentTemplateListener);
+
+        waitToCreateComponentTemplate.await(10, TimeUnit.SECONDS);
+
+        IndexTemplateV2 globalIndexTemplate = new IndexTemplateV2(org.elasticsearch.common.collect.List.of("*"), null,
+            org.elasticsearch.common.collect.List.of("ct-with-index-hidden-setting"), null, null, null);
+
+        expectThrows(InvalidIndexTemplateException.class, () ->
+            metadataIndexTemplateService.putIndexTemplateV2("testing", true, "template-referencing-ct-with-hidden-index-setting",
+                TimeValue.timeValueSeconds(30L), globalIndexTemplate, new ActionListener<AcknowledgedResponse>() {
+                    @Override
+                    public void onResponse(AcknowledgedResponse response) {
+                        fail("the listener should not be invoked as the validation should be executed before any cluster state updates " +
+                            "are issued");
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        fail("the listener should not be invoked as the validation should be executed before any cluster state updates " +
+                            "are issued");
+                    }
+                }));
+    }
+
     /**
      * Test that if we have a pre-existing v2 template and put a "*" v1 template, we generate a warning
      */