Transition Transforms to using hidden indices for notifcations index (#53773)

This commit changes the Transforms notifications index to be hidden
index, with a hidden alias.

This commit also removes the temporary hack in
MetaDataCreateIndexService that prevents deprecation warnings for known
dot-prefixed index names which are not hidden/system indices, as this
was the last index pattern to need that hack.

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -23,7 +23,6 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.Version;
@@ -55,7 +54,6 @@ import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -90,7 +88,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -116,14 +113,6 @@ public class MetaDataCreateIndexService {
 
     public static final int MAX_INDEX_NAME_BYTES = 255;
 
-    /**
-     * These index patterns will be converted to hidden indices, at which point they should be removed from this list.
-     */
-    private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton(
-        ".data-frame-notifications-*",
-        ".transform-notifications-*"
-    ));
-
     private final Settings settings;
     private final ClusterService clusterService;
     private final IndicesService indicesService;
@@ -194,11 +183,7 @@ public class MetaDataCreateIndexService {
             List<SystemIndexDescriptor> matchingDescriptors = systemIndexDescriptors.stream()
                 .filter(descriptor -> descriptor.matchesIndexPattern(index))
                 .collect(toList());
-            if (DOT_INDICES_EXCLUSIONS.run(index)) {
+            if (matchingDescriptors.isEmpty() && (isHidden == null || isHidden == Boolean.FALSE)) {
-                assert Objects.isNull(isHidden) || Boolean.FALSE.equals(isHidden) : "when converting a special-cased index to be a " +
-                    "hidden index, it must be removed from the exclusions list";
-                logger.debug("not emitting deprecation warning about index [{}] because it is in the exclusions list", index);
-            } else if (matchingDescriptors.isEmpty() && (isHidden == null || isHidden == Boolean.FALSE)) {
                 DEPRECATION_LOGGER.deprecated("index name [{}] starts with a dot '.', in the next major version, index names " +
                     "starting with a dot are reserved for hidden indices and system indices", index);
             } else if (matchingDescriptors.size() > 1) {

server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java

@@ -638,41 +638,6 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
         }
     }
 
-    public void testIndexNameExclusionsList() {
-        // this test case should be removed when DOT_INDICES_EXCLUSIONS is empty
-        List<String> excludedNames = Arrays.asList(
-            ".data-frame-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
-            ".transform-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT)
-        );
-
-        ThreadPool testThreadPool = new TestThreadPool(getTestName());
-        try {
-            MetaDataCreateIndexService checkerService = new MetaDataCreateIndexService(
-                Settings.EMPTY,
-                ClusterServiceUtils.createClusterService(testThreadPool),
-                null,
-                null,
-                null,
-                null,
-                null,
-                testThreadPool,
-                null,
-                Collections.emptyList(),
-                false
-            );
-
-            excludedNames.forEach(name -> {
-                checkerService.validateDotIndex(name, ClusterState.EMPTY_STATE, false);
-            });
-
-            excludedNames.forEach(name -> {
-                expectThrows(AssertionError.class, () -> checkerService.validateDotIndex(name, ClusterState.EMPTY_STATE, true));
-            });
-        } finally {
-            testThreadPool.shutdown();
-        }
-    }
-
     public void testParseMappingsAppliesDataFromTemplateAndRequest() throws Exception {
         IndexTemplateMetaData templateMetaData = addMatchingTemplate(templateBuilder -> {
             templateBuilder.putAlias(AliasMetaData.builder("alias1"));

x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformAuditorIT.java

@@ -7,7 +7,9 @@
 package org.elasticsearch.xpack.transform.integration;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
 import org.junit.Before;
@@ -94,7 +96,16 @@ public class TransformAuditorIT extends TransformRestTestCase {
                 .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
                 .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0);
 
-        createIndex(TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED, settings.build());
+        // These indices should only exist if created in previous versions, ignore the deprecation warning for this test
+        RequestOptions options = expectWarnings("index name [" + TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED + "] starts " +
+            "with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices " +
+            "and system indices");
+        Request request = new Request("PUT", "/" + TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED);
+        String entity = "{\"settings\": " + Strings.toString(settings.build()) + "}";
+        request.setJsonEntity(entity);
+        request.setOptions(options);
+        client().performRequest(request);
+
         assertBusy(() -> {
             assertTrue(aliasExists(TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED,
                     TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS));

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java

@@ -78,7 +78,10 @@ class TransformClusterStateListener implements ClusterStateListener {
         }
 
         final IndicesAliasesRequest request = client.admin().indices().prepareAliases()
-                .addAlias(TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED, TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS)
+                .addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                    .index(TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED)
+                    .alias(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS)
+                    .isHidden(true))
                 .request();
 
         executeAsyncWithOrigin(client.threadPool().getThreadContext(), TRANSFORM_ORIGIN, request,

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java

@@ -105,9 +105,10 @@ public final class TransformInternalIndex {
                     // the audits are expected to be small
                     .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                     .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+                    .put(IndexMetaData.SETTING_INDEX_HIDDEN, true)
             )
             .putMapping(MapperService.SINGLE_MAPPING_NAME, Strings.toString(auditMappings()))
-            .putAlias(AliasMetaData.builder(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS))
+            .putAlias(AliasMetaData.builder(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS).isHidden(true))
             .build();
         return transformTemplate;
     }