Use exclusions list instead of fake system indices (#51586)

This commit switches the strategy for managing dot-prefixed indices that
should be hidden indices from using "fake" system indices to an explicit
exclusions list that must be updated when those indices are converted to
hidden indices.

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -23,6 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.Version;
@@ -54,6 +55,7 @@ import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -88,6 +90,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -113,6 +116,19 @@ public class MetaDataCreateIndexService {
 
     public static final int MAX_INDEX_NAME_BYTES = 255;
 
+    /**
+     * These index patterns will be converted to hidden indices, at which point they should be removed from this list.
+     */
+    private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton(
+        ".slm-history-*",
+        ".watch-history-*",
+        ".ml-anomalies-*",
+        ".ml-notifications-*",
+        ".ml-annotations*",
+        ".data-frame-notifications-*",
+        ".transform-notifications-*"
+    ));
+
     private final Settings settings;
     private final ClusterService clusterService;
     private final IndicesService indicesService;
@@ -163,7 +179,11 @@ public class MetaDataCreateIndexService {
             List<SystemIndexDescriptor> matchingDescriptors = systemIndexDescriptors.stream()
                 .filter(descriptor -> descriptor.matchesIndexPattern(index))
                 .collect(toList());
-            if (matchingDescriptors.isEmpty() && (isHidden == null || isHidden == Boolean.FALSE)) {
+            if (DOT_INDICES_EXCLUSIONS.run(index)) {
+                assert Objects.isNull(isHidden) || Boolean.FALSE.equals(isHidden) : "when converting a special-cased index to be a " +
+                    "hidden index, it must be removed from the exclusions list";
+                logger.debug("not emitting deprecation warning about index [{}] because it is in the exclusions list", index);
+            } else if (matchingDescriptors.isEmpty() && (isHidden == null || isHidden == Boolean.FALSE)) {
                 DEPRECATION_LOGGER.deprecated("index name [{}] starts with a dot '.', in the next major version, index names " +
                     "starting with a dot are reserved for hidden indices and system indices", index);
             } else if (matchingDescriptors.size() > 1) {

server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java

@@ -638,6 +638,46 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
         }
     }
 
+    public void testIndexNameExclusionsList() {
+        // this test case should be removed when DOT_INDICES_EXCLUSIONS is empty
+        List<String> excludedNames = Arrays.asList(
+            ".slm-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
+            ".watch-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
+            ".ml-anomalies-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
+            ".ml-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
+            ".ml-annotations-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
+            ".data-frame-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
+            ".transform-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT)
+        );
+
+        ThreadPool testThreadPool = new TestThreadPool(getTestName());
+        try {
+            MetaDataCreateIndexService checkerService = new MetaDataCreateIndexService(
+                Settings.EMPTY,
+                ClusterServiceUtils.createClusterService(testThreadPool),
+                null,
+                null,
+                null,
+                null,
+                null,
+                testThreadPool,
+                null,
+                Collections.emptyList(),
+                false
+            );
+
+            excludedNames.forEach(name -> {
+                checkerService.validateIndexName(name, ClusterState.EMPTY_STATE, false);
+            });
+
+            excludedNames.forEach(name -> {
+                expectThrows(AssertionError.class, () -> checkerService.validateIndexName(name, ClusterState.EMPTY_STATE, true));
+            });
+        } finally {
+            testThreadPool.shutdown();
+        }
+    }
+
     public void testParseMappingsAppliesDataFromTemplateAndRequest() throws Exception {
         IndexTemplateMetaData templateMetaData = addMatchingTemplate(templateBuilder -> {
             templateBuilder.putAlias(AliasMetaData.builder("alias1"));

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java

@@ -10,7 +10,6 @@ public final class AnomalyDetectorsIndexFields {
     public static final String CONFIG_INDEX = ".ml-config";
 
     public static final String RESULTS_INDEX_PREFIX = ".ml-anomalies-";
-    public static final String RESULTS_INDEX_PATTERN = RESULTS_INDEX_PREFIX + "*";
 
     public static final String STATE_INDEX_PREFIX = ".ml-state";
     public static final String STATE_INDEX_PATTERN = STATE_INDEX_PREFIX + "*";

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java

@@ -29,9 +29,8 @@ import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.IndexModule;
-import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.SystemIndexPlugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.script.ScriptService;
@@ -133,7 +132,7 @@ import java.util.function.Supplier;
 
 import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN;
 
-public class IndexLifecycle extends Plugin implements SystemIndexPlugin {
+public class IndexLifecycle extends Plugin implements ActionPlugin {
     private final SetOnce<IndexLifecycleService> indexLifecycleInitialisationService = new SetOnce<>();
     private final SetOnce<ILMHistoryStore> ilmHistoryStore = new SetOnce<>();
     private final SetOnce<SnapshotLifecycleService> snapshotLifecycleService = new SetOnce<>();
@@ -337,12 +336,4 @@ public class IndexLifecycle extends Plugin implements SystemIndexPlugin {
             throw new ElasticsearchException("unable to close index lifecycle services", e);
         }
     }
-
-    @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
-        //TODO: The SLM history store should be non-dot-prefixed hidden indices, but need to be here for now
-        // to prevent warnings
-        return Collections.singletonList(new SystemIndexDescriptor(".slm-history-*",
-            "Contains a history of Snapshot Lifecycle Management operations"));
-    }
 }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java

@@ -130,7 +130,6 @@ import org.elasticsearch.xpack.core.ml.action.UpdateModelSnapshotAction;
 import org.elasticsearch.xpack.core.ml.action.UpdateProcessAction;
 import org.elasticsearch.xpack.core.ml.action.ValidateDetectorAction;
 import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction;
-import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.MlDataFrameAnalysisNamedXContentProvider;
 import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
 import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
@@ -1055,16 +1054,11 @@ public class MachineLearning extends Plugin implements SystemIndexPlugin, Analys
 
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
-        //TODO: Some of the below should be converted to non-dot-prefixed hidden indices. I am not sure which ones.
-        // Either way, they should be here until then to prevent deprecation warnings
         return Collections.unmodifiableList(Arrays.asList(
-            new SystemIndexDescriptor(MlMetaIndex.INDEX_NAME, this.getClass().getSimpleName()),
+            new SystemIndexDescriptor(MlMetaIndex.INDEX_NAME, "Contains scheduling and anomaly tracking metadata"),
-            new SystemIndexDescriptor(AnomalyDetectorsIndexFields.STATE_INDEX_PATTERN, this.getClass().getSimpleName()),
+            new SystemIndexDescriptor(AnomalyDetectorsIndexFields.STATE_INDEX_PATTERN, "Contains ML model state"),
-            new SystemIndexDescriptor(AnomalyDetectorsIndexFields.CONFIG_INDEX, this.getClass().getSimpleName()),
+            new SystemIndexDescriptor(AnomalyDetectorsIndexFields.CONFIG_INDEX, "Contains ML configuration data"),
-            new SystemIndexDescriptor(AnomalyDetectorsIndexFields.RESULTS_INDEX_PATTERN, this.getClass().getSimpleName()),
+            new SystemIndexDescriptor(InferenceIndexConstants.INDEX_PATTERN, "Contains ML model configuration and statistics")
-            new SystemIndexDescriptor(InferenceIndexConstants.INDEX_PATTERN, this.getClass().getSimpleName()),
-            new SystemIndexDescriptor(".ml-notifications-*", this.getClass().getSimpleName()),
-            new SystemIndexDescriptor(AnnotationIndex.INDEX_PATTERN, this.getClass().getSimpleName())
         ));
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -1111,12 +1111,12 @@ public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin,
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return Collections.unmodifiableList(Arrays.asList(
-            new SystemIndexDescriptor(SECURITY_MAIN_ALIAS, this.getClass().getSimpleName()),
+            new SystemIndexDescriptor(SECURITY_MAIN_ALIAS, "Contains Security configuration"),
-            new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, this.getClass().getSimpleName()),
+            new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, "Contains Security configuration"),
-            new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, this.getClass().getSimpleName()),
+            new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, "Contains Security configuration"),
 
-            new SystemIndexDescriptor(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, this.getClass().getSimpleName()),
+            new SystemIndexDescriptor(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, "Contains auth token data"),
-            new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, this.getClass().getSimpleName())
+            new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, "Contains auth token data")
             ));
     }
 }

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java

@@ -326,11 +326,8 @@ public class Transform extends Plugin implements SystemIndexPlugin, PersistentTa
 
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
-        return Collections.unmodifiableList(Arrays.asList(
+        return Collections.singletonList(
-            // TODO: Verify that these should be system indices, rather than hidden indices
+            new SystemIndexDescriptor(TransformInternalIndexConstants.INDEX_NAME_PATTERN, "Contains Transform configuration data")
-            new SystemIndexDescriptor(TransformInternalIndexConstants.INDEX_NAME_PATTERN, this.getClass().getSimpleName()),
+        );
-            new SystemIndexDescriptor(TransformInternalIndexConstants.AUDIT_INDEX_PATTERN, this.getClass().getSimpleName()),
-            new SystemIndexDescriptor(TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED, this.getClass().getSimpleName())
-        ));
     }
 }

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java

@@ -700,12 +700,8 @@ public class Watcher extends Plugin implements SystemIndexPlugin, ScriptPlugin,
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
         return Collections.unmodifiableList(Arrays.asList(
-            new SystemIndexDescriptor(Watch.INDEX, this.getClass().getSimpleName()),
+            new SystemIndexDescriptor(Watch.INDEX, "Contains Watch definitions"),
-            new SystemIndexDescriptor(TriggeredWatchStoreField.INDEX_NAME, this.getClass().getSimpleName()),
+            new SystemIndexDescriptor(TriggeredWatchStoreField.INDEX_NAME, "Used to track current and queued Watch execution")
-
-            //TODO: The below should be converted to hidden indices, but need to be included here until then to prevent
-            // deprecation warnings
-            new SystemIndexDescriptor(".watch-history-*", this.getClass().getSimpleName())
         ));
     }
 }