From 10c81793512c7e38048237e8deb19a5c9880a2f1 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Thu, 30 Jan 2020 16:31:27 -0700 Subject: [PATCH] Use exclusions list instead of fake system indices (#51586) This commit switches the strategy for managing dot-prefixed indices that should be hidden indices from using "fake" system indices to an explicit exclusions list that must be updated when those indices are converted to hidden indices. --- .../metadata/MetaDataCreateIndexService.java | 22 +++++++++- .../MetaDataCreateIndexServiceTests.java | 40 +++++++++++++++++++ .../AnomalyDetectorsIndexFields.java | 1 - .../xpack/ilm/IndexLifecycle.java | 13 +----- .../xpack/ml/MachineLearning.java | 14 ++----- .../xpack/security/Security.java | 10 ++--- .../xpack/transform/Transform.java | 9 ++--- .../elasticsearch/xpack/watcher/Watcher.java | 8 +--- 8 files changed, 77 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 69d8a6576f5..00d1410f363 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -23,6 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.Version; @@ -54,6 +55,7 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -88,6 +90,7 @@ import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -113,6 +116,19 @@ public class MetaDataCreateIndexService { public static final int MAX_INDEX_NAME_BYTES = 255; + /** + * These index patterns will be converted to hidden indices, at which point they should be removed from this list. + */ + private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton( + ".slm-history-*", + ".watch-history-*", + ".ml-anomalies-*", + ".ml-notifications-*", + ".ml-annotations*", + ".data-frame-notifications-*", + ".transform-notifications-*" + )); + private final Settings settings; private final ClusterService clusterService; private final IndicesService indicesService; @@ -163,7 +179,11 @@ public class MetaDataCreateIndexService { List matchingDescriptors = systemIndexDescriptors.stream() .filter(descriptor -> descriptor.matchesIndexPattern(index)) .collect(toList()); - if (matchingDescriptors.isEmpty() && (isHidden == null || isHidden == Boolean.FALSE)) { + if (DOT_INDICES_EXCLUSIONS.run(index)) { + assert Objects.isNull(isHidden) || Boolean.FALSE.equals(isHidden) : "when converting a special-cased index to be a " + + "hidden index, it must be removed from the exclusions list"; + logger.debug("not emitting deprecation warning about index [{}] because it is in the exclusions list", index); + } else if (matchingDescriptors.isEmpty() && (isHidden == null || isHidden == Boolean.FALSE)) { DEPRECATION_LOGGER.deprecated("index name [{}] starts with a dot '.', in the next major version, index names " + "starting with a dot are reserved for hidden indices and system indices", index); } else if (matchingDescriptors.size() > 1) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 91568da048c..bfd1a987a69 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -638,6 +638,46 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase { } } + public void testIndexNameExclusionsList() { + // this test case should be removed when DOT_INDICES_EXCLUSIONS is empty + List excludedNames = Arrays.asList( + ".slm-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), + ".watch-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), + ".ml-anomalies-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), + ".ml-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), + ".ml-annotations-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), + ".data-frame-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), + ".transform-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT) + ); + + ThreadPool testThreadPool = new TestThreadPool(getTestName()); + try { + MetaDataCreateIndexService checkerService = new MetaDataCreateIndexService( + Settings.EMPTY, + ClusterServiceUtils.createClusterService(testThreadPool), + null, + null, + null, + null, + null, + testThreadPool, + null, + Collections.emptyList(), + false + ); + + excludedNames.forEach(name -> { + checkerService.validateIndexName(name, ClusterState.EMPTY_STATE, false); + }); + + excludedNames.forEach(name -> { + expectThrows(AssertionError.class, () -> checkerService.validateIndexName(name, ClusterState.EMPTY_STATE, true)); + }); + } finally { + testThreadPool.shutdown(); + } + } + public void testParseMappingsAppliesDataFromTemplateAndRequest() throws Exception { IndexTemplateMetaData templateMetaData = addMatchingTemplate(templateBuilder -> { templateBuilder.putAlias(AliasMetaData.builder("alias1")); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java index d62e2de207d..abcfc19efbf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java @@ -10,7 +10,6 @@ public final class AnomalyDetectorsIndexFields { public static final String CONFIG_INDEX = ".ml-config"; public static final String RESULTS_INDEX_PREFIX = ".ml-anomalies-"; - public static final String RESULTS_INDEX_PATTERN = RESULTS_INDEX_PREFIX + "*"; public static final String STATE_INDEX_PREFIX = ".ml-state"; public static final String STATE_INDEX_PATTERN = STATE_INDEX_PREFIX + "*"; diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index 485c14099bc..6b479c4b2b5 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -29,9 +29,8 @@ import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexModule; -import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptService; @@ -133,7 +132,7 @@ import java.util.function.Supplier; import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN; -public class IndexLifecycle extends Plugin implements SystemIndexPlugin { +public class IndexLifecycle extends Plugin implements ActionPlugin { private final SetOnce indexLifecycleInitialisationService = new SetOnce<>(); private final SetOnce ilmHistoryStore = new SetOnce<>(); private final SetOnce snapshotLifecycleService = new SetOnce<>(); @@ -337,12 +336,4 @@ public class IndexLifecycle extends Plugin implements SystemIndexPlugin { throw new ElasticsearchException("unable to close index lifecycle services", e); } } - - @Override - public Collection getSystemIndexDescriptors() { - //TODO: The SLM history store should be non-dot-prefixed hidden indices, but need to be here for now - // to prevent warnings - return Collections.singletonList(new SystemIndexDescriptor(".slm-history-*", - "Contains a history of Snapshot Lifecycle Management operations")); - } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 9b4ca2dd675..900846ce844 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -130,7 +130,6 @@ import org.elasticsearch.xpack.core.ml.action.UpdateModelSnapshotAction; import org.elasticsearch.xpack.core.ml.action.UpdateProcessAction; import org.elasticsearch.xpack.core.ml.action.ValidateDetectorAction; import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction; -import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; import org.elasticsearch.xpack.core.ml.dataframe.analyses.MlDataFrameAnalysisNamedXContentProvider; import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; @@ -1055,16 +1054,11 @@ public class MachineLearning extends Plugin implements SystemIndexPlugin, Analys @Override public Collection getSystemIndexDescriptors() { - //TODO: Some of the below should be converted to non-dot-prefixed hidden indices. I am not sure which ones. - // Either way, they should be here until then to prevent deprecation warnings return Collections.unmodifiableList(Arrays.asList( - new SystemIndexDescriptor(MlMetaIndex.INDEX_NAME, this.getClass().getSimpleName()), - new SystemIndexDescriptor(AnomalyDetectorsIndexFields.STATE_INDEX_PATTERN, this.getClass().getSimpleName()), - new SystemIndexDescriptor(AnomalyDetectorsIndexFields.CONFIG_INDEX, this.getClass().getSimpleName()), - new SystemIndexDescriptor(AnomalyDetectorsIndexFields.RESULTS_INDEX_PATTERN, this.getClass().getSimpleName()), - new SystemIndexDescriptor(InferenceIndexConstants.INDEX_PATTERN, this.getClass().getSimpleName()), - new SystemIndexDescriptor(".ml-notifications-*", this.getClass().getSimpleName()), - new SystemIndexDescriptor(AnnotationIndex.INDEX_PATTERN, this.getClass().getSimpleName()) + new SystemIndexDescriptor(MlMetaIndex.INDEX_NAME, "Contains scheduling and anomaly tracking metadata"), + new SystemIndexDescriptor(AnomalyDetectorsIndexFields.STATE_INDEX_PATTERN, "Contains ML model state"), + new SystemIndexDescriptor(AnomalyDetectorsIndexFields.CONFIG_INDEX, "Contains ML configuration data"), + new SystemIndexDescriptor(InferenceIndexConstants.INDEX_PATTERN, "Contains ML model configuration and statistics") )); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 989b7c2c64d..145302608f5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -1111,12 +1111,12 @@ public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin, @Override public Collection getSystemIndexDescriptors() { return Collections.unmodifiableList(Arrays.asList( - new SystemIndexDescriptor(SECURITY_MAIN_ALIAS, this.getClass().getSimpleName()), - new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, this.getClass().getSimpleName()), - new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, this.getClass().getSimpleName()), + new SystemIndexDescriptor(SECURITY_MAIN_ALIAS, "Contains Security configuration"), + new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, "Contains Security configuration"), + new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, "Contains Security configuration"), - new SystemIndexDescriptor(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, this.getClass().getSimpleName()), - new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, this.getClass().getSimpleName()) + new SystemIndexDescriptor(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, "Contains auth token data"), + new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, "Contains auth token data") )); } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index 26c77a9ad25..453a998b13a 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -326,11 +326,8 @@ public class Transform extends Plugin implements SystemIndexPlugin, PersistentTa @Override public Collection getSystemIndexDescriptors() { - return Collections.unmodifiableList(Arrays.asList( - // TODO: Verify that these should be system indices, rather than hidden indices - new SystemIndexDescriptor(TransformInternalIndexConstants.INDEX_NAME_PATTERN, this.getClass().getSimpleName()), - new SystemIndexDescriptor(TransformInternalIndexConstants.AUDIT_INDEX_PATTERN, this.getClass().getSimpleName()), - new SystemIndexDescriptor(TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED, this.getClass().getSimpleName()) - )); + return Collections.singletonList( + new SystemIndexDescriptor(TransformInternalIndexConstants.INDEX_NAME_PATTERN, "Contains Transform configuration data") + ); } } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index 53152bb1589..a71695f38e4 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -700,12 +700,8 @@ public class Watcher extends Plugin implements SystemIndexPlugin, ScriptPlugin, @Override public Collection getSystemIndexDescriptors() { return Collections.unmodifiableList(Arrays.asList( - new SystemIndexDescriptor(Watch.INDEX, this.getClass().getSimpleName()), - new SystemIndexDescriptor(TriggeredWatchStoreField.INDEX_NAME, this.getClass().getSimpleName()), - - //TODO: The below should be converted to hidden indices, but need to be included here until then to prevent - // deprecation warnings - new SystemIndexDescriptor(".watch-history-*", this.getClass().getSimpleName()) + new SystemIndexDescriptor(Watch.INDEX, "Contains Watch definitions"), + new SystemIndexDescriptor(TriggeredWatchStoreField.INDEX_NAME, "Used to track current and queued Watch execution") )); } }