honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use exclusions list instead of fake system indices (#51586) 

This commit switches the strategy for managing dot-prefixed indices that
should be hidden indices from using "fake" system indices to an explicit
exclusions list that must be updated when those indices are converted to
hidden indices.
This commit is contained in:
Gordon Brown 2020-01-30 16:31:27 -07:00 committed by GitHub
parent 8d2370bf00
commit 10c8179351
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 77 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -23,6 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.Version;
@ -54,6 +55,7 @@ import org.elasticsearch.common.ValidationException;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
@ -88,6 +90,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@ -113,6 +116,19 @@ public class MetaDataCreateIndexService {
public static final int MAX_INDEX_NAME_BYTES = 255;
/**
* These index patterns will be converted to hidden indices, at which point they should be removed from this list.
*/
private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton(
".slm-history-*",
".watch-history-*",
".ml-anomalies-*",
".ml-notifications-*",
".ml-annotations*",
".data-frame-notifications-*",
".transform-notifications-*"
));
private final Settings settings;
private final ClusterService clusterService;
private final IndicesService indicesService;
@ -163,7 +179,11 @@ public class MetaDataCreateIndexService {
List<SystemIndexDescriptor> matchingDescriptors = systemIndexDescriptors.stream()
.filter(descriptor -> descriptor.matchesIndexPattern(index))
.collect(toList());
if (matchingDescriptors.isEmpty() && (isHidden == null || isHidden == Boolean.FALSE)) {
if (DOT_INDICES_EXCLUSIONS.run(index)) {
assert Objects.isNull(isHidden) || Boolean.FALSE.equals(isHidden) : "when converting a special-cased index to be a " +
"hidden index, it must be removed from the exclusions list";
logger.debug("not emitting deprecation warning about index [{}] because it is in the exclusions list", index);
} else if (matchingDescriptors.isEmpty() && (isHidden == null || isHidden == Boolean.FALSE)) {
DEPRECATION_LOGGER.deprecated("index name [{}] starts with a dot '.', in the next major version, index names " +
"starting with a dot are reserved for hidden indices and system indices", index);
} else if (matchingDescriptors.size() > 1) {

40
server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
View File

@ -638,6 +638,46 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
}
}
public void testIndexNameExclusionsList() {
// this test case should be removed when DOT_INDICES_EXCLUSIONS is empty
List<String> excludedNames = Arrays.asList(
".slm-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
".watch-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
".ml-anomalies-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
".ml-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
".ml-annotations-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
".data-frame-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
".transform-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT)
);
ThreadPool testThreadPool = new TestThreadPool(getTestName());
try {
MetaDataCreateIndexService checkerService = new MetaDataCreateIndexService(
Settings.EMPTY,
ClusterServiceUtils.createClusterService(testThreadPool),
null,
null,
null,
null,
null,
testThreadPool,
null,
Collections.emptyList(),
false
);
excludedNames.forEach(name -> {
checkerService.validateIndexName(name, ClusterState.EMPTY_STATE, false);
});
excludedNames.forEach(name -> {
expectThrows(AssertionError.class, () -> checkerService.validateIndexName(name, ClusterState.EMPTY_STATE, true));
});
} finally {
testThreadPool.shutdown();
}
}
public void testParseMappingsAppliesDataFromTemplateAndRequest() throws Exception {
IndexTemplateMetaData templateMetaData = addMatchingTemplate(templateBuilder -> {
templateBuilder.putAlias(AliasMetaData.builder("alias1"));

1
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndexFields.java
View File

@ -10,7 +10,6 @@ public final class AnomalyDetectorsIndexFields {
public static final String CONFIG_INDEX = ".ml-config";
public static final String RESULTS_INDEX_PREFIX = ".ml-anomalies-";
public static final String RESULTS_INDEX_PATTERN = RESULTS_INDEX_PREFIX + "*";
public static final String STATE_INDEX_PREFIX = ".ml-state";
public static final String STATE_INDEX_PATTERN = STATE_INDEX_PREFIX + "*";

13
x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java
View File

@ -29,9 +29,8 @@ import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.indices.SystemIndexDescriptor;
import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.SystemIndexPlugin;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.script.ScriptService;
@ -133,7 +132,7 @@ import java.util.function.Supplier;
import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN;
public class IndexLifecycle extends Plugin implements SystemIndexPlugin {
public class IndexLifecycle extends Plugin implements ActionPlugin {
private final SetOnce<IndexLifecycleService> indexLifecycleInitialisationService = new SetOnce<>();
private final SetOnce<ILMHistoryStore> ilmHistoryStore = new SetOnce<>();
private final SetOnce<SnapshotLifecycleService> snapshotLifecycleService = new SetOnce<>();
@ -337,12 +336,4 @@ public class IndexLifecycle extends Plugin implements SystemIndexPlugin {
throw new ElasticsearchException("unable to close index lifecycle services", e);
}
}
@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
//TODO: The SLM history store should be non-dot-prefixed hidden indices, but need to be here for now
// to prevent warnings
return Collections.singletonList(new SystemIndexDescriptor(".slm-history-*",
"Contains a history of Snapshot Lifecycle Management operations"));
}
}

14
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
View File

@ -130,7 +130,6 @@ import org.elasticsearch.xpack.core.ml.action.UpdateModelSnapshotAction;
import org.elasticsearch.xpack.core.ml.action.UpdateProcessAction;
import org.elasticsearch.xpack.core.ml.action.ValidateDetectorAction;
import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction;
import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex;
import org.elasticsearch.xpack.core.ml.dataframe.analyses.MlDataFrameAnalysisNamedXContentProvider;
import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider;
import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider;
@ -1055,16 +1054,11 @@ public class MachineLearning extends Plugin implements SystemIndexPlugin, Analys
@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
//TODO: Some of the below should be converted to non-dot-prefixed hidden indices. I am not sure which ones.
// Either way, they should be here until then to prevent deprecation warnings
return Collections.unmodifiableList(Arrays.asList(
new SystemIndexDescriptor(MlMetaIndex.INDEX_NAME, this.getClass().getSimpleName()),
new SystemIndexDescriptor(AnomalyDetectorsIndexFields.STATE_INDEX_PATTERN, this.getClass().getSimpleName()),
new SystemIndexDescriptor(AnomalyDetectorsIndexFields.CONFIG_INDEX, this.getClass().getSimpleName()),
new SystemIndexDescriptor(AnomalyDetectorsIndexFields.RESULTS_INDEX_PATTERN, this.getClass().getSimpleName()),
new SystemIndexDescriptor(InferenceIndexConstants.INDEX_PATTERN, this.getClass().getSimpleName()),
new SystemIndexDescriptor(".ml-notifications-*", this.getClass().getSimpleName()),
new SystemIndexDescriptor(AnnotationIndex.INDEX_PATTERN, this.getClass().getSimpleName())
new SystemIndexDescriptor(MlMetaIndex.INDEX_NAME, "Contains scheduling and anomaly tracking metadata"),
new SystemIndexDescriptor(AnomalyDetectorsIndexFields.STATE_INDEX_PATTERN, "Contains ML model state"),
new SystemIndexDescriptor(AnomalyDetectorsIndexFields.CONFIG_INDEX, "Contains ML configuration data"),
new SystemIndexDescriptor(InferenceIndexConstants.INDEX_PATTERN, "Contains ML model configuration and statistics")
));
}
}

10
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
View File

@ -1111,12 +1111,12 @@ public class Security extends Plugin implements SystemIndexPlugin, IngestPlugin,
@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
return Collections.unmodifiableList(Arrays.asList(
new SystemIndexDescriptor(SECURITY_MAIN_ALIAS, this.getClass().getSimpleName()),
new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, this.getClass().getSimpleName()),
new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, this.getClass().getSimpleName()),
new SystemIndexDescriptor(SECURITY_MAIN_ALIAS, "Contains Security configuration"),
new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_6, "Contains Security configuration"),
new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7, "Contains Security configuration"),
new SystemIndexDescriptor(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, this.getClass().getSimpleName()),
new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, this.getClass().getSimpleName())
new SystemIndexDescriptor(RestrictedIndicesNames.SECURITY_TOKENS_ALIAS, "Contains auth token data"),
new SystemIndexDescriptor(RestrictedIndicesNames.INTERNAL_SECURITY_TOKENS_INDEX_7, "Contains auth token data")
));
}
}

9
x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
View File

@ -326,11 +326,8 @@ public class Transform extends Plugin implements SystemIndexPlugin, PersistentTa
@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
return Collections.unmodifiableList(Arrays.asList(
// TODO: Verify that these should be system indices, rather than hidden indices
new SystemIndexDescriptor(TransformInternalIndexConstants.INDEX_NAME_PATTERN, this.getClass().getSimpleName()),
new SystemIndexDescriptor(TransformInternalIndexConstants.AUDIT_INDEX_PATTERN, this.getClass().getSimpleName()),
new SystemIndexDescriptor(TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED, this.getClass().getSimpleName())
));
return Collections.singletonList(
new SystemIndexDescriptor(TransformInternalIndexConstants.INDEX_NAME_PATTERN, "Contains Transform configuration data")
);
}
}

8
x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
View File

@ -700,12 +700,8 @@ public class Watcher extends Plugin implements SystemIndexPlugin, ScriptPlugin,
@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors() {
return Collections.unmodifiableList(Arrays.asList(
new SystemIndexDescriptor(Watch.INDEX, this.getClass().getSimpleName()),
new SystemIndexDescriptor(TriggeredWatchStoreField.INDEX_NAME, this.getClass().getSimpleName()),
//TODO: The below should be converted to hidden indices, but need to be included here until then to prevent
// deprecation warnings
new SystemIndexDescriptor(".watch-history-*", this.getClass().getSimpleName())
new SystemIndexDescriptor(Watch.INDEX, "Contains Watch definitions"),
new SystemIndexDescriptor(TriggeredWatchStoreField.INDEX_NAME, "Used to track current and queued Watch execution")
));
}
}