honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add version guards around ML hidden indices settings (#54322)

This commit is contained in:
Przemysław Witek 2020-03-27 14:50:57 +01:00 committed by GitHub
parent 5983f6aceb
commit 2eb079b67f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 77 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
View File

@ -37,6 +37,8 @@ public class AnnotationIndex {
private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
private static final Version HIDDEN_INTRODUCED_VERSION = Version.V_7_7_0;
/**
* Create the .ml-annotations index with correct mappings if it does not already
* exist. This index is read and written by the UI results views, so needs to
@ -45,11 +47,21 @@ public class AnnotationIndex {
public static void createAnnotationsIndexIfNecessary(Settings settings, Client client, ClusterState state,
final ActionListener<Boolean> finalListener) {
boolean isHiddenAttributeAvailable = state.nodes().getMinNodeVersion().onOrAfter(HIDDEN_INTRODUCED_VERSION);
final ActionListener<Boolean> createAliasListener = ActionListener.wrap(success -> {
IndicesAliasesRequest.AliasActions addReadAliasAction =
IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(READ_ALIAS_NAME);
IndicesAliasesRequest.AliasActions addWriteAliasAction =
IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(WRITE_ALIAS_NAME);
if (isHiddenAttributeAvailable) {
addReadAliasAction.isHidden(true);
addWriteAliasAction.isHidden(true);
}
final IndicesAliasesRequest request =
client.admin().indices().prepareAliases()
.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(READ_ALIAS_NAME).isHidden(true))
.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(WRITE_ALIAS_NAME).isHidden(true))
.addAliasAction(addReadAliasAction)
.addAliasAction(addWriteAliasAction)
.request();
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
ActionListener.<AcknowledgedResponse>wrap(r -> finalListener.onResponse(r.isAcknowledged()), finalListener::onFailure),
@ -63,13 +75,18 @@ public class AnnotationIndex {
// Create the annotations index if it doesn't exist already.
if (mlLookup.containsKey(INDEX_NAME) == false) {
Settings.Builder settingsBuilder =
Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1");
if (isHiddenAttributeAvailable) {
settingsBuilder.put(IndexMetaData.SETTING_INDEX_HIDDEN, true);
}
CreateIndexRequest createIndexRequest =
new CreateIndexRequest(INDEX_NAME)
.mapping(SINGLE_MAPPING_NAME, annotationsMapping(), XContentType.JSON)
.settings(Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
.put(IndexMetaData.SETTING_INDEX_HIDDEN, true));
.settings(settingsBuilder);
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
ActionListener.<CreateIndexResponse>wrap(

34
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java
View File

@ -8,6 +8,7 @@ package org.elasticsearch.xpack.core.ml.utils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
@ -38,6 +39,8 @@ public final class MlIndexAndAlias {
private static final Logger logger = LogManager.getLogger(MlIndexAndAlias.class);
static final Version HIDDEN_INTRODUCED_VERSION = Version.V_7_7_0;
// Visible for testing
static final Comparator<String> INDEX_NAME_COMPARATOR = new Comparator<String>() {
@ -76,6 +79,8 @@ public final class MlIndexAndAlias {
String alias,
ActionListener<Boolean> listener) {
boolean isHiddenAttributeAvailable = clusterState.nodes().getMinNodeVersion().onOrAfter(HIDDEN_INTRODUCED_VERSION);
String legacyIndexWithoutSuffix = indexPatternPrefix;
String indexPattern = indexPatternPrefix + "*";
// The initial index name must be suitable for rollover functionality.
@ -88,7 +93,7 @@ public final class MlIndexAndAlias {
if (concreteIndexNames.length == 0) {
if (indexPointedByCurrentWriteAlias.isPresent() == false) {
createFirstConcreteIndex(client, firstConcreteIndex, alias, true, listener);
createFirstConcreteIndex(client, firstConcreteIndex, alias, true, isHiddenAttributeAvailable, listener);
return;
}
logger.error(
@ -96,7 +101,7 @@ public final class MlIndexAndAlias {
indexPattern, alias, indexPointedByCurrentWriteAlias.get());
} else if (concreteIndexNames.length == 1 && concreteIndexNames[0].equals(legacyIndexWithoutSuffix)) {
if (indexPointedByCurrentWriteAlias.isPresent() == false) {
createFirstConcreteIndex(client, firstConcreteIndex, alias, true, listener);
createFirstConcreteIndex(client, firstConcreteIndex, alias, true, isHiddenAttributeAvailable, listener);
return;
}
if (indexPointedByCurrentWriteAlias.get().getIndex().getName().equals(legacyIndexWithoutSuffix)) {
@ -105,8 +110,10 @@ public final class MlIndexAndAlias {
firstConcreteIndex,
alias,
false,
isHiddenAttributeAvailable,
ActionListener.wrap(
unused -> updateWriteAlias(client, alias, legacyIndexWithoutSuffix, firstConcreteIndex, listener),
unused -> updateWriteAlias(
client, alias, legacyIndexWithoutSuffix, firstConcreteIndex, isHiddenAttributeAvailable, listener),
listener::onFailure)
);
return;
@ -118,7 +125,7 @@ public final class MlIndexAndAlias {
if (indexPointedByCurrentWriteAlias.isPresent() == false) {
assert concreteIndexNames.length > 0;
String latestConcreteIndexName = Arrays.stream(concreteIndexNames).max(INDEX_NAME_COMPARATOR).get();
updateWriteAlias(client, alias, null, latestConcreteIndexName, listener);
updateWriteAlias(client, alias, null, latestConcreteIndexName, isHiddenAttributeAvailable, listener);
return;
}
}
@ -130,12 +137,17 @@ public final class MlIndexAndAlias {
String index,
String alias,
boolean addAlias,
boolean isHiddenAttributeAvailable,
ActionListener<Boolean> listener) {
CreateIndexRequestBuilder requestBuilder = client.admin()
.indices()
.prepareCreate(index);
if (addAlias) {
requestBuilder.addAlias(new Alias(alias).isHidden(true));
Alias newAlias = new Alias(alias);
if (isHiddenAttributeAvailable) {
newAlias.isHidden(true);
}
requestBuilder.addAlias(newAlias);
}
CreateIndexRequest request = requestBuilder.request();
@ -149,7 +161,7 @@ public final class MlIndexAndAlias {
// Adding an alias that already exists is idempotent. So, no need to double check if the alias exists
// as well.
if (ExceptionsHelper.unwrapCause(createIndexFailure) instanceof ResourceAlreadyExistsException) {
updateWriteAlias(client, alias, null, index, listener);
updateWriteAlias(client, alias, null, index, isHiddenAttributeAvailable, listener);
} else {
listener.onFailure(createIndexFailure);
}
@ -161,11 +173,13 @@ public final class MlIndexAndAlias {
String alias,
@Nullable String currentIndex,
String newIndex,
boolean isHiddenAttributeAvailable,
ActionListener<Boolean> listener) {
IndicesAliasesRequestBuilder requestBuilder = client.admin()
.indices()
.prepareAliases()
.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias).isHidden(true));
IndicesAliasesRequest.AliasActions addNewAliasAction = IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias);
if (isHiddenAttributeAvailable) {
addNewAliasAction.isHidden(true);
}
IndicesAliasesRequestBuilder requestBuilder = client.admin().indices().prepareAliases().addAliasAction(addNewAliasAction);
if (currentIndex != null) {
requestBuilder.removeAlias(currentIndex, alias);
}

7
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java
View File

@ -26,8 +26,11 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.ThreadPool;
@ -37,6 +40,7 @@ import org.mockito.ArgumentCaptor;
import org.mockito.InOrder;
import org.mockito.stubbing.Answer;
import java.net.InetAddress;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
@ -46,6 +50,7 @@ import java.util.function.Function;
import java.util.stream.Stream;
import static java.util.stream.Collectors.toMap;
import static org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias.HIDDEN_INTRODUCED_VERSION;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
@ -251,6 +256,8 @@ public class MlIndexAndAliasTests extends ESTestCase {
return ClusterState.builder(ClusterName.DEFAULT)
.metaData(MetaData.builder()
.indices(ImmutableOpenMap.<String, IndexMetaData>builder().putAll(indices).build()).build())
.nodes(DiscoveryNodes.builder()
.add(new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), HIDDEN_INTRODUCED_VERSION)))
.build();
}

26
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
View File

@ -12,6 +12,7 @@ import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
@ -143,6 +144,7 @@ public class JobResultsProvider {
private static final int RECORDS_SIZE_PARAM = 10000;
public static final int BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE = 20;
private static final double ESTABLISHED_MEMORY_CV_THRESHOLD = 0.1;
public static final Version HIDDEN_INTRODUCED_VERSION = Version.V_7_7_0;
private final Client client;
private final Settings settings;
@ -259,6 +261,8 @@ public class JobResultsProvider {
* Create the Elasticsearch index and the mappings
*/
public void createJobResultIndex(Job job, ClusterState state, final ActionListener<Boolean> finalListener) {
boolean isHiddenAttributeAvailable = state.nodes().getMinNodeVersion().onOrAfter(HIDDEN_INTRODUCED_VERSION);
Collection<String> termFields = (job.getAnalysisConfig() != null) ? job.getAnalysisConfig().termFields() : Collections.emptyList();
String readAliasName = AnomalyDetectorsIndex.jobResultsAliasedName(job.getId());
@ -283,15 +287,23 @@ public class JobResultsProvider {
final String indexName = tempIndexName;
ActionListener<Boolean> indexAndMappingsListener = ActionListener.wrap(success -> {
IndicesAliasesRequest.AliasActions addReadAliasAction =
IndicesAliasesRequest.AliasActions.add()
.index(indexName)
.alias(readAliasName)
.filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId()));
IndicesAliasesRequest.AliasActions addWriteAliasAction =
IndicesAliasesRequest.AliasActions.add()
.index(indexName)
.alias(writeAliasName);
if (isHiddenAttributeAvailable) {
addReadAliasAction.isHidden(true);
addWriteAliasAction.isHidden(true);
}
final IndicesAliasesRequest request =
client.admin().indices().prepareAliases()
.addAliasAction(
IndicesAliasesRequest.AliasActions.add()
.index(indexName)
.alias(readAliasName)
.isHidden(true)
.filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId())))
.addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(writeAliasName).isHidden(true))
.addAliasAction(addReadAliasAction)
.addAliasAction(addWriteAliasAction)
.request();
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
ActionListener.<AcknowledgedResponse>wrap(r -> finalListener.onResponse(true), finalListener::onFailure),

4
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java
View File

@ -14,6 +14,7 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.collect.ImmutableOpenMap;
@ -178,9 +179,12 @@ public class AutodetectProcessManagerTests extends ESTestCase {
.build())
.build())
.build();
DiscoveryNodes nodes = mock(DiscoveryNodes.class);
when(nodes.getMinNodeVersion()).thenReturn(JobResultsProvider.HIDDEN_INTRODUCED_VERSION);
clusterState = mock(ClusterState.class);
when(clusterState.getMetaData()).thenReturn(metaData);
when(clusterState.metaData()).thenReturn(metaData);
when(clusterState.nodes()).thenReturn(nodes);
nativeStorageProvider = mock(NativeStorageProvider.class);
doAnswer(invocationOnMock -> {