Add version guards around ML hidden indices settings (#54322)

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java

@@ -37,6 +37,8 @@ public class AnnotationIndex {
 
     private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
 
+    private static final Version HIDDEN_INTRODUCED_VERSION = Version.V_7_7_0;
+
     /**
      * Create the .ml-annotations index with correct mappings if it does not already
      * exist. This index is read and written by the UI results views, so needs to
@@ -45,11 +47,21 @@ public class AnnotationIndex {
     public static void createAnnotationsIndexIfNecessary(Settings settings, Client client, ClusterState state,
                                                          final ActionListener<Boolean> finalListener) {
 
+        boolean isHiddenAttributeAvailable = state.nodes().getMinNodeVersion().onOrAfter(HIDDEN_INTRODUCED_VERSION);
+
         final ActionListener<Boolean> createAliasListener = ActionListener.wrap(success -> {
+            IndicesAliasesRequest.AliasActions addReadAliasAction =
+                IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(READ_ALIAS_NAME);
+            IndicesAliasesRequest.AliasActions addWriteAliasAction =
+                IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(WRITE_ALIAS_NAME);
+            if (isHiddenAttributeAvailable) {
+                addReadAliasAction.isHidden(true);
+                addWriteAliasAction.isHidden(true);
+            }
             final IndicesAliasesRequest request =
                 client.admin().indices().prepareAliases()
-                    .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(READ_ALIAS_NAME).isHidden(true))
+                    .addAliasAction(addReadAliasAction)
-                    .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(WRITE_ALIAS_NAME).isHidden(true))
+                    .addAliasAction(addWriteAliasAction)
                     .request();
             executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
                 ActionListener.<AcknowledgedResponse>wrap(r -> finalListener.onResponse(r.isAcknowledged()), finalListener::onFailure),
@@ -63,13 +75,18 @@ public class AnnotationIndex {
             // Create the annotations index if it doesn't exist already.
             if (mlLookup.containsKey(INDEX_NAME) == false) {
 
+                Settings.Builder settingsBuilder =
+                    Settings.builder()
+                        .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
+                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1");
+                if (isHiddenAttributeAvailable) {
+                    settingsBuilder.put(IndexMetaData.SETTING_INDEX_HIDDEN, true);
+                }
+
                 CreateIndexRequest createIndexRequest =
                     new CreateIndexRequest(INDEX_NAME)
                         .mapping(SINGLE_MAPPING_NAME, annotationsMapping(), XContentType.JSON)
-                        .settings(Settings.builder()
+                        .settings(settingsBuilder);
-                            .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
-                            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
-                            .put(IndexMetaData.SETTING_INDEX_HIDDEN, true));
 
                 executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
                     ActionListener.<CreateIndexResponse>wrap(

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAlias.java

@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.core.ml.utils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
@@ -38,6 +39,8 @@ public final class MlIndexAndAlias {
 
     private static final Logger logger = LogManager.getLogger(MlIndexAndAlias.class);
 
+    static final Version HIDDEN_INTRODUCED_VERSION = Version.V_7_7_0;
+
     // Visible for testing
     static final Comparator<String> INDEX_NAME_COMPARATOR = new Comparator<String>() {
 
@@ -76,6 +79,8 @@ public final class MlIndexAndAlias {
                                                       String alias,
                                                       ActionListener<Boolean> listener) {
 
+        boolean isHiddenAttributeAvailable = clusterState.nodes().getMinNodeVersion().onOrAfter(HIDDEN_INTRODUCED_VERSION);
+
         String legacyIndexWithoutSuffix = indexPatternPrefix;
         String indexPattern = indexPatternPrefix + "*";
         // The initial index name must be suitable for rollover functionality.
@@ -88,7 +93,7 @@ public final class MlIndexAndAlias {
 
         if (concreteIndexNames.length == 0) {
             if (indexPointedByCurrentWriteAlias.isPresent() == false) {
-                createFirstConcreteIndex(client, firstConcreteIndex, alias, true, listener);
+                createFirstConcreteIndex(client, firstConcreteIndex, alias, true, isHiddenAttributeAvailable, listener);
                 return;
             }
             logger.error(
@@ -96,7 +101,7 @@ public final class MlIndexAndAlias {
                 indexPattern, alias, indexPointedByCurrentWriteAlias.get());
         } else if (concreteIndexNames.length == 1 && concreteIndexNames[0].equals(legacyIndexWithoutSuffix)) {
             if (indexPointedByCurrentWriteAlias.isPresent() == false) {
-                createFirstConcreteIndex(client, firstConcreteIndex, alias, true, listener);
+                createFirstConcreteIndex(client, firstConcreteIndex, alias, true, isHiddenAttributeAvailable, listener);
                 return;
             }
             if (indexPointedByCurrentWriteAlias.get().getIndex().getName().equals(legacyIndexWithoutSuffix)) {
@@ -105,8 +110,10 @@ public final class MlIndexAndAlias {
                     firstConcreteIndex,
                     alias,
                     false,
+                    isHiddenAttributeAvailable,
                     ActionListener.wrap(
-                        unused -> updateWriteAlias(client, alias, legacyIndexWithoutSuffix, firstConcreteIndex, listener),
+                        unused -> updateWriteAlias(
+                            client, alias, legacyIndexWithoutSuffix, firstConcreteIndex, isHiddenAttributeAvailable, listener),
                         listener::onFailure)
                 );
                 return;
@@ -118,7 +125,7 @@ public final class MlIndexAndAlias {
             if (indexPointedByCurrentWriteAlias.isPresent() == false) {
                 assert concreteIndexNames.length > 0;
                 String latestConcreteIndexName = Arrays.stream(concreteIndexNames).max(INDEX_NAME_COMPARATOR).get();
-                updateWriteAlias(client, alias, null, latestConcreteIndexName, listener);
+                updateWriteAlias(client, alias, null, latestConcreteIndexName, isHiddenAttributeAvailable, listener);
                 return;
             }
         }
@@ -130,12 +137,17 @@ public final class MlIndexAndAlias {
                                                  String index,
                                                  String alias,
                                                  boolean addAlias,
+                                                 boolean isHiddenAttributeAvailable,
                                                  ActionListener<Boolean> listener) {
         CreateIndexRequestBuilder requestBuilder = client.admin()
             .indices()
             .prepareCreate(index);
         if (addAlias) {
-            requestBuilder.addAlias(new Alias(alias).isHidden(true));
+            Alias newAlias = new Alias(alias);
+            if (isHiddenAttributeAvailable) {
+                newAlias.isHidden(true);
+            }
+            requestBuilder.addAlias(newAlias);
         }
         CreateIndexRequest request = requestBuilder.request();
 
@@ -149,7 +161,7 @@ public final class MlIndexAndAlias {
                     // Adding an alias that already exists is idempotent. So, no need to double check if the alias exists
                     // as well.
                     if (ExceptionsHelper.unwrapCause(createIndexFailure) instanceof ResourceAlreadyExistsException) {
-                        updateWriteAlias(client, alias, null, index, listener);
+                        updateWriteAlias(client, alias, null, index, isHiddenAttributeAvailable, listener);
                     } else {
                         listener.onFailure(createIndexFailure);
                     }
@@ -161,11 +173,13 @@ public final class MlIndexAndAlias {
                                          String alias,
                                          @Nullable String currentIndex,
                                          String newIndex,
+                                         boolean isHiddenAttributeAvailable,
                                          ActionListener<Boolean> listener) {
-        IndicesAliasesRequestBuilder requestBuilder = client.admin()
+        IndicesAliasesRequest.AliasActions addNewAliasAction = IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias);
-            .indices()
+        if (isHiddenAttributeAvailable) {
-            .prepareAliases()
+            addNewAliasAction.isHidden(true);
-            .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(newIndex).alias(alias).isHidden(true));
+        }
+        IndicesAliasesRequestBuilder requestBuilder = client.admin().indices().prepareAliases().addAliasAction(addNewAliasAction);
         if (currentIndex != null) {
             requestBuilder.removeAlias(currentIndex, alias);
         }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/utils/MlIndexAndAliasTests.java

@@ -26,8 +26,11 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -37,6 +40,7 @@ import org.mockito.ArgumentCaptor;
 import org.mockito.InOrder;
 import org.mockito.stubbing.Answer;
 
+import java.net.InetAddress;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
@@ -46,6 +50,7 @@ import java.util.function.Function;
 import java.util.stream.Stream;
 
 import static java.util.stream.Collectors.toMap;
+import static org.elasticsearch.xpack.core.ml.utils.MlIndexAndAlias.HIDDEN_INTRODUCED_VERSION;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -251,6 +256,8 @@ public class MlIndexAndAliasTests extends ESTestCase {
         return ClusterState.builder(ClusterName.DEFAULT)
             .metaData(MetaData.builder()
                 .indices(ImmutableOpenMap.<String, IndexMetaData>builder().putAll(indices).build()).build())
+            .nodes(DiscoveryNodes.builder()
+                .add(new DiscoveryNode("", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), HIDDEN_INTRODUCED_VERSION)))
             .build();
     }
 

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java

@@ -12,6 +12,7 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
@@ -143,6 +144,7 @@ public class JobResultsProvider {
     private static final int RECORDS_SIZE_PARAM = 10000;
     public static final int BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE = 20;
     private static final double ESTABLISHED_MEMORY_CV_THRESHOLD = 0.1;
+    public static final Version HIDDEN_INTRODUCED_VERSION = Version.V_7_7_0;
 
     private final Client client;
     private final Settings settings;
@@ -259,6 +261,8 @@ public class JobResultsProvider {
      * Create the Elasticsearch index and the mappings
      */
     public void createJobResultIndex(Job job, ClusterState state, final ActionListener<Boolean> finalListener) {
+        boolean isHiddenAttributeAvailable = state.nodes().getMinNodeVersion().onOrAfter(HIDDEN_INTRODUCED_VERSION);
+
         Collection<String> termFields = (job.getAnalysisConfig() != null) ? job.getAnalysisConfig().termFields() : Collections.emptyList();
 
         String readAliasName = AnomalyDetectorsIndex.jobResultsAliasedName(job.getId());
@@ -283,15 +287,23 @@ public class JobResultsProvider {
         final String indexName = tempIndexName;
 
         ActionListener<Boolean> indexAndMappingsListener = ActionListener.wrap(success -> {
+            IndicesAliasesRequest.AliasActions addReadAliasAction =
+                IndicesAliasesRequest.AliasActions.add()
+                    .index(indexName)
+                    .alias(readAliasName)
+                    .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId()));
+            IndicesAliasesRequest.AliasActions addWriteAliasAction =
+                IndicesAliasesRequest.AliasActions.add()
+                    .index(indexName)
+                    .alias(writeAliasName);
+            if (isHiddenAttributeAvailable) {
+                addReadAliasAction.isHidden(true);
+                addWriteAliasAction.isHidden(true);
+            }
             final IndicesAliasesRequest request =
                 client.admin().indices().prepareAliases()
-                    .addAliasAction(
+                    .addAliasAction(addReadAliasAction)
-                        IndicesAliasesRequest.AliasActions.add()
+                    .addAliasAction(addWriteAliasAction)
-                            .index(indexName)
-                            .alias(readAliasName)
-                            .isHidden(true)
-                            .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId())))
-                    .addAliasAction(IndicesAliasesRequest.AliasActions.add().index(indexName).alias(writeAliasName).isHidden(true))
                     .request();
             executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request,
                     ActionListener.<AcknowledgedResponse>wrap(r -> finalListener.onResponse(true), finalListener::onFailure),

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java

@@ -14,6 +14,7 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -178,9 +179,12 @@ public class AutodetectProcessManagerTests extends ESTestCase {
                         .build())
                 .build())
             .build();
+        DiscoveryNodes nodes = mock(DiscoveryNodes.class);
+        when(nodes.getMinNodeVersion()).thenReturn(JobResultsProvider.HIDDEN_INTRODUCED_VERSION);
         clusterState = mock(ClusterState.class);
         when(clusterState.getMetaData()).thenReturn(metaData);
         when(clusterState.metaData()).thenReturn(metaData);
+        when(clusterState.nodes()).thenReturn(nodes);
         nativeStorageProvider = mock(NativeStorageProvider.class);
 
         doAnswer(invocationOnMock -> {