Prevent snapshots to be mounted as system indices (#61517) (#61727)

System indices can be snapshotted and are therefore potential candidates 
to be mounted as searchable snapshot indices. As of today nothing 
prevents a snapshot to be mounted under an index name starting with . 
and this can lead to conflicting situations because searchable snapshot 
indices are read-only and Elasticsearch expects some system indices 
to be writable; because searchable snapshot indices will soon use an 
internal system index (#60522) to speed up recoveries and we should
prevent the system index to be itself a searchable snapshot index 
(leading to some deadlock situation for recovery).

This commit introduces a changes to prevent snapshots to be mounted 
as a system index.

server/src/main/java/org/elasticsearch/node/Node.java

@@ -673,6 +673,7 @@ public class Node implements Closeable {
                     b.bind(RerouteService.class).toInstance(rerouteService);
                     b.bind(ShardLimitValidator.class).toInstance(shardLimitValidator);
                     b.bind(FsHealthService.class).toInstance(fsHealthService);
+                    b.bind(SystemIndices.class).toInstance(systemIndices);
                 }
             );
             injector = modules.createInjector();

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java

@@ -65,6 +65,7 @@ import org.elasticsearch.plugins.PersistentTaskPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.plugins.SystemIndexPlugin;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.rest.RestController;
@@ -102,7 +103,8 @@ import java.util.stream.Collectors;
 import static java.util.stream.Collectors.toList;
 
 public class LocalStateCompositeXPackPlugin extends XPackPlugin implements ScriptPlugin, ActionPlugin, IngestPlugin, NetworkPlugin,
-        ClusterPlugin, DiscoveryPlugin, MapperPlugin, AnalysisPlugin, PersistentTaskPlugin, EnginePlugin, IndexStorePlugin {
+        ClusterPlugin, DiscoveryPlugin, MapperPlugin, AnalysisPlugin, PersistentTaskPlugin, EnginePlugin, IndexStorePlugin,
+        SystemIndexPlugin {
 
     private XPackLicenseState licenseState;
     private SSLService sslService;

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java

@@ -24,6 +24,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoriesService;
@@ -62,6 +63,7 @@ public class TransportMountSearchableSnapshotAction extends TransportMasterNodeA
     private final Client client;
     private final RepositoriesService repositoriesService;
     private final XPackLicenseState licenseState;
+    private final SystemIndices systemIndices;
 
     @Inject
     public TransportMountSearchableSnapshotAction(
@@ -72,7 +74,8 @@ public class TransportMountSearchableSnapshotAction extends TransportMasterNodeA
         RepositoriesService repositoriesService,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver,
-        XPackLicenseState licenseState
+        XPackLicenseState licenseState,
+        SystemIndices systemIndices
     ) {
         super(
             MountSearchableSnapshotAction.NAME,
@@ -86,6 +89,7 @@ public class TransportMountSearchableSnapshotAction extends TransportMasterNodeA
         this.client = client;
         this.repositoriesService = repositoriesService;
         this.licenseState = Objects.requireNonNull(licenseState);
+        this.systemIndices = Objects.requireNonNull(systemIndices);
     }
 
     @Override
@@ -130,6 +134,11 @@ public class TransportMountSearchableSnapshotAction extends TransportMasterNodeA
     ) {
         SearchableSnapshots.ensureValidLicense(licenseState);
 
+        final String mountedIndexName = request.mountedIndexName();
+        if (systemIndices.isSystemIndex(mountedIndexName)) {
+            throw new ElasticsearchException("system index [{}] cannot be mounted as searchable snapshots", mountedIndexName);
+        }
+
         final String repoName = request.repositoryName();
         final String snapName = request.snapshotName();
         final String indexName = request.snapshotIndexName();
@@ -166,7 +175,7 @@ public class TransportMountSearchableSnapshotAction extends TransportMasterNodeA
                         .indices(indexName)
                         // Always rename it to the desired mounted index name
                         .renamePattern(".+")
-                        .renameReplacement(request.mountedIndexName())
+                        .renameReplacement(mountedIndexName)
                         // Pass through index settings, adding the index-level settings required to use searchable snapshots
                         .indexSettings(
                             Settings.builder()

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/LocalStateSearchableSnapshots.java

@@ -7,24 +7,32 @@
 package org.elasticsearch.xpack.searchablesnapshots;
 
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 
 import java.nio.file.Path;
+import java.util.Collection;
 
 public class LocalStateSearchableSnapshots extends LocalStateCompositeXPackPlugin {
 
+    private final SearchableSnapshots plugin;
+
     public LocalStateSearchableSnapshots(final Settings settings, final Path configPath) {
         super(settings, configPath);
-        LocalStateSearchableSnapshots thisVar = this;
-
-        plugins.add(new SearchableSnapshots(settings) {
+        this.plugin = new SearchableSnapshots(settings) {
 
             @Override
             protected XPackLicenseState getLicenseState() {
-                return thisVar.getLicenseState();
+                return LocalStateSearchableSnapshots.this.getLicenseState();
             }
 
-        });
+        };
+        plugins.add(plugin);
+    }
+
+    @Override
+    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+        return plugin.getSystemIndexDescriptors(settings);
     }
 }

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsSystemIndicesIntegTests.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.searchablesnapshots;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.OriginSettingClient;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.SystemIndexDescriptor;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SystemIndexPlugin;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction;
+import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class SearchableSnapshotsSystemIndicesIntegTests extends BaseSearchableSnapshotsIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        final List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(TestSystemIndexPlugin.class);
+        return plugins;
+    }
+
+    public void testCannotMountSystemIndex() throws Exception {
+        executeTest(TestSystemIndexPlugin.INDEX_NAME, new OriginSettingClient(client(), ClientHelper.SEARCHABLE_SNAPSHOTS_ORIGIN));
+    }
+
+    public void testCannotMountSnapshotBlobCacheIndex() throws Exception {
+        executeTest(SearchableSnapshotsConstants.SNAPSHOT_BLOB_CACHE_INDEX, client());
+    }
+
+    private void executeTest(final String indexName, final Client client) throws Exception {
+        final boolean isHidden = randomBoolean();
+        createAndPopulateIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, isHidden));
+
+        final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        createRepo(repositoryName);
+
+        final String snapshotName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        final CreateSnapshotResponse snapshotResponse = client.admin()
+            .cluster()
+            .prepareCreateSnapshot(repositoryName, snapshotName)
+            .setIndices(indexName)
+            .setWaitForCompletion(true)
+            .get();
+
+        final int numPrimaries = getNumShards(indexName).numPrimaries;
+        assertThat(snapshotResponse.getSnapshotInfo().successfulShards(), equalTo(numPrimaries));
+        assertThat(snapshotResponse.getSnapshotInfo().failedShards(), equalTo(0));
+
+        if (randomBoolean()) {
+            assertAcked(client.admin().indices().prepareClose(indexName));
+        } else {
+            assertAcked(client.admin().indices().prepareDelete(indexName));
+        }
+
+        final MountSearchableSnapshotRequest mountRequest = new MountSearchableSnapshotRequest(
+            indexName,
+            repositoryName,
+            snapshotName,
+            indexName,
+            Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, randomBoolean()).build(),
+            Strings.EMPTY_ARRAY,
+            true
+        );
+
+        final ElasticsearchException exception = expectThrows(
+            ElasticsearchException.class,
+            () -> client.execute(MountSearchableSnapshotAction.INSTANCE, mountRequest).actionGet()
+        );
+        assertThat(exception.getMessage(), containsString("system index [" + indexName + "] cannot be mounted as searchable snapshots"));
+    }
+
+    public static class TestSystemIndexPlugin extends Plugin implements SystemIndexPlugin {
+
+        static final String INDEX_NAME = ".test-system-index";
+
+        @Override
+        public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
+            return org.elasticsearch.common.collect.List.of(
+                new SystemIndexDescriptor(INDEX_NAME, "System index for [" + getTestClass().getName() + ']')
+            );
+        }
+    }
+}