Default to zero replicas for searchable snapshots (#57802)

Today a mounted searchable snapshot defaults to having the same replica
configuration as the index that was snapshotted. This commit changes this
behaviour so that we default to zero replicas on these indices, but allow the
user to override this in the mount request.

Relates #50999

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java

@@ -169,6 +169,8 @@ public class TransportMountSearchableSnapshotAction extends TransportMasterNodeA
                         // Pass through index settings, adding the index-level settings required to use searchable snapshots
                         .indexSettings(
                             Settings.builder()
+                                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) // can be overridden
+                                .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, false) // can be overridden
                                 .put(request.indexSettings())
                                 .put(buildIndexSettings(request.repositoryName(), snapshotId, indexId))
                                 .build()

x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java

@@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotR
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -153,6 +154,13 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT
                 new ByteSizeValue(randomLongBetween(10, 100_000))
             );
         }
+        final int expectedReplicas;
+        if (randomBoolean()) {
+            expectedReplicas = numberOfReplicas();
+            indexSettingsBuilder.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, expectedReplicas);
+        } else {
+            expectedReplicas = 0;
+        }
         final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
             restoredIndexName,
             fsRepoName,
@@ -178,6 +186,8 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT
         assertTrue(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(settings));
         assertTrue(SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING.exists(settings));
         assertTrue(SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING.exists(settings));
+        assertThat(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.get(settings).toString(), equalTo("false"));
+        assertThat(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(settings), equalTo(expectedReplicas));
 
         assertRecovered(restoredIndexName, originalAllHits, originalBarHits);
         assertSearchableSnapshotStats(restoredIndexName, cacheEnabled, nonCachedExtensions);
@@ -425,6 +435,152 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT
         }
     }
 
+    public void testMountedSnapshotHasNoReplicasByDefault() throws Exception {
+        final String fsRepoName = randomAlphaOfLength(10);
+        final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        final String restoredIndexName = randomBoolean() ? indexName : randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        final String snapshotName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+
+        final Path repo = randomRepoPath();
+        assertAcked(
+            client().admin().cluster().preparePutRepository(fsRepoName).setType("fs").setSettings(Settings.builder().put("location", repo))
+        );
+
+        final int dataNodesCount = internalCluster().numDataNodes();
+        final Settings.Builder originalIndexSettings = Settings.builder();
+        originalIndexSettings.put(INDEX_SOFT_DELETES_SETTING.getKey(), true);
+        if (randomBoolean()) {
+            originalIndexSettings.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas());
+        }
+        if (randomBoolean()) {
+            final int replicaLimit = between(0, dataNodesCount);
+            originalIndexSettings.put(
+                IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS,
+                replicaLimit == dataNodesCount ? "0-all" : "0-" + replicaLimit
+            );
+        }
+        assertAcked(prepareCreate(indexName, originalIndexSettings));
+        ensureGreen(indexName);
+
+        final List<IndexRequestBuilder> indexRequestBuilders = new ArrayList<>();
+        for (int i = between(10, 100); i >= 0; i--) {
+            indexRequestBuilders.add(client().prepareIndex(indexName, "_doc").setSource("foo", randomBoolean() ? "bar" : "baz"));
+        }
+        indexRandom(true, true, indexRequestBuilders);
+        refresh(indexName);
+        assertThat(
+            client().admin().indices().prepareForceMerge(indexName).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(),
+            equalTo(0)
+        );
+
+        CreateSnapshotResponse createSnapshotResponse = client().admin()
+            .cluster()
+            .prepareCreateSnapshot(fsRepoName, snapshotName)
+            .setWaitForCompletion(true)
+            .get();
+        final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
+        assertThat(snapshotInfo.successfulShards(), greaterThan(0));
+        assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
+
+        assertAcked(client().admin().indices().prepareDelete(indexName));
+
+        {
+            logger.info("--> restoring index [{}] with default replica counts", restoredIndexName);
+            Settings.Builder indexSettingsBuilder = Settings.builder()
+                .put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true)
+                .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), Boolean.FALSE.toString());
+            final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+                restoredIndexName,
+                fsRepoName,
+                snapshotName,
+                indexName,
+                indexSettingsBuilder.build(),
+                Strings.EMPTY_ARRAY,
+                true
+            );
+
+            final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, req).get();
+            assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
+            ensureGreen(restoredIndexName);
+
+            final Settings settings = client().admin()
+                .indices()
+                .prepareGetSettings(restoredIndexName)
+                .get()
+                .getIndexToSettings()
+                .get(restoredIndexName);
+            assertThat(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(settings), equalTo(0));
+            assertThat(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.get(settings).toString(), equalTo("false"));
+
+            assertAcked(client().admin().indices().prepareDelete(restoredIndexName));
+        }
+
+        {
+            final int replicaCount = numberOfReplicas();
+
+            logger.info("--> restoring index [{}] with specific replica count", restoredIndexName);
+            Settings.Builder indexSettingsBuilder = Settings.builder()
+                .put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true)
+                .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), Boolean.FALSE.toString())
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicaCount);
+            final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+                restoredIndexName,
+                fsRepoName,
+                snapshotName,
+                indexName,
+                indexSettingsBuilder.build(),
+                Strings.EMPTY_ARRAY,
+                true
+            );
+
+            final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, req).get();
+            assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
+            ensureGreen(restoredIndexName);
+
+            final Settings settings = client().admin()
+                .indices()
+                .prepareGetSettings(restoredIndexName)
+                .get()
+                .getIndexToSettings()
+                .get(restoredIndexName);
+            assertThat(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(settings), equalTo(replicaCount));
+            assertThat(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.get(settings).toString(), equalTo("false"));
+
+            assertAcked(client().admin().indices().prepareDelete(restoredIndexName));
+        }
+
+        {
+            final int replicaLimit = between(0, dataNodesCount);
+            logger.info("--> restoring index [{}] with auto-expand replicas configured", restoredIndexName);
+            Settings.Builder indexSettingsBuilder = Settings.builder()
+                .put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true)
+                .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), Boolean.FALSE.toString())
+                .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, replicaLimit == dataNodesCount ? "0-all" : "0-" + replicaLimit);
+            final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
+                restoredIndexName,
+                fsRepoName,
+                snapshotName,
+                indexName,
+                indexSettingsBuilder.build(),
+                Strings.EMPTY_ARRAY,
+                true
+            );
+
+            final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, req).get();
+            assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
+            ensureGreen(restoredIndexName);
+
+            final ClusterState state = client().admin().cluster().prepareState().clear().setRoutingTable(true).get().getState();
+            assertThat(
+                state.toString(),
+                state.routingTable().index(restoredIndexName).shard(0).size(),
+                equalTo(Math.min(replicaLimit + 1, dataNodesCount))
+            );
+
+            assertAcked(client().admin().indices().prepareDelete(restoredIndexName));
+        }
+    }
+
     private void assertRecovered(String indexName, TotalHits originalAllHits, TotalHits originalBarHits) throws Exception {
         assertRecovered(indexName, originalAllHits, originalBarHits, true);
     }
@@ -588,4 +744,5 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT
             }
         }
     }
+
 }