honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Default to zero replicas for searchable snapshots (#57802) 

Today a mounted searchable snapshot defaults to having the same replica
configuration as the index that was snapshotted. This commit changes this
behaviour so that we default to zero replicas on these indices, but allow the
user to override this in the mount request.

Relates #50999
This commit is contained in:
David Turner 2020-06-16 09:46:57 +01:00
parent e046b0a8fa
commit 423697f414
2 changed files with 159 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java
View File

@ -169,6 +169,8 @@ public class TransportMountSearchableSnapshotAction extends TransportMasterNodeA
// Pass through index settings, adding the index-level settings required to use searchable snapshots
.indexSettings(
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) // can be overridden
.put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, false) // can be overridden
.put(request.indexSettings())
.put(buildIndexSettings(request.repositoryName(), snapshotId, indexId))
.build()

157
x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
View File

@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotR
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.admin.indices.shrink.ResizeType;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRouting;
@ -153,6 +154,13 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT
new ByteSizeValue(randomLongBetween(10, 100_000))
);
}
final int expectedReplicas;
if (randomBoolean()) {
expectedReplicas = numberOfReplicas();
indexSettingsBuilder.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, expectedReplicas);
} else {
expectedReplicas = 0;
}
final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
restoredIndexName,
fsRepoName,
@ -178,6 +186,8 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT
assertTrue(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(settings));
assertTrue(SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING.exists(settings));
assertTrue(SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING.exists(settings));
assertThat(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.get(settings).toString(), equalTo("false"));
assertThat(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(settings), equalTo(expectedReplicas));
assertRecovered(restoredIndexName, originalAllHits, originalBarHits);
assertSearchableSnapshotStats(restoredIndexName, cacheEnabled, nonCachedExtensions);
@ -425,6 +435,152 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT
}
}
public void testMountedSnapshotHasNoReplicasByDefault() throws Exception {
final String fsRepoName = randomAlphaOfLength(10);
final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
final String restoredIndexName = randomBoolean() ? indexName : randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
final String snapshotName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
final Path repo = randomRepoPath();
assertAcked(
client().admin().cluster().preparePutRepository(fsRepoName).setType("fs").setSettings(Settings.builder().put("location", repo))
);
final int dataNodesCount = internalCluster().numDataNodes();
final Settings.Builder originalIndexSettings = Settings.builder();
originalIndexSettings.put(INDEX_SOFT_DELETES_SETTING.getKey(), true);
if (randomBoolean()) {
originalIndexSettings.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas());
}
if (randomBoolean()) {
final int replicaLimit = between(0, dataNodesCount);
originalIndexSettings.put(
IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS,
replicaLimit == dataNodesCount ? "0-all" : "0-" + replicaLimit
);
}
assertAcked(prepareCreate(indexName, originalIndexSettings));
ensureGreen(indexName);
final List<IndexRequestBuilder> indexRequestBuilders = new ArrayList<>();
for (int i = between(10, 100); i >= 0; i--) {
indexRequestBuilders.add(client().prepareIndex(indexName, "_doc").setSource("foo", randomBoolean() ? "bar" : "baz"));
}
indexRandom(true, true, indexRequestBuilders);
refresh(indexName);
assertThat(
client().admin().indices().prepareForceMerge(indexName).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(),
equalTo(0)
);
CreateSnapshotResponse createSnapshotResponse = client().admin()
.cluster()
.prepareCreateSnapshot(fsRepoName, snapshotName)
.setWaitForCompletion(true)
.get();
final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
assertThat(snapshotInfo.successfulShards(), greaterThan(0));
assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
assertAcked(client().admin().indices().prepareDelete(indexName));
{
logger.info("--> restoring index [{}] with default replica counts", restoredIndexName);
Settings.Builder indexSettingsBuilder = Settings.builder()
.put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true)
.put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), Boolean.FALSE.toString());
final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
restoredIndexName,
fsRepoName,
snapshotName,
indexName,
indexSettingsBuilder.build(),
Strings.EMPTY_ARRAY,
true
);
final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, req).get();
assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
ensureGreen(restoredIndexName);
final Settings settings = client().admin()
.indices()
.prepareGetSettings(restoredIndexName)
.get()
.getIndexToSettings()
.get(restoredIndexName);
assertThat(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(settings), equalTo(0));
assertThat(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.get(settings).toString(), equalTo("false"));
assertAcked(client().admin().indices().prepareDelete(restoredIndexName));
}
{
final int replicaCount = numberOfReplicas();
logger.info("--> restoring index [{}] with specific replica count", restoredIndexName);
Settings.Builder indexSettingsBuilder = Settings.builder()
.put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true)
.put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), Boolean.FALSE.toString())
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicaCount);
final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
restoredIndexName,
fsRepoName,
snapshotName,
indexName,
indexSettingsBuilder.build(),
Strings.EMPTY_ARRAY,
true
);
final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, req).get();
assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
ensureGreen(restoredIndexName);
final Settings settings = client().admin()
.indices()
.prepareGetSettings(restoredIndexName)
.get()
.getIndexToSettings()
.get(restoredIndexName);
assertThat(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(settings), equalTo(replicaCount));
assertThat(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.get(settings).toString(), equalTo("false"));
assertAcked(client().admin().indices().prepareDelete(restoredIndexName));
}
{
final int replicaLimit = between(0, dataNodesCount);
logger.info("--> restoring index [{}] with auto-expand replicas configured", restoredIndexName);
Settings.Builder indexSettingsBuilder = Settings.builder()
.put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true)
.put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), Boolean.FALSE.toString())
.put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, replicaLimit == dataNodesCount ? "0-all" : "0-" + replicaLimit);
final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest(
restoredIndexName,
fsRepoName,
snapshotName,
indexName,
indexSettingsBuilder.build(),
Strings.EMPTY_ARRAY,
true
);
final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, req).get();
assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
ensureGreen(restoredIndexName);
final ClusterState state = client().admin().cluster().prepareState().clear().setRoutingTable(true).get().getState();
assertThat(
state.toString(),
state.routingTable().index(restoredIndexName).shard(0).size(),
equalTo(Math.min(replicaLimit + 1, dataNodesCount))
);
assertAcked(client().admin().indices().prepareDelete(restoredIndexName));
}
}
private void assertRecovered(String indexName, TotalHits originalAllHits, TotalHits originalBarHits) throws Exception {
assertRecovered(indexName, originalAllHits, originalBarHits, true);
}
@ -588,4 +744,5 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT
}
}
}
}