diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java index 97c723096a8..73d69acb8dc 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java @@ -69,6 +69,7 @@ import java.util.function.LongSupplier; import java.util.function.Supplier; import static org.apache.lucene.store.BufferedIndexInput.bufferSize; +import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_EXCLUDED_FILE_TYPES_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING; @@ -78,6 +79,7 @@ import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SN import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_UNCACHED_CHUNK_SIZE_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SEARCHABLE_SNAPSHOTS_THREAD_POOL_NAME; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY; /** * Implementation of {@link Directory} that exposes files from a snapshot as a Lucene directory. Because snapshot are immutable this @@ -438,6 +440,20 @@ public class SearchableSnapshotDirectory extends BaseDirectory { ThreadPool threadPool ) throws IOException { + if (SNAPSHOT_REPOSITORY_SETTING.exists(indexSettings.getSettings()) == false + || SNAPSHOT_INDEX_ID_SETTING.exists(indexSettings.getSettings()) == false + || SNAPSHOT_SNAPSHOT_NAME_SETTING.exists(indexSettings.getSettings()) == false + || SNAPSHOT_SNAPSHOT_ID_SETTING.exists(indexSettings.getSettings()) == false) { + + throw new IllegalArgumentException( + "directly setting [" + + INDEX_STORE_TYPE_SETTING.getKey() + + "] to [" + + SNAPSHOT_DIRECTORY_FACTORY_KEY + + "] is not permitted; use the mount snapshot API instead" + ); + } + final Repository repository = repositories.repository(SNAPSHOT_REPOSITORY_SETTING.get(indexSettings.getSettings())); if (repository instanceof BlobStoreRepository == false) { throw new IllegalArgumentException("Repository [" + repository + "] is not searchable"); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index 23adbe5d524..1003e5be1af 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -82,32 +82,38 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng public static final Setting SNAPSHOT_REPOSITORY_SETTING = Setting.simpleString( "index.store.snapshot.repository_name", Setting.Property.IndexScope, - Setting.Property.PrivateIndex + Setting.Property.PrivateIndex, + Setting.Property.NotCopyableOnResize ); public static final Setting SNAPSHOT_SNAPSHOT_NAME_SETTING = Setting.simpleString( "index.store.snapshot.snapshot_name", Setting.Property.IndexScope, - Setting.Property.PrivateIndex + Setting.Property.PrivateIndex, + Setting.Property.NotCopyableOnResize ); public static final Setting SNAPSHOT_SNAPSHOT_ID_SETTING = Setting.simpleString( "index.store.snapshot.snapshot_uuid", Setting.Property.IndexScope, - Setting.Property.PrivateIndex + Setting.Property.PrivateIndex, + Setting.Property.NotCopyableOnResize ); public static final Setting SNAPSHOT_INDEX_ID_SETTING = Setting.simpleString( "index.store.snapshot.index_uuid", Setting.Property.IndexScope, - Setting.Property.PrivateIndex + Setting.Property.PrivateIndex, + Setting.Property.NotCopyableOnResize ); public static final Setting SNAPSHOT_CACHE_ENABLED_SETTING = Setting.boolSetting( "index.store.snapshot.cache.enabled", true, - Setting.Property.IndexScope + Setting.Property.IndexScope, + Setting.Property.NotCopyableOnResize ); public static final Setting SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING = Setting.boolSetting( "index.store.snapshot.cache.prewarm.enabled", true, - Setting.Property.IndexScope + Setting.Property.IndexScope, + Setting.Property.NotCopyableOnResize ); // The file extensions that are excluded from the cache public static final Setting> SNAPSHOT_CACHE_EXCLUDED_FILE_TYPES_SETTING = Setting.listSetting( @@ -115,13 +121,15 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng emptyList(), Function.identity(), Setting.Property.IndexScope, - Setting.Property.NodeScope + Setting.Property.NodeScope, + Setting.Property.NotCopyableOnResize ); public static final Setting SNAPSHOT_UNCACHED_CHUNK_SIZE_SETTING = Setting.byteSizeSetting( "index.store.snapshot.uncached_chunk_size", new ByteSizeValue(-1, ByteSizeUnit.BYTES), Setting.Property.IndexScope, - Setting.Property.NodeScope + Setting.Property.NodeScope, + Setting.Property.NotCopyableOnResize ); private volatile Supplier repositoriesServiceSupplier; diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java index 4f6f738c29e..b18010ef2d9 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java @@ -50,6 +50,7 @@ import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.store.ByteArrayIndexInput; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -99,6 +100,10 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyMap; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_REPOSITORY_SETTING; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -683,6 +688,32 @@ public class SearchableSnapshotDirectoryTests extends ESTestCase { } } + public void testRequiresAdditionalSettings() { + final List> requiredSettings = org.elasticsearch.common.collect.List.of( + SNAPSHOT_REPOSITORY_SETTING, + SNAPSHOT_INDEX_ID_SETTING, + SNAPSHOT_SNAPSHOT_NAME_SETTING, + SNAPSHOT_SNAPSHOT_ID_SETTING + ); + + for (int i = 0; i < requiredSettings.size(); i++) { + final Settings.Builder settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT); + for (int j = 0; j < requiredSettings.size(); j++) { + if (i != j) { + settings.put(requiredSettings.get(j).getKey(), randomAlphaOfLength(10)); + } + } + final IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("test").settings(settings).build(), Settings.EMPTY); + expectThrows( + IllegalArgumentException.class, + () -> SearchableSnapshotDirectory.create(null, null, indexSettings, null, null, null) + ); + } + } + private static void assertThat( String reason, IndexInput actual, @@ -727,4 +758,5 @@ public class SearchableSnapshotDirectoryTests extends ESTestCase { .build() ); } + } diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 4c77f2c9f63..80e3edf756e 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -69,6 +70,7 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT public void testCreateAndRestoreSearchableSnapshot() throws Exception { final String fsRepoName = randomAlphaOfLength(10); final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final String aliasName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); final String restoredIndexName = randomBoolean() ? indexName : randomAlphaOfLength(10).toLowerCase(Locale.ROOT); final String snapshotName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); @@ -84,6 +86,8 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT // Peer recovery always copies .liv files but we do not permit writing to searchable snapshot directories so this doesn't work, but // we can bypass this by forcing soft deletes to be used. TODO this restriction can be lifted when #55142 is resolved. assertAcked(prepareCreate(indexName, Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true))); + assertAcked(client().admin().indices().prepareAliases().addAlias(indexName, aliasName)); + final List indexRequestBuilders = new ArrayList<>(); for (int i = between(10, 10_000); i >= 0; i--) { indexRequestBuilders.add(client().prepareIndex(indexName, "_doc").setSource("foo", randomBoolean() ? "bar" : "baz")); @@ -178,8 +182,14 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT assertRecovered(restoredIndexName, originalAllHits, originalBarHits); assertSearchableSnapshotStats(restoredIndexName, cacheEnabled, nonCachedExtensions); + assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(0)); + assertAcked(client().admin().indices().prepareAliases().addAlias(restoredIndexName, aliasName)); + assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(1)); + assertRecovered(aliasName, originalAllHits, originalBarHits, false); + internalCluster().fullRestart(); assertRecovered(restoredIndexName, originalAllHits, originalBarHits); + assertRecovered(aliasName, originalAllHits, originalBarHits, false); assertSearchableSnapshotStats(restoredIndexName, cacheEnabled, nonCachedExtensions); internalCluster().ensureAtLeastNumDataNodes(2); @@ -217,6 +227,46 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT assertRecovered(restoredIndexName, originalAllHits, originalBarHits); assertSearchableSnapshotStats(restoredIndexName, cacheEnabled, nonCachedExtensions); + + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings(restoredIndexName) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .putNull(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey()) + ) + ); + + final String clonedIndexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + assertAcked( + client().admin() + .indices() + .prepareResizeIndex(restoredIndexName, clonedIndexName) + .setResizeType(ResizeType.CLONE) + .setSettings(Settings.builder().putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()).build()) + ); + ensureGreen(clonedIndexName); + assertRecovered(clonedIndexName, originalAllHits, originalBarHits, false); + + final Settings clonedIndexSettings = client().admin() + .indices() + .prepareGetSettings(clonedIndexName) + .get() + .getIndexToSettings() + .get(clonedIndexName); + assertFalse(clonedIndexSettings.hasValue(IndexModule.INDEX_STORE_TYPE_SETTING.getKey())); + assertFalse(clonedIndexSettings.hasValue(SearchableSnapshots.SNAPSHOT_REPOSITORY_SETTING.getKey())); + assertFalse(clonedIndexSettings.hasValue(SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING.getKey())); + assertFalse(clonedIndexSettings.hasValue(SearchableSnapshots.SNAPSHOT_SNAPSHOT_ID_SETTING.getKey())); + assertFalse(clonedIndexSettings.hasValue(SearchableSnapshots.SNAPSHOT_INDEX_ID_SETTING.getKey())); + + assertAcked(client().admin().indices().prepareDelete(restoredIndexName)); + assertThat(client().admin().indices().prepareGetAliases(aliasName).get().getAliases().size(), equalTo(0)); + assertAcked(client().admin().indices().prepareAliases().addAlias(clonedIndexName, aliasName)); + assertRecovered(aliasName, originalAllHits, originalBarHits, false); + } public void testCanMountSnapshotTakenWhileConcurrentlyIndexing() throws Exception { @@ -376,6 +426,12 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT } private void assertRecovered(String indexName, TotalHits originalAllHits, TotalHits originalBarHits) throws Exception { + assertRecovered(indexName, originalAllHits, originalBarHits, true); + } + + private void assertRecovered(String indexName, TotalHits originalAllHits, TotalHits originalBarHits, boolean checkRecoveryStats) + throws Exception { + final Thread[] threads = new Thread[between(1, 5)]; final AtomicArray allHits = new AtomicArray<>(threads.length); final AtomicArray barHits = new AtomicArray<>(threads.length); @@ -406,12 +462,17 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT ensureGreen(indexName); latch.countDown(); - final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName).get(); - for (List recoveryStates : recoveryResponse.shardRecoveryStates().values()) { - for (RecoveryState recoveryState : recoveryStates) { - logger.info("Checking {}[{}]", recoveryState.getShardId(), recoveryState.getPrimary() ? "p" : "r"); - assertThat(recoveryState.getIndex().recoveredFileCount(), lessThanOrEqualTo(1)); // we make a new commit so we write a new - // `segments_n` file + if (checkRecoveryStats) { + final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName).get(); + for (List recoveryStates : recoveryResponse.shardRecoveryStates().values()) { + for (RecoveryState recoveryState : recoveryStates) { + logger.info("Checking {}[{}]", recoveryState.getShardId(), recoveryState.getPrimary() ? "p" : "r"); + assertThat( + Strings.toString(recoveryState), // we make a new commit so we write a new `segments_n` file + recoveryState.getIndex().recoveredFileCount(), + lessThanOrEqualTo(1) + ); + } } }