From f775cf859470115ee0776b24387abbd7a723d94c Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Mon, 21 Sep 2020 12:06:47 +0200 Subject: [PATCH] Add test for snapshot incrementality of snapshot-backed indices (#62641) This commit adds a test that verifies that snapshots incrementality is respected when a snapshot-backed index is snapshotted. This test mounts a snapshot as a snapshot-backed index, creates a new snapshot from it and then verifies that no new data blobs were added to the repository. --- .../BaseSearchableSnapshotsIntegTestCase.java | 20 ++++++-- .../SearchableSnapshotsIntegTests.java | 48 +++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java index bebeac728be..86bf146a3c7 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java @@ -139,14 +139,29 @@ public abstract class BaseSearchableSnapshotsIntegTestCase extends ESIntegTestCa return snapshotInfo.snapshotId(); } - protected String mountSnapshot(String repositoryName, String snapshotName, String indexName, Settings indexSettings) throws Exception { + protected String mountSnapshot(String repositoryName, String snapshotName, String indexName, Settings restoredIndexSettings) + throws Exception { final String restoredIndexName = randomBoolean() ? indexName : randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + mountSnapshot(repositoryName, snapshotName, indexName, restoredIndexName, restoredIndexSettings); + return restoredIndexName; + } + + protected void mountSnapshot( + String repositoryName, + String snapshotName, + String indexName, + String restoredIndexName, + Settings restoredIndexSettings + ) throws Exception { final MountSearchableSnapshotRequest mountRequest = new MountSearchableSnapshotRequest( restoredIndexName, repositoryName, snapshotName, indexName, - Settings.builder().put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), Boolean.FALSE.toString()).put(indexSettings).build(), + Settings.builder() + .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), Boolean.FALSE.toString()) + .put(restoredIndexSettings) + .build(), Strings.EMPTY_ARRAY, true ); @@ -154,7 +169,6 @@ public abstract class BaseSearchableSnapshotsIntegTestCase extends ESIntegTestCa final RestoreSnapshotResponse restoreResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, mountRequest).get(); assertThat(restoreResponse.getRestoreInfo().successfulShards(), equalTo(getNumShards(restoredIndexName).numPrimaries)); assertThat(restoreResponse.getRestoreInfo().failedShards(), equalTo(0)); - return restoredIndexName; } protected void createRepo(String fsRepoName) { diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 68e01d979ed..5a9a45da93c 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -11,6 +11,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; @@ -39,6 +40,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest; @@ -660,6 +662,52 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT } } + public void testSnapshotMountedIndexLeavesBlobsUntouched() throws Exception { + final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final int numShards = between(1, 3); + createAndPopulateIndex( + indexName, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(INDEX_SOFT_DELETES_SETTING.getKey(), true) + ); + ensureGreen(indexName); + forceMerge(); + + final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final Path repositoryLocation = randomRepoPath(); + createFsRepository(repositoryName, repositoryLocation); + + final SnapshotId snapshotOne = createSnapshot(repositoryName, Collections.singletonList(indexName)); + assertAcked(client().admin().indices().prepareDelete(indexName)); + + final SnapshotStatus snapshotOneStatus = client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(snapshotOne.getName()) + .get() + .getSnapshots() + .get(0); + final int snapshotOneTotalFileCount = snapshotOneStatus.getStats().getTotalFileCount(); + assertThat(snapshotOneTotalFileCount, greaterThan(0)); + + mountSnapshot(repositoryName, snapshotOne.getName(), indexName, indexName, Settings.EMPTY); + ensureGreen(indexName); + + final SnapshotId snapshotTwo = createSnapshot(repositoryName, Collections.singletonList(indexName)); + final SnapshotStatus snapshotTwoStatus = client().admin() + .cluster() + .prepareSnapshotStatus(repositoryName) + .setSnapshots(snapshotTwo.getName()) + .get() + .getSnapshots() + .get(0); + assertThat(snapshotTwoStatus.getStats().getTotalFileCount(), equalTo(snapshotOneTotalFileCount)); + assertThat(snapshotTwoStatus.getStats().getIncrementalFileCount(), equalTo(numShards)); // one segment_N per shard + assertThat(snapshotTwoStatus.getStats().getProcessedFileCount(), equalTo(numShards)); // one segment_N per shard + } + private void assertTotalHits(String indexName, TotalHits originalAllHits, TotalHits originalBarHits) throws Exception { final Thread[] threads = new Thread[between(1, 5)]; final AtomicArray allHits = new AtomicArray<>(threads.length);