From 62eabdac6e52ac6f4342c18d4782baab8fea140b Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 6 Jul 2020 12:26:42 +0200 Subject: [PATCH] Dry up Snapshot ITs further (#59035) (#59052) Some more obvious cleaning up of the snapshot ITs. follow up to #58818 --- .../discovery/SnapshotDisruptionIT.java | 12 +- .../BlobStoreRepositoryCleanupIT.java | 8 +- .../snapshots/BlobStoreIncrementalityIT.java | 15 +- .../DedicatedClusterSnapshotRestoreIT.java | 197 ++----- ...etadataLoadingDuringSnapshotRestoreIT.java | 2 +- .../snapshots/RepositoriesIT.java | 2 +- .../SharedClusterSnapshotRestoreIT.java | 540 ++++-------------- .../snapshots/SnapshotShardsServiceIT.java | 13 +- .../snapshots/SnapshotStatusApisIT.java | 12 +- .../AbstractSnapshotIntegTestCase.java | 44 ++ 10 files changed, 225 insertions(+), 620 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java index bc9a65c8f91..d57aff18d05 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java @@ -165,10 +165,7 @@ public class SnapshotDisruptionIT extends AbstractSnapshotIntegTestCase { createRandomIndex(idxName); - createRepository("test-repo", "fs", Settings.builder() - .put("location", randomRepoPath()) - .put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "fs"); final String masterNode1 = internalCluster().getMasterName(); @@ -248,7 +245,7 @@ public class SnapshotDisruptionIT extends AbstractSnapshotIntegTestCase { index(idxName, "type", JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject()); final String repoName = "test-repo"; - createRepository(repoName, "mock", randomRepoPath()); + createRepository(repoName, "mock"); final String masterNode = internalCluster().getMasterName(); @@ -304,7 +301,7 @@ public class SnapshotDisruptionIT extends AbstractSnapshotIntegTestCase { final String dataNode = internalCluster().startDataOnlyNode(); ensureStableCluster(4); final String repoName = "test-repo"; - createRepository(repoName, "mock", randomRepoPath()); + createRepository(repoName, "mock"); final String indexName = "index-one"; createIndex(indexName); @@ -363,8 +360,7 @@ public class SnapshotDisruptionIT extends AbstractSnapshotIntegTestCase { } private void createRandomIndex(String idxName) throws InterruptedException { - assertAcked(prepareCreate(idxName, 0, Settings.builder().put("number_of_shards", between(1, 20)) - .put("number_of_replicas", 0))); + assertAcked(prepareCreate(idxName, 0, indexSettingsNoReplicas(between(1, 5)))); logger.info("--> indexing some data"); final int numdocs = randomIntBetween(10, 100); IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs]; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java index bd14b82ca54..fc4ac0d0d63 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRes import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.RepositoryCleanupInProgress; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; @@ -81,9 +80,7 @@ public class BlobStoreRepositoryCleanupIT extends AbstractSnapshotIntegTestCase internalCluster().startMasterOnlyNodes(2); internalCluster().startDataOnlyNodes(1); - createRepository(repoName, "mock", Settings.builder() - .put("location", randomRepoPath()).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository(repoName, "mock"); logger.info("--> snapshot"); client().admin().cluster().prepareCreateSnapshot(repoName, "test-snap") @@ -112,8 +109,7 @@ public class BlobStoreRepositoryCleanupIT extends AbstractSnapshotIntegTestCase internalCluster().startNodes(Settings.EMPTY); final String repoName = "test-repo"; - createRepository(repoName, "fs", Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository(repoName, "fs"); logger.info("--> create three snapshots"); for (int i = 0; i < 3; ++i) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java index 0493c79f704..9eb5a9caac5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/BlobStoreIncrementalityIT.java @@ -28,11 +28,9 @@ import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; import java.io.IOException; @@ -78,7 +76,7 @@ public class BlobStoreIncrementalityIT extends AbstractSnapshotIntegTestCase { final String snapshot1 = "snap-1"; final String repo = "test-repo"; - createRepository(repo, "fs", randomRepoPath()); + createRepository(repo, "fs"); logger.info("--> creating snapshot 1"); client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setIndices(indexName).setWaitForCompletion(true).get(); @@ -152,7 +150,7 @@ public class BlobStoreIncrementalityIT extends AbstractSnapshotIntegTestCase { final String snapshot1 = "snap-1"; final String repo = "test-repo"; - createRepository(repo, "fs", randomRepoPath()); + createRepository(repo, "fs"); logger.info("--> creating snapshot 1"); client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setIndices(indexName).setWaitForCompletion(true).get(); @@ -172,9 +170,9 @@ public class BlobStoreIncrementalityIT extends AbstractSnapshotIntegTestCase { assertThat(secondSnapshotShardStatus.getIncrementalFileCount(), greaterThan(0)); } - private void assertCountInIndexThenDelete(String index, long expectedCount) throws ExecutionException, InterruptedException { + private void assertCountInIndexThenDelete(String index, long expectedCount) { logger.info("--> asserting that index [{}] contains [{}] documents", index, expectedCount); - assertThat(getCountForIndex(index), is(expectedCount)); + assertDocCount(index, expectedCount); logger.info("--> deleting index [{}]", index); assertThat(client().admin().indices().prepareDelete(index).get().isAcknowledged(), is(true)); } @@ -203,9 +201,4 @@ public class BlobStoreIncrementalityIT extends AbstractSnapshotIntegTestCase { assertThat(restoreInfo.totalShards(), is(1)); assertThat(restoreInfo.failedShards(), is(0)); } - - private long getCountForIndex(String indexName) throws ExecutionException, InterruptedException { - return client().search(new SearchRequest(new SearchRequest(indexName).source( - new SearchSourceBuilder().size(0).trackTotalHits(true)))).get().getHits().getTotalHits().value; - } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index c3d7801508e..7e33d11d1db 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -45,7 +45,6 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -62,7 +61,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -117,7 +115,6 @@ import static org.elasticsearch.index.seqno.RetentionLeaseActions.RETAIN_ALL; import static org.elasticsearch.test.NodeRoles.nonMasterNode; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFutureThrows; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertRequestBuilderThrows; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; @@ -225,7 +222,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest setSettingValue.accept("new value"); assertSettingValue.accept("new value"); - createRepository("test-repo", "fs", randomRepoPath()); + createRepository("test-repo", "fs"); createFullSnapshot("test-repo", "test-snap"); assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet() .getSnapshots().get(0).state(), @@ -372,17 +369,9 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest logger.info("--> start 2 nodes"); Client client = client(); - assertAcked(prepareCreate("test-idx", 2, Settings.builder().put("number_of_shards", 2) - .put("number_of_replicas", 0))); + assertAcked(prepareCreate("test-idx", 2, indexSettingsNoReplicas(2))); ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); - + indexRandomDocs("test-idx", 100); final Path repoPath = randomRepoPath(); createRepository("test-repo", "mock", Settings.builder().put("location", repoPath).put("random", randomAlphaOfLength(10)) @@ -419,16 +408,9 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest nodes.add(internalCluster().startNode()); Client client = client(); - assertAcked(prepareCreate("test-idx", 2, Settings.builder().put("number_of_shards", 2) - .put("number_of_replicas", 0))); + assertAcked(prepareCreate("test-idx", 2, indexSettingsNoReplicas(2))); ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + indexRandomDocs("test-idx", 100); Path repo = randomRepoPath(); createRepository("test-repo", "mock", @@ -490,16 +472,9 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest cluster().wipeIndices("_all"); logger.info("--> create an index that will have some unallocated shards"); - assertAcked(prepareCreate("test-idx-some", 2, Settings.builder().put("number_of_shards", 6) - .put("number_of_replicas", 0))); + assertAcked(prepareCreate("test-idx-some", 2, indexSettingsNoReplicas(6))); ensureGreen(); - - logger.info("--> indexing some data into test-idx-some"); - for (int i = 0; i < 100; i++) { - index("test-idx-some", "doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client().prepareSearch("test-idx-some").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + indexRandomDocs("test-idx-some", 100); logger.info("--> shutdown one of the nodes"); internalCluster().stopRandomDataNode(); @@ -508,8 +483,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest equalTo(false)); logger.info("--> create an index that will have all allocated shards"); - assertAcked(prepareCreate("test-idx-all", 1, Settings.builder().put("number_of_shards", 6) - .put("number_of_replicas", 0))); + assertAcked(prepareCreate("test-idx-all", 1, indexSettingsNoReplicas(6))); ensureGreen("test-idx-all"); logger.info("--> create an index that will be closed"); @@ -531,7 +505,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest .put("number_of_replicas", 0)).setWaitForActiveShards(ActiveShardCount.NONE).get()); assertTrue(client().admin().indices().prepareExists("test-idx-none").get().isExists()); - createRepository("test-repo", "fs", randomRepoPath()); + createRepository("test-repo", "fs"); logger.info("--> start snapshot with default settings without a closed index - should fail"); CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1") @@ -603,8 +577,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(6)); assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), equalTo(6)); assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0)); - - assertThat(client().prepareSearch("test-idx-all").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + assertDocCount("test-idx-all", 100L); logger.info("--> restore snapshot for the partial index"); cluster().wipeIndices("test-idx-some"); @@ -614,9 +587,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(6)); assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), allOf(greaterThan(0), lessThan(6))); assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), greaterThan(0)); - - assertThat(client().prepareSearch("test-idx-some").setSize(0).get().getHits().getTotalHits().value, allOf(greaterThan(0L), - lessThan(100L))); + assertThat(getCountForIndex("test-idx-some"), allOf(greaterThan(0L), lessThan(100L))); logger.info("--> restore snapshot for the index that didn't have any shards snapshotted successfully"); cluster().wipeIndices("test-idx-none"); @@ -626,9 +597,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(6)); assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), equalTo(0)); assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(6)); - - assertThat(client().prepareSearch("test-idx-some").setSize(0).get().getHits().getTotalHits().value, allOf(greaterThan(0L), - lessThan(100L))); + assertThat(getCountForIndex("test-idx-some"), allOf(greaterThan(0L), lessThan(100L))); logger.info("--> restore snapshot for the closed index that was snapshotted completely"); restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-2") @@ -637,8 +606,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(4)); assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), equalTo(4)); assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0)); - - assertThat(client().prepareSearch("test-idx-closed").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + assertDocCount("test-idx-closed", 100L); } public void testRestoreIndexWithShardsMissingInLocalGateway() throws Exception { @@ -650,20 +618,13 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest internalCluster().startNodes(2, nodeSettings); cluster().wipeIndices("_all"); - createRepository("test-repo", "fs", randomRepoPath()); + createRepository("test-repo", "fs"); int numberOfShards = 6; logger.info("--> create an index that will have some unallocated shards"); - assertAcked(prepareCreate("test-idx", 2, Settings.builder().put("number_of_shards", numberOfShards) - .put("number_of_replicas", 0))); + assertAcked(prepareCreate("test-idx", 2, indexSettingsNoReplicas(numberOfShards))); ensureGreen(); - - logger.info("--> indexing some data into test-idx"); - for (int i = 0; i < 100; i++) { - index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client().prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + indexRandomDocs("test-idx", 100); logger.info("--> start snapshot"); assertThat(client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setIndices("test-idx") @@ -691,7 +652,6 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest equalTo(6)); ensureGreen("test-idx"); - assertThat(client().prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); IntSet reusedShards = new IntHashSet(); List recoveryStates = client().admin().indices().prepareRecoveries("test-idx").get() @@ -786,23 +746,13 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest internalCluster().startDataOnlyNodes(2); final Path repoPath = randomRepoPath(); - createRepository("test-repo", "fs", Settings.builder() - .put("location", repoPath).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "fs", repoPath); maybeInitWithOldSnapshotVersion("test-repo", repoPath); - assertAcked(prepareCreate("test-idx", 0, Settings.builder().put("number_of_shards", between(1, 20)) - .put("number_of_replicas", 0))); + assertAcked(prepareCreate("test-idx", 0, indexSettingsNoReplicas(between(1, 20)))); ensureGreen(); - logger.info("--> indexing some data"); - final int numdocs = randomIntBetween(10, 100); - IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs]; - for (int i = 0; i < builders.length; i++) { - builders[i] = client().prepareIndex("test-idx", "type1", Integer.toString(i)).setSource("field1", "bar " + i); - } - indexRandom(true, builders); - flushAndRefresh(); + indexRandomDocs("test-idx", randomIntBetween(10, 100)); final int numberOfShards = getNumShards("test-idx").numPrimaries; logger.info("number of shards: {}", numberOfShards); @@ -838,23 +788,12 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest internalCluster().startDataOnlyNodes(2); final Path repoPath = randomRepoPath(); - createRepository("test-repo", "mock", Settings.builder() - .put("location", repoPath).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "mock", repoPath); maybeInitWithOldSnapshotVersion("test-repo", repoPath); - assertAcked(prepareCreate("test-idx", 0, Settings.builder().put("number_of_shards", between(1, 20)) - .put("number_of_replicas", 0))); + assertAcked(prepareCreate("test-idx", 0, indexSettingsNoReplicas(between(1, 20)))); ensureGreen(); - - logger.info("--> indexing some data"); - final int numdocs = randomIntBetween(10, 100); - IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs]; - for (int i = 0; i < builders.length; i++) { - builders[i] = client().prepareIndex("test-idx", "type1", Integer.toString(i)).setSource("field1", "bar " + i); - } - indexRandom(true, builders); - flushAndRefresh(); + indexRandomDocs("test-idx", randomIntBetween(10, 100)); final int numberOfShards = getNumShards("test-idx").numPrimaries; logger.info("number of shards: {}", numberOfShards); @@ -897,28 +836,13 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest internalCluster().startMasterOnlyNodes(2); internalCluster().startDataOnlyNodes(2); - logger.info("--> creating repository"); final Path repoPath = randomRepoPath(); - assertAcked(client().admin().cluster().preparePutRepository("test-repo") - .setType("mock").setSettings(Settings.builder() - .put("location", repoPath) - .put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); + createRepository("test-repo", "mock", repoPath); maybeInitWithOldSnapshotVersion("test-repo", repoPath); - assertAcked(prepareCreate("test-idx", 0, Settings.builder() - .put("number_of_shards", 6).put("number_of_replicas", 0))); + assertAcked(prepareCreate("test-idx", 0, indexSettingsNoReplicas(6))); ensureGreen(); - - logger.info("--> indexing some data"); - final int numdocs = randomIntBetween(50, 100); - IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs]; - for (int i = 0; i < builders.length; i++) { - builders[i] = client().prepareIndex("test-idx", "type1", - Integer.toString(i)).setSource("field1", "bar " + i); - } - indexRandom(true, builders); - flushAndRefresh(); + indexRandomDocs("test-idx", randomIntBetween(50, 100)); logger.info("--> stopping random data node, which should cause shards to go missing"); internalCluster().stopRandomDataNode(); @@ -973,23 +897,11 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest final String sourceIdx = "test-idx"; final String shrunkIdx = "test-idx-shrunk"; - logger.info("--> creating repository"); - assertAcked(client.admin().cluster().preparePutRepository(repo).setType("fs") - .setSettings(Settings.builder().put("location", randomRepoPath()) - .put("compress", randomBoolean()))); + createRepository(repo, "fs"); - assertAcked(prepareCreate(sourceIdx, 0, Settings.builder() - .put("number_of_shards", between(2, 10)).put("number_of_replicas", 0))); + assertAcked(prepareCreate(sourceIdx, 0, indexSettingsNoReplicas(between(2, 10)))); ensureGreen(); - - logger.info("--> indexing some data"); - IndexRequestBuilder[] builders = new IndexRequestBuilder[randomIntBetween(10, 100)]; - for (int i = 0; i < builders.length; i++) { - builders[i] = client().prepareIndex(sourceIdx, "type1", - Integer.toString(i)).setSource("field1", "bar " + i); - } - indexRandom(true, builders); - flushAndRefresh(); + indexRandomDocs(sourceIdx, randomIntBetween(10, 100)); logger.info("--> shrink the index"); assertAcked(client.admin().indices().prepareUpdateSettings(sourceIdx) @@ -1139,27 +1051,13 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest logger.info("--> starting a master node and two data nodes"); internalCluster().startMasterOnlyNode(); internalCluster().startDataOnlyNodes(2); - logger.info("--> creating repository"); final Path repoPath = randomRepoPath(); - assertAcked(client().admin().cluster().preparePutRepository("test-repo") - .setType("mock").setSettings(Settings.builder() - .put("location", repoPath) - .put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES))); + createRepository("test-repo", "mock", repoPath); maybeInitWithOldSnapshotVersion("test-repo", repoPath); - assertAcked(prepareCreate("test-idx", 0, Settings.builder() - .put("number_of_shards", 5).put("number_of_replicas", 0))); + assertAcked(prepareCreate("test-idx", 0, indexSettingsNoReplicas(5))); ensureGreen(); - logger.info("--> indexing some data"); - final int numdocs = randomIntBetween(50, 100); - IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs]; - for (int i = 0; i < builders.length; i++) { - builders[i] = client().prepareIndex("test-idx", "type1", - Integer.toString(i)).setSource("field1", "bar " + i); - } - indexRandom(true, builders); - flushAndRefresh(); + indexRandomDocs("test-idx", randomIntBetween(50, 100)); final String dataNode = blockNodeWithIndex("test-repo", "test-idx"); logger.info("--> snapshot"); ServiceDisruptionScheme disruption = new BusyMasterServiceDisruption(random(), Priority.HIGH); @@ -1192,23 +1090,13 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest internalCluster().startMasterOnlyNode(); final List dataNodes = internalCluster().startDataOnlyNodes(2); final Path repoPath = randomRepoPath(); - createRepository("test-repo", "mock", Settings.builder() - .put("location", repoPath).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "mock", repoPath); maybeInitWithOldSnapshotVersion("test-repo", repoPath); - assertAcked(prepareCreate("test-idx", 0, Settings.builder() - .put("number_of_shards", 2).put("number_of_replicas", 0))); + assertAcked(prepareCreate("test-idx", 0, indexSettingsNoReplicas(2))); ensureGreen(); - logger.info("--> indexing some data"); - final int numdocs = randomIntBetween(50, 100); - IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs]; - for (int i = 0; i < builders.length; i++) { - builders[i] = client().prepareIndex("test-idx", "type1", - Integer.toString(i)).setSource("field1", "bar " + i); - } - indexRandom(true, builders); - flushAndRefresh(); + indexRandomDocs("test-idx", randomIntBetween(50, 100)); + blockAllDataNodes("test-repo"); logger.info("--> snapshot"); client(internalCluster().getMasterName()).admin().cluster() @@ -1238,16 +1126,11 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest public void testRetentionLeasesClearedOnRestore() throws Exception { final String repoName = "test-repo-retention-leases"; - createRepository(repoName, "fs", Settings.builder() - .put("location", randomRepoPath()).put("compress", randomBoolean())); + createRepository(repoName, "fs"); final String indexName = "index-retention-leases"; final int shardCount = randomIntBetween(1, 5); - assertAcked(client().admin().indices().prepareCreate(indexName) - .setSettings(Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shardCount) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)) - .get()); + assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(indexSettingsNoReplicas(shardCount))); final ShardId shardId = new ShardId(resolveIndex(indexName), randomIntBetween(0, shardCount - 1)); final int snapshotDocCount = iterations(10, 1000); @@ -1257,7 +1140,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest indexRequestBuilders[i] = client().prepareIndex(indexName, "_doc").setSource("field", "value"); } indexRandom(true, indexRequestBuilders); - assertHitCount(client().prepareSearch(indexName).setSize(0).get(), snapshotDocCount); + assertDocCount(indexName, snapshotDocCount); final String leaseId = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); logger.debug("--> adding retention lease with id {} to {}", leaseId, shardId); @@ -1298,7 +1181,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest assertThat(restoreResponse.getRestoreInfo().failedShards(), equalTo(0)); ensureGreen(); - assertHitCount(client().prepareSearch(indexName).setSize(0).get(), snapshotDocCount); + assertDocCount(indexName, snapshotDocCount); final RetentionLeases restoredRetentionLeases = Arrays.stream(client().admin().indices().prepareStats(indexName).get() .getShards()).filter(s -> s.getShardRouting().shardId().equals(shardId)).findFirst().get() @@ -1316,7 +1199,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest final String otherDataNode = internalCluster().startDataOnlyNode(); final String repoName = "test-repo"; - createRepository(repoName, "mock", randomRepoPath()); + createRepository(repoName, "mock"); blockAllDataNodes(repoName); final String snapshotName = "test-snap"; final ActionFuture snapshotResponse = diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java index c59cae2e827..c12a0b9b764 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MetadataLoadingDuringSnapshotRestoreIT.java @@ -71,7 +71,7 @@ public class MetadataLoadingDuringSnapshotRestoreIT extends AbstractSnapshotInte client().prepareIndex("others", "other").setSource("rank", 4), client().prepareIndex("others", "other").setSource("rank", 5)); - createRepository("repository", CountingMockRepositoryPlugin.TYPE, randomRepoPath()); + createRepository("repository", CountingMockRepositoryPlugin.TYPE); // Creating a snapshot does not load any metadata CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("repository", "snap") diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java index 63ab432fa1f..7654b4a33fd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoriesIT.java @@ -68,7 +68,7 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase { assertThat(repositoriesMetadata.repository("test-repo-1").type(), equalTo("fs")); logger.info("--> creating another repository"); - createRepository("test-repo-2", "fs", randomRepoPath()); + createRepository("test-repo-2", "fs"); logger.info("--> check that both repositories are in cluster state"); clusterStateResponse = client.admin().cluster().prepareState().clear().setMetadata(true).get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 45217329d18..5bbf357d3bb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -44,7 +44,6 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResp import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.action.ingest.GetPipelineResponse; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; @@ -168,42 +167,12 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .build(); } - private Settings.Builder randomRepoSettings() { - Settings.Builder repoSettings = Settings.builder(); - repoSettings.put("location", randomRepoPath()); - if (randomBoolean()) { - repoSettings.put("compress", randomBoolean()); - } - if (randomBoolean()) { - repoSettings.put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES); - } else { - if (randomBoolean()) { - repoSettings.put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES); - } else { - repoSettings.put("chunk_size", (String) null); - } - } - return repoSettings; - } - public void testBasicWorkFlow() throws Exception { Client client = client(); - - createRepository("test-repo", "fs", randomRepoSettings()); - - createIndex("test-idx-1", "test-idx-2", "test-idx-3"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx-1", "_doc", Integer.toString(i), "foo", "bar" + i); - index("test-idx-2", "_doc", Integer.toString(i), "foo", "baz" + i); - index("test-idx-3", "_doc", Integer.toString(i), "foo", "baz" + i); - } - refresh(); - assertHitCount(client.prepareSearch("test-idx-1").setSize(0).get(), 100L); - assertHitCount(client.prepareSearch("test-idx-2").setSize(0).get(), 100L); - assertHitCount(client.prepareSearch("test-idx-3").setSize(0).get(), 100L); + createRepository("test-repo", "fs"); + createIndexWithRandomDocs("test-idx-1", 100); + createIndexWithRandomDocs("test-idx-2", 100); + createIndexWithRandomDocs("test-idx-3", 100); ActionFuture flushResponseFuture = null; if (randomBoolean()) { @@ -268,9 +237,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas client.prepareDelete("test-idx-3", "_doc", Integer.toString(i)).get(); } assertAllSuccessful(refresh()); - assertHitCount(client.prepareSearch("test-idx-1").setSize(0).get(), 50L); - assertHitCount(client.prepareSearch("test-idx-2").setSize(0).get(), 50L); - assertHitCount(client.prepareSearch("test-idx-3").setSize(0).get(), 50L); + assertDocCount("test-idx-1", 50L); + assertDocCount("test-idx-2", 50L); + assertDocCount("test-idx-3", 50L); logger.info("--> close indices"); client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); @@ -281,11 +250,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - for (int i=0; i<5; i++) { - assertHitCount(client.prepareSearch("test-idx-1").setSize(0).get(), 100L); - assertHitCount(client.prepareSearch("test-idx-2").setSize(0).get(), 100L); - assertHitCount(client.prepareSearch("test-idx-3").setSize(0).get(), 50L); - } + assertDocCount("test-idx-1", 100L); + assertDocCount("test-idx-2", 100L); + assertDocCount("test-idx-3", 50L); assertNull(client.admin().indices().prepareGetSettings("test-idx-1").get().getSetting("test-idx-1", MetadataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey())); @@ -306,9 +273,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); ensureGreen(); - for (int i=0; i<5; i++) { - assertHitCount(client.prepareSearch("test-idx-1").setSize(0).get(), 100L); - } + assertDocCount("test-idx-1", 100); ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); assertThat(clusterState.getMetadata().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetadata().hasIndex("test-idx-2"), equalTo(false)); @@ -369,7 +334,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testFreshIndexUUID() { Client client = client(); - createRepository("test-repo", "fs", randomRepoSettings()); + createRepository("test-repo", "fs"); createIndex("test"); String originalIndexUUID = client().admin().indices().prepareGetSettings("test").get() @@ -419,9 +384,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } public void testRestoreWithDifferentMappingsAndSettings() throws Exception { - Client client = client(); - - createRepository("test-repo", "fs", randomRepoSettings()); + createRepository("test-repo", "fs"); logger.info("--> create index with foo type"); assertAcked(prepareCreate("test-idx", 2, Settings.builder() @@ -448,7 +411,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas ensureGreen(); logger.info("--> close index"); - client.admin().indices().prepareClose("test-idx").get(); + client().admin().indices().prepareClose("test-idx").get(); logger.info("--> restore all indices from the snapshot"); RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap") @@ -462,29 +425,27 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(mappings.get("bar"), nullValue()); logger.info("--> assert that old settings are restored"); - GetSettingsResponse getSettingsResponse = client.admin().indices().prepareGetSettings("test-idx").execute().actionGet(); + GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test-idx").execute().actionGet(); assertThat(getSettingsResponse.getSetting("test-idx", "index.refresh_interval"), equalTo("10s")); } public void testEmptySnapshot() throws Exception { - Client client = client(); - - createRepository("test-repo", "fs", Settings.builder().put("location", randomRepoPath())); + createRepository("test-repo", "fs"); logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") + CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") .setWaitForCompletion(true).get(); assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0)); - assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), - equalTo(SnapshotState.SUCCESS)); + assertThat(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get() + .getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); } public void testRestoreAliases() throws Exception { Client client = client(); - createRepository("test-repo", "fs", randomRepoPath()); + createRepository("test-repo", "fs"); logger.info("--> create test indices"); createIndex("test-idx-1", "test-idx-2", "test-idx-3"); @@ -542,7 +503,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testRestoreTemplates() throws Exception { Client client = client(); - createRepository("test-repo", "fs", randomRepoPath()); + createRepository("test-repo", "fs"); logger.info("--> creating test template"); assertThat(client.admin().indices() @@ -592,7 +553,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testIncludeGlobalState() throws Exception { Client client = client(); - createRepository("test-repo", "fs", randomRepoPath()); + createRepository("test-repo", "fs"); boolean testTemplate = randomBoolean(); boolean testPipeline = randomBoolean(); @@ -722,15 +683,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertNotNull(getStoredScriptResponse.getSource()); } - createIndex("test-idx"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + createIndexWithRandomDocs("test-idx", 100); logger.info("--> snapshot without global state but with indices"); createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-no-global-state-with-index") @@ -769,10 +722,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template"); assertFalse(client().admin().cluster().prepareGetPipeline("barbaz").get().isFound()); assertNull(client().admin().cluster().prepareGetStoredScript("foobar").get().getSource()); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + assertDocCount("test-idx", 100L); } - public void testSnapshotFileFailureDuringSnapshot() { + public void testSnapshotFileFailureDuringSnapshot() throws InterruptedException { disableRepoConsistencyCheck("This test uses a purposely broken repository so it would fail consistency checks"); Client client = client(); @@ -785,15 +738,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .put("random_control_io_exception_rate", 0.2)) .setVerify(false)); - createIndex("test-idx"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + createIndexWithRandomDocs("test-idx", 100); logger.info("--> snapshot"); try { @@ -839,15 +784,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas Settings.builder().put("location", randomRepoPath()) .put("random", randomAlphaOfLength(10)).put("random_data_file_io_exception_rate", 0.3)); - createIndex("test-idx"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + createIndexWithRandomDocs("test-idx", 100); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") @@ -908,13 +845,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas ensureGreen(); final NumShards numShards = getNumShards("test-idx"); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + indexRandomDocs("test-idx", 100); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") @@ -941,7 +872,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas if (restoreInfo.successfulShards() == restoreInfo.totalShards()) { // All shards were restored, we must find the exact number of hits - assertHitCount(client.prepareSearch("test-idx").setSize(0).get(), 100L); + assertDocCount("test-idx", 100L); } else { // One or more shards failed to be restored. This can happen when there is // only 1 data node: a shard failed because of the random IO exceptions @@ -963,16 +894,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas Client client = client(); createRepository("test-repo", "fs", repositoryLocation); - - createIndex("test-idx"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + createIndexWithRandomDocs("test-idx", 100); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster() @@ -1039,10 +961,6 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas final String indexName = "unrestorable-index"; Settings restoreIndexSettings = Settings.builder().put("index.routing.allocation.include._name", randomAlphaOfLength(5)).build(); - Consumer checkUnassignedInfo = unassignedInfo -> { - assertThat(unassignedInfo.getReason(), equalTo(UnassignedInfo.Reason.NEW_INDEX_RESTORED)); - }; - Runnable fixupAction =() -> { // remove the shard allocation filtering settings and use the Reroute API to retry the failed shards assertAcked(client().admin().indices().prepareUpdateSettings(indexName) @@ -1052,7 +970,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertAcked(client().admin().cluster().prepareReroute().setRetryFailed(true)); }; - unrestorableUseCase(indexName, Settings.EMPTY, Settings.EMPTY, restoreIndexSettings, checkUnassignedInfo, fixupAction); + unrestorableUseCase(indexName, Settings.EMPTY, Settings.EMPTY, restoreIndexSettings, + unassignedInfo -> assertThat(unassignedInfo.getReason(), equalTo(UnassignedInfo.Reason.NEW_INDEX_RESTORED)), fixupAction); } /** Execute the unrestorable test use case **/ @@ -1070,11 +989,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas // index some documents final int nbDocs = scaledRandomIntBetween(10, 100); - for (int i = 0; i < nbDocs; i++) { - index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i); - } - flushAndRefresh(indexName); - assertThat(client().prepareSearch(indexName).setSize(0).get().getHits().getTotalHits().value, equalTo((long) nbDocs)); + indexRandomDocs(indexName, nbDocs); // create a snapshot final NumShards numShards = getNumShards(indexName); @@ -1147,23 +1062,14 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas ensureGreen(indexName); refresh(indexName); - assertThat(client().prepareSearch(indexName).setSize(0).get().getHits().getTotalHits().value, equalTo((long) nbDocs)); + assertDocCount(indexName, nbDocs); } public void testDeletionOfFailingToRecoverIndexShouldStopRestore() throws Exception { Path repositoryLocation = randomRepoPath(); Client client = client(); createRepository("test-repo", "fs", repositoryLocation); - - createIndex("test-idx"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + createIndexWithRandomDocs("test-idx", 100); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") @@ -1206,21 +1112,18 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0)); - SearchResponse countResponse = client.prepareSearch("test-idx").setSize(0).get(); - assertThat(countResponse.getHits().getTotalHits().value, equalTo(100L)); + assertDocCount("test-idx", 100L); } - public void testUnallocatedShards() throws Exception { - Client client = client(); - - createRepository("test-repo", "fs", randomRepoPath()); + public void testUnallocatedShards() { + createRepository("test-repo", "fs"); logger.info("--> creating index that cannot be allocated"); prepareCreate("test-idx", 2, Settings.builder().put(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "tag", "nowhere") .put("index.number_of_shards", 3)).setWaitForActiveShards(ActiveShardCount.NONE).get(); logger.info("--> snapshot"); - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") + CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") .setWaitForCompletion(true).setIndices("test-idx").get(); assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.FAILED)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0)); @@ -1256,7 +1159,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas // Store number of files after each snapshot numberOfFiles[i] = numberOfFiles(repo); } - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(10L * numberOfSnapshots)); + assertDocCount("test-idx", 10L * numberOfSnapshots); int numberOfFilesBeforeDeletion = numberOfFiles(repo); logger.info("--> delete all snapshots except the first one and last one"); @@ -1283,7 +1186,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(10L * numberOfSnapshots)); + assertDocCount("test-idx", 10L * numberOfSnapshots); logger.info("--> delete the last snapshot"); client.admin().cluster().prepareDeleteSnapshot("test-repo", lastSnapshot).get(); @@ -1295,9 +1198,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas Client client = client(); Path repo = randomRepoPath(); - createRepository("test-repo", "fs", Settings.builder() - .put("location", repo).put("compress", false) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "fs", repo); final String[] indices = {"test-idx-1", "test-idx-2"}; createIndex(indices); @@ -1342,9 +1243,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas Client client = client(); Path repo = randomRepoPath(); - createRepository("test-repo", "fs", Settings.builder() - .put("location", repo).put("compress", false) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "fs", repo); createIndex("test-idx-1", "test-idx-2"); logger.info("--> indexing some data"); @@ -1428,7 +1327,6 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas flushAndRefresh("test-idx-1", "test-idx-2"); SnapshotInfo snapshotInfo = createFullSnapshot("test-repo", "test-snap"); - CreateSnapshotResponse createSnapshotResponse; final Path globalStatePath = repo.resolve("meta-" + snapshotInfo.snapshotId().getUUID() + ".dat"); if (randomBoolean()) { @@ -1464,7 +1362,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas disableRepoConsistencyCheck("This test uses a purposely broken repository so it would fail consistency checks"); Path repo = randomRepoPath(); - createRepository("test-repo", "fs", Settings.builder().put("location", repo).put("compress", false)); + createRepository("test-repo", "fs", repo); createIndex("test-idx-1", "test-idx-2"); logger.info("--> indexing some data"); @@ -1505,7 +1403,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testSnapshotClosedIndex() throws Exception { Client client = client(); - createRepository("test-repo", "fs", randomRepoPath()); + createRepository("test-repo", "fs"); createIndex("test-idx", "test-idx-closed"); ensureGreen(); @@ -1525,7 +1423,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testRenameOnRestore() throws Exception { Client client = client(); - createRepository("test-repo", "fs", randomRepoPath()); + createRepository("test-repo", "fs"); createIndex("test-idx-1", "test-idx-2", "test-idx-3"); ensureGreen(); @@ -1536,14 +1434,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .addAlias("test-idx-3", "alias-3", false) ); - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx-1", "_doc", Integer.toString(i), "foo", "bar" + i); - index("test-idx-2", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); - assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + indexRandomDocs("test-idx-1", 100); + indexRandomDocs("test-idx-2", 100); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") @@ -1557,8 +1449,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .setRenamePattern("(.+)").setRenameReplacement("$1-copy").setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareSearch("test-idx-1-copy").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); - assertThat(client.prepareSearch("test-idx-2-copy").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + assertDocCount("test-idx-1-copy", 100L); + assertDocCount("test-idx-2-copy", 100L); logger.info("--> close just restored indices"); client.admin().indices().prepareClose("test-idx-1-copy", "test-idx-2-copy").get(); @@ -1568,9 +1460,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .setRenamePattern("(.+)").setRenameReplacement("$1-copy").setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareSearch("test-idx-1-copy").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); - assertThat(client.prepareSearch("test-idx-2-copy").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); - + assertDocCount("test-idx-1-copy", 100L); + assertDocCount("test-idx-2-copy", 100L); logger.info("--> close indices"); assertAcked(client.admin().indices().prepareClose("test-idx-1", "test-idx-2-copy")); @@ -1652,13 +1543,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas // Create index on 2 nodes and make sure each node has a primary by setting no replicas assertAcked(prepareCreate("test-idx", 2, Settings.builder().put("number_of_replicas", 0))); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + indexRandomDocs("test-idx", 100); // Pick one node and block it String blockedNode = blockNodeWithIndex("test-repo", "test-idx"); @@ -1697,8 +1582,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap") .setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + assertDocCount("test-idx", 100L); } public void testDeleteRepositoryWhileSnapshotting() throws Exception { @@ -1712,12 +1596,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas // Create index on 2 nodes and make sure each node has a primary by setting no replicas assertAcked(prepareCreate("test-idx", 2, Settings.builder().put("number_of_replicas", 0))); - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + indexRandomDocs("test-idx", 100); // Pick one node and block it String blockedNode = blockNodeWithIndex("test-repo", "test-idx"); @@ -1776,25 +1655,14 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap") .setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + assertDocCount("test-idx", 100); } public void testReadonlyRepository() throws Exception { Client client = client(); Path repositoryLocation = randomRepoPath(); - createRepository("test-repo", "fs", Settings.builder() - .put("location", repositoryLocation).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); - - createIndex("test-idx"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); + createRepository("test-repo", "fs", repositoryLocation); + createIndexWithRandomDocs("test-idx", 100); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") @@ -1818,8 +1686,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("readonly-repo", "test-snap") .setWaitForCompletion(true).setIndices("test-idx").execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + assertDocCount("test-idx", 100L); logger.info("--> list available shapshots"); GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("readonly-repo").get(); @@ -1851,15 +1718,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas throttleRestore && (throttleRestoreViaRecoverySettings == false) ? "10k" : "0") .put("max_snapshot_bytes_per_sec", throttleSnapshot ? "10k" : "0")); - createIndex("test-idx"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + createIndexWithRandomDocs("test-idx", 100); logger.info("--> snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") @@ -1879,7 +1738,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap") .setWaitForCompletion(true).execute().actionGet(); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + assertDocCount("test-idx", 100L); long snapshotPause = 0L; long restorePause = 0L; @@ -1910,15 +1769,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .put("location", randomRepoPath()).put("compress", randomBoolean()) .put("chunk_size", 100, ByteSizeUnit.BYTES)); - createIndex("test-idx"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + createIndexWithRandomDocs("test-idx", 100); logger.info("--> snapshot"); client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") @@ -1951,7 +1802,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertBusy(() -> assertTrue(restoreSnapshotResponse.isDone())); assertThat(restoreSnapshotResponse.get().getRestoreInfo().totalShards(), greaterThan(0)); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + assertDocCount("test-idx", 100L); } public void testSnapshotStatus() throws Exception { @@ -1962,13 +1813,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas // Create index on 2 nodes and make sure each node has a primary by setting no replicas assertAcked(prepareCreate("test-idx", 2, Settings.builder().put("number_of_replicas", 0))); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + indexRandomDocs("test-idx", 100); // Pick one node and block it String blockedNode = blockNodeWithIndex("test-repo", "test-idx"); @@ -2070,23 +1915,13 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testSnapshotRelocatingPrimary() throws Exception { Client client = client(); - createRepository("test-repo", "fs", Settings.builder() - .put("location", randomRepoPath()).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "fs"); // Create index on two nodes and make sure each node has a primary by setting no replicas - assertAcked(prepareCreate("test-idx", 2, Settings.builder() - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_NUMBER_OF_SHARDS, between(2, 10)))); + assertAcked(prepareCreate("test-idx", 2, indexSettingsNoReplicas(between(2, 10)))); ensureGreen("test-idx"); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + indexRandomDocs("test-idx", 100); logger.info("--> start relocations"); allowNodes("test-idx", 1); @@ -2109,21 +1944,13 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testSnapshotDeleteRelocatingPrimaryIndex() throws Exception { final String repoName = "test-repo"; - createRepository(repoName, "fs", randomRepoPath()); + createRepository(repoName, "fs"); // Create index on two nodes and make sure each node has a primary by setting no replicas final String indexName = "test-idx"; - assertAcked(prepareCreate(indexName, 2, Settings.builder() - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_NUMBER_OF_SHARDS, between(2, 10)))); + assertAcked(prepareCreate(indexName, 2, indexSettingsNoReplicas(between(2, 10)))); ensureGreen(indexName); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client().prepareSearch(indexName).setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + indexRandomDocs(indexName, 100); logger.info("--> start relocations"); allowNodes(indexName, 1); @@ -2150,24 +1977,14 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testSnapshotMoreThanOnce() throws InterruptedException { Client client = client(); - createRepository("test-repo", "fs", Settings.builder() - .put("location", randomRepoPath()).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "fs"); // only one shard final Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build(); assertAcked(prepareCreate("test").setSettings(indexSettings)); ensureGreen(); - logger.info("--> indexing"); - - final int numdocs = randomIntBetween(10, 100); - IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs]; - for (int i = 0; i < builders.length; i++) { - builders[i] = client().prepareIndex("test", "_doc", Integer.toString(i)).setSource("foo", "bar" + i); - } - indexRandom(true, builders); - flushAndRefresh(); + indexRandomDocs("test", randomIntBetween(10, 100)); assertNoFailures(client().admin().indices().prepareForceMerge("test").setFlush(true).setMaxNumSegments(1).get()); CreateSnapshotResponse createSnapshotResponseFirst = client.admin().cluster().prepareCreateSnapshot("test-repo", "test") @@ -2230,10 +2047,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testChangeSettingsOnRestore() throws Exception { Client client = client(); - createRepository("test-repo", "fs", Settings.builder() - .put("location", randomRepoPath()) - .put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "fs"); logger.info("--> create test index with case-preserving search analyzer"); @@ -2341,9 +2155,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testRecreateBlocksOnRestore() throws Exception { Client client = client(); - createRepository("test-repo", "fs", Settings.builder() - .put("location", randomRepoPath()).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "fs"); Settings.Builder indexSettings = Settings.builder() .put(indexSettings()) @@ -2446,8 +2258,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .execute().actionGet(); } refresh(); - - assertThat(client.prepareSearch(dataStream).setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + assertDocCount(dataStream, 100L); logger.info("--> snapshot"); ActionFuture future = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") @@ -2479,30 +2290,17 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testCloseOrDeleteIndexDuringSnapshot() throws Exception { disableRepoConsistencyCheck("This test intentionally leaves a broken repository"); - Client client = client(); - createRepository("test-repo", "mock", Settings.builder() .put("location", randomRepoPath()).put("compress", randomBoolean()) .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES) .put("block_on_data", true)); - - createIndex("test-idx-1", "test-idx-2", "test-idx-3"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx-1", "_doc", Integer.toString(i), "foo", "bar" + i); - index("test-idx-2", "_doc", Integer.toString(i), "foo", "baz" + i); - index("test-idx-3", "_doc", Integer.toString(i), "foo", "baz" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); - assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); - assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + createIndexWithRandomDocs("test-idx-1", 100); + createIndexWithRandomDocs("test-idx-2", 100); + createIndexWithRandomDocs("test-idx-3", 100); logger.info("--> snapshot"); - ActionFuture future = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") + ActionFuture future = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") .setIndices("test-idx-*").setWaitForCompletion(true).setPartial(false).execute(); logger.info("--> wait for block to kick in"); waitForBlockOnAnyDataNode("test-repo", TimeValue.timeValueMinutes(1)); @@ -2512,7 +2310,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas if (randomBoolean()) { try { logger.info("--> delete index while non-partial snapshot is running"); - client.admin().indices().prepareDelete("test-idx-1").get(); + client().admin().indices().prepareDelete("test-idx-1").get(); fail("Expected deleting index to fail during snapshot"); } catch (SnapshotInProgressException e) { assertThat(e.getMessage(), containsString("Cannot delete indices that are being snapshotted: [[test-idx-1/")); @@ -2520,7 +2318,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas } else { try { logger.info("--> close index while non-partial snapshot is running"); - client.admin().indices().prepareClose("test-idx-1").get(); + client().admin().indices().prepareClose("test-idx-1").get(); fail("Expected closing index to fail during snapshot"); } catch (SnapshotInProgressException e) { assertThat(e.getMessage(), containsString("Cannot close indices that are being snapshotted: [[test-idx-1/")); @@ -2540,21 +2338,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testCloseIndexDuringRestore() throws Exception { Client client = client(); - createRepository("test-repo", "mock", Settings.builder() - .put("location", randomRepoPath()).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); - - createIndex("test-idx-1", "test-idx-2"); - ensureGreen(); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index("test-idx-1", "_doc", Integer.toString(i), "foo", "bar" + i); - index("test-idx-2", "_doc", Integer.toString(i), "foo", "baz" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); - assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + createRepository("test-repo", "mock"); + createIndexWithRandomDocs("test-idx-1", 100); + createIndexWithRandomDocs("test-idx-2", 100); logger.info("--> snapshot"); assertThat(client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") @@ -2601,18 +2387,12 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas Client client = client(); final String repoName = "test-repo"; - createRepository(repoName, "mock", randomRepoPath()); + createRepository(repoName, "mock"); logger.info("--> creating index"); final String indexName = "test-idx"; assertAcked(prepareCreate(indexName).setWaitForActiveShards(ActiveShardCount.ALL)); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch(indexName).setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + indexRandomDocs(indexName, 100); logger.info("--> take snapshots"); final String snapshotName = "test-snap"; @@ -2669,9 +2449,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas final Client client = client(); - createRepository("test-repo", "fs", Settings.builder() - .put("location", randomRepoPath()).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "fs"); expectThrows(InvalidSnapshotNameException.class, () -> client.admin().cluster().prepareCreateSnapshot("test-repo", "_foo").get()); @@ -2736,9 +2514,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testRestoreSnapshotWithCorruptedGlobalState() throws Exception { final Path repo = randomRepoPath(); final String repoName = "test-repo"; - createRepository(repoName, "fs", Settings.builder() - .put("location", repo) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository(repoName, "fs", repo); createIndex("test-idx-1", "test-idx-2"); indexRandom(true, @@ -2799,8 +2575,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas for (int i = 0; i < nbIndices; i++) { String indexName = "test-idx-" + i; - assertAcked(prepareCreate(indexName).setSettings(Settings.builder() - .put(SETTING_NUMBER_OF_SHARDS, Math.min(2, numberOfShards())).put(SETTING_NUMBER_OF_REPLICAS, 0))); + assertAcked(prepareCreate(indexName).setSettings(indexSettingsNoReplicas(Math.min(2, numberOfShards())))); int nbDocs = randomIntBetween(1, 10); nbDocsPerIndex.put(indexName, nbDocs); @@ -2868,8 +2643,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas final int nDocs = randomIntBetween(1, 10); logger.info("--> creating index [{}] with [{}] documents in it", indexName, nDocs); - assertAcked(prepareCreate(indexName).setSettings(Settings.builder() - .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0))); + assertAcked(prepareCreate(indexName).setSettings(indexSettingsNoReplicas(1))); final IndexRequestBuilder[] documents = new IndexRequestBuilder[nDocs]; for (int j = 0; j < nDocs; j++) { @@ -2914,7 +2688,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .get(); ensureGreen(); - assertHitCount(client().prepareSearch(indexName).setSize(0).get(), nDocs); + assertDocCount(indexName, nDocs); logger.info("--> indexing [{}] more documents into [{}]", nDocs, indexName); for (int j = 0; j < nDocs; j++) { @@ -2939,18 +2713,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas final String snapshotName = "test-snap"; final String indexName = "test-idx"; final Client client = client(); - final Path repo = randomRepoPath(); - createRepository(repositoryName, "fs", Settings.builder() - .put("location", repo).put("compress", false) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository(repositoryName, "fs"); logger.info("--> creating an index and indexing documents"); - createIndex(indexName); - ensureGreen(); - for (int i = 0; i < 10; i++) { - index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); + createIndexWithRandomDocs(indexName, 10); logger.info("--> take first snapshot"); CreateSnapshotResponse createSnapshotResponse = client.admin() @@ -3024,10 +2790,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas // Create index on 2 nodes and make sure each node has a primary by setting no replicas assertAcked(prepareCreate(indexName, 1, Settings.builder().put("number_of_replicas", 0))); ensureGreen(); - for (int i = 0; i < 10; i++) { - index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); + indexRandomDocs(indexName, 10); // make sure we return only the in-progress snapshot when taking the first snapshot on a clean repository // take initial snapshot with a block, making sure we only get 1 in-progress snapshot returned @@ -3146,7 +2909,6 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testSnapshotCanceledOnRemovedShard() throws Exception { final int numPrimaries = 1; final int numReplicas = 1; - final int numDocs = 100; final String repo = "test-repo"; final String index = "test-idx"; final String snapshot = "test-snap"; @@ -3154,11 +2916,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", numPrimaries).put("number_of_replicas", numReplicas))); - logger.info("--> indexing some data"); - for (int i = 0; i < numDocs; i++) { - index(index, "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); + indexRandomDocs(index, 100); createRepository(repo, "mock", Settings.builder() .put("location", randomRepoPath()).put("random", randomAlphaOfLength(10)).put("wait_after_unblock", 200)); @@ -3206,19 +2964,14 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas // test that we can take a snapshot after a failed one, even if a partial index-N was written .put("random", randomAlphaOfLength(10)))); - logger.info("--> indexing some data"); assertAcked(prepareCreate("test-idx").setSettings( // the less the number of shards, the less control files we have, so we are giving a higher probability of // triggering an IOException toward the end when writing the pending-index-* files, which are the files // that caused problems with writing subsequent snapshots if they happened to be lingering in the repository - Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0))); + indexSettingsNoReplicas(1))); ensureGreen(); final int numDocs = randomIntBetween(1, 5); - for (int i = 0; i < numDocs; i++) { - index("test-idx", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits().value, equalTo((long) numDocs)); + indexRandomDocs("test-idx", numDocs); logger.info("--> snapshot with potential I/O failures"); try { @@ -3263,27 +3016,16 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .setSettings(Settings.builder().put("location", repoPath))); logger.info("--> creating good index"); - assertAcked(prepareCreate("test-idx-good") - .setSettings(Settings.builder() - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0))); + assertAcked(prepareCreate("test-idx-good").setSettings(indexSettingsNoReplicas(1))); ensureGreen(); - final int numDocs = randomIntBetween(1, 5); - for (int i = 0; i < numDocs; i++) { - index("test-idx-good", "_doc", Integer.toString(i), "foo", "bar" + i); - } - refresh(); - + indexRandomDocs("test-idx-good", randomIntBetween(1, 5)); logger.info("--> creating bad index"); assertAcked(prepareCreate("test-idx-bad") .setWaitForActiveShards(ActiveShardCount.NONE) - .setSettings(Settings.builder() - .put(SETTING_NUMBER_OF_SHARDS, 1) - .put(SETTING_NUMBER_OF_REPLICAS, 0) + .setSettings(indexSettingsNoReplicas(1) // set shard allocation to none so the primary cannot be // allocated - simulates a "bad" index that fails to snapshot - .put(EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), - "none"))); + .put(EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "none"))); logger.info("--> snapshot bad index and get status"); client.admin().cluster() @@ -3336,8 +3078,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas logger.info("--> creating random number of indices"); final int numIndices = randomIntBetween(1, 10); for (int i = 0; i < numIndices; i++) { - assertAcked(prepareCreate("test-idx-" + i).setSettings(Settings.builder() - .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0))); + assertAcked(prepareCreate("test-idx-" + i).setSettings(indexSettingsNoReplicas(1))); } logger.info("--> creating random number of snapshots"); @@ -3408,19 +3149,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas final String indexName = "test-idx"; final Client client = client(); - createRepository(repositoryName, "fs", Settings.builder() - .put("location", randomRepoPath()) - .put("compress", false) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository(repositoryName, "fs"); logger.info("--> creating an index and indexing documents"); final String dataNode = internalCluster().getDataNodeInstance(ClusterService.class).localNode().getName(); - final Settings settings = - Settings - .builder() - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) - .put("index.routing.allocation.include._name", dataNode) - .build(); + final Settings settings = indexSettingsNoReplicas(1).put("index.routing.allocation.include._name", dataNode).build(); createIndex(indexName, settings); ensureGreen(); for (int i = 0; i < 5; i++) { @@ -3587,10 +3319,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas public void testRestoreIncreasesPrimaryTerms() { final String indexName = randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT); - createIndex(indexName, Settings.builder() - .put(SETTING_NUMBER_OF_SHARDS, 2) - .put(SETTING_NUMBER_OF_REPLICAS, 0) - .build()); + createIndex(indexName, indexSettingsNoReplicas(2).build()); ensureGreen(indexName); if (randomBoolean()) { @@ -3608,7 +3337,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas final Map primaryTerms = IntStream.range(0, numPrimaries) .boxed().collect(Collectors.toMap(shardId -> shardId, indexMetadata::primaryTerm)); - createRepository("test-repo", "fs", randomRepoSettings()); + createRepository("test-repo", "fs"); final CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") .setWaitForCompletion(true).setIndices(indexName).get(); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(numPrimaries)); @@ -3629,7 +3358,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(restoredIndexMetadata.getSettings().get(IndexMetadata.SETTING_HISTORY_UUID), notNullValue()); } - public void testSnapshotDifferentIndicesBySameName() { + public void testSnapshotDifferentIndicesBySameName() throws InterruptedException { String indexName = "testindex"; String repoName = "test-repo"; Path absolutePath = randomRepoPath().toAbsolutePath(); @@ -3639,11 +3368,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas createIndex(indexName, Settings.builder().put("index.number_of_shards", initialShardCount).build()); ensureGreen(); - logger.info("--> indexing some documents"); final int docCount = initialShardCount * randomIntBetween(1, 10); - for (int i = 0; i < docCount; i++) { - index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i); - } + indexRandomDocs(indexName, docCount); createRepository(repoName, "fs", absolutePath); @@ -3658,11 +3384,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas createIndex(indexName, Settings.builder().put("index.number_of_shards", newShardCount).build()); ensureGreen(); - logger.info("--> indexing some documents"); final int newDocCount = newShardCount * randomIntBetween(1, 10); - for (int i = 0; i < newDocCount; i++) { - index(indexName, "_doc", Integer.toString(i), "foo", "bar" + i); - } + indexRandomDocs(indexName, newDocCount); logger.info("--> snapshot with [{}] shards", newShardCount); final SnapshotInfo snapshot2 = createFullSnapshot(repoName, "snap-2"); @@ -3677,8 +3400,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .setRenameReplacement("restored-2").setWaitForCompletion(true).get(); logger.info("--> verify doc counts"); - assertHitCount(client().prepareSearch("restored-1").setSize(0).get(), docCount); - assertHitCount(client().prepareSearch("restored-2").setSize(0).get(), newDocCount); + assertDocCount("restored-1", docCount); + assertDocCount("restored-2", newDocCount); final String snapshotToDelete; final String snapshotToRestore; @@ -3699,14 +3422,12 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas .setRenameReplacement("restored-3").setWaitForCompletion(true).get(); logger.info("--> verify doc counts"); - assertHitCount(client().prepareSearch("restored-3").setSize(0).get(), expectedCount); + assertDocCount("restored-3", expectedCount); } public void testBulkDeleteWithOverlappingPatterns() { final int numberOfSnapshots = between(5, 15); - createRepository("test-repo", "fs", Settings.builder() - .put("location", randomRepoPath()).put("compress", false) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "fs"); final String[] indices = {"test-idx-1", "test-idx-2", "test-idx-3"}; createIndex(indices); @@ -3732,42 +3453,25 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas assertThat(getSnapshotsResponse.getSnapshots(), empty()); } - public void testHiddenIndicesIncludedInSnapshot() { + public void testHiddenIndicesIncludedInSnapshot() throws InterruptedException { Client client = client(); final String normalIndex = "normal-index"; final String hiddenIndex = "hidden-index"; final String dottedHiddenIndex = ".index-hidden"; final String repoName = "test-repo"; - createRepository(repoName, "fs", randomRepoSettings()); + createRepository(repoName, "fs"); logger.info("--> creating indices"); - createIndex(normalIndex, Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,3)) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .build()); - createIndex(hiddenIndex, Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,3)) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) - .build()); - createIndex(dottedHiddenIndex, Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1,3)) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) - .build()); + createIndex(normalIndex, indexSettingsNoReplicas(randomIntBetween(1,3)).build()); + createIndex(hiddenIndex, indexSettingsNoReplicas(randomIntBetween(1,3)).put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build()); + createIndex(dottedHiddenIndex, + indexSettingsNoReplicas(randomIntBetween(1,3)).put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build()); ensureGreen(); - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - index(normalIndex, "_doc", Integer.toString(i), "foo", "bar" + i); - index(hiddenIndex, "_doc", Integer.toString(i), "foo", "baz" + i); - index(dottedHiddenIndex, "_doc", Integer.toString(i), "foo", "baz" + i); - } - refresh(); - assertHitCount(client.prepareSearch(normalIndex).setSize(0).get(), 100L); - assertHitCount(client.prepareSearch(hiddenIndex).setSize(0).get(), 100L); - assertHitCount(client.prepareSearch(dottedHiddenIndex).setSize(0).get(), 100L); + indexRandomDocs(normalIndex, 100); + indexRandomDocs(hiddenIndex, 100); + indexRandomDocs(dottedHiddenIndex, 100); logger.info("--> taking a snapshot"); final String snapName = "test-snap"; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java index 9c27ffcfc96..e96f30f9186 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShardsServiceIT.java @@ -20,8 +20,6 @@ package org.elasticsearch.snapshots; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.plugins.Plugin; @@ -53,17 +51,12 @@ public class SnapshotShardsServiceIT extends AbstractSnapshotIntegTestCase { internalCluster().startMasterOnlyNode(); internalCluster().startDataOnlyNode(); - createRepository("test-repo", "mock", Settings.builder() - .put("location", randomRepoPath()).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)); + createRepository("test-repo", "mock"); final int shards = between(1, 10); - assertAcked(prepareCreate("test-index", 0, Settings.builder().put("number_of_shards", shards).put("number_of_replicas", 0))); + assertAcked(prepareCreate("test-index", 0, indexSettingsNoReplicas(shards))); ensureGreen(); - final int numDocs = scaledRandomIntBetween(50, 100); - for (int i = 0; i < numDocs; i++) { - index("test-index", "doc", Integer.toString(i)); - } + indexRandomDocs("test-index", scaledRandomIntBetween(50, 100)); logger.info("--> blocking repository"); String blockedNode = blockNodeWithIndex("test-repo", "test-index"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java index 4988c08ea79..0e159546a78 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java @@ -29,7 +29,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.SnapshotsInProgress; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.core.internal.io.IOUtils; @@ -50,7 +49,7 @@ public class SnapshotStatusApisIT extends AbstractSnapshotIntegTestCase { public void testStatusApiConsistency() { Client client = client(); - createRepository("test-repo", "fs", randomRepoPath()); + createRepository("test-repo", "fs"); createIndex("test-idx-1", "test-idx-2", "test-idx-3"); ensureGreen(); @@ -158,7 +157,7 @@ public class SnapshotStatusApisIT extends AbstractSnapshotIntegTestCase { } public void testGetSnapshotsWithoutIndices() { - createRepository("test-repo", "fs", randomRepoPath()); + createRepository("test-repo", "fs"); logger.info("--> snapshot"); final SnapshotInfo snapshotInfo = @@ -199,7 +198,7 @@ public class SnapshotStatusApisIT extends AbstractSnapshotIntegTestCase { index(indexTwo, "_doc", "some_doc_id", "foo", "bar"); final String repoName = "test-repo"; - createRepository(repoName, "mock", randomRepoPath()); + createRepository(repoName, "mock"); blockDataNode(repoName, dataNodeOne); @@ -279,9 +278,6 @@ public class SnapshotStatusApisIT extends AbstractSnapshotIntegTestCase { } private static Settings singleShardOneNode(String node) { - return Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.routing.allocation.include._name", node).build(); + return indexSettingsNoReplicas(1).put("index.routing.allocation.include._name", node).build(); } } diff --git a/server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 41b113ff71d..96ca9a6e64f 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/server/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -21,12 +21,16 @@ package org.elasticsearch.snapshots; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -39,6 +43,7 @@ import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.snapshots.mockstore.MockRepository; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -280,6 +285,19 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase { createRepository(repoName, type, Settings.builder().put("location", location)); } + protected void createRepository(String repoName, String type) { + Settings.Builder settings = Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean()); + if (rarely()) { + settings = settings.put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES); + } + createRepository(repoName, type, settings); + } + + protected static Settings.Builder indexSettingsNoReplicas(int shards) { + return Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0); + } + /** * Randomly write an empty snapshot of an older version to an empty repository to simulate an older repository metadata format. */ @@ -329,4 +347,30 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase { assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS)); return snapshotInfo; } + + protected void createIndexWithRandomDocs(String indexName, int docCount) throws InterruptedException { + createIndex(indexName); + ensureGreen(); + indexRandomDocs(indexName, docCount); + } + + protected void indexRandomDocs(String index, int numdocs) throws InterruptedException { + logger.info("--> indexing [{}] documents into [{}]", numdocs, index); + IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs]; + for (int i = 0; i < builders.length; i++) { + builders[i] = client().prepareIndex(index, "_doc").setId(Integer.toString(i)).setSource("field1", "bar " + i); + } + indexRandom(true, builders); + flushAndRefresh(index); + assertDocCount(index, numdocs); + } + + protected long getCountForIndex(String indexName) { + return client().search(new SearchRequest(new SearchRequest(indexName).source( + new SearchSourceBuilder().size(0).trackTotalHits(true)))).actionGet().getHits().getTotalHits().value; + } + + protected void assertDocCount(String index, long count) { + assertEquals(getCountForIndex(index), count); + } }