honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove ShardClusterSnapshotRestoreIT and fold test in DataStreamsSnapshotsIT (#62470) (#62523) 

ShardClusterSnapshotRestoreIT is confusing as we already have a
very complete SharedClusterSnapshotRestoreIT test suite. This
commit removes ShardClusterSnapshotRestoreIT and folds its
unique test in DataStreamsSnapshotsIT.
This commit is contained in:
Tanguy Leroux 2020-09-17 12:37:17 +02:00 committed by GitHub
parent fe3bf86620
commit a39ddb8a0f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 77 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
View File

@ -6,29 +6,34 @@
package org.elasticsearch.datastreams;
import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.core.action.CreateDataStreamAction;
import org.elasticsearch.xpack.core.action.DeleteDataStreamAction;
import org.elasticsearch.xpack.core.action.GetDataStreamAction;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.common.collect.List;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
import org.elasticsearch.snapshots.SnapshotInProgressException;
import org.elasticsearch.snapshots.SnapshotInfo;
import org.elasticsearch.snapshots.SnapshotRestoreException;
import org.elasticsearch.snapshots.SnapshotState;
import org.elasticsearch.snapshots.mockstore.MockRepository;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.core.action.CreateDataStreamAction;
import org.elasticsearch.xpack.core.action.DeleteDataStreamAction;
import org.elasticsearch.xpack.core.action.GetDataStreamAction;
import org.elasticsearch.xpack.datastreams.DataStreamsPlugin;
import org.hamcrest.Matchers;
import org.junit.After;
@ -40,12 +45,14 @@ import java.util.Collections;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@ESIntegTestCase.ClusterScope(transportClientRatio = 0)
public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
@ -261,7 +268,7 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(DS_BACKING_INDEX_NAME));
}
public void testDataStreamAndBackingIndidcesAreRenamedUsingRegex() {
public void testDataStreamAndBackingIndicesAreRenamedUsingRegex() {
CreateSnapshotResponse createSnapshotResponse = client.admin()
.cluster()
.prepareCreateSnapshot(REPO, SNAPSHOT)
@ -411,4 +418,67 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
assertThat(restoreSnapshotResponse.getRestoreInfo().indices(), empty());
}
public void testDeleteDataStreamDuringSnapshot() throws Exception {
Client client = client();
// this test uses a MockRepository
assertAcked(client().admin().cluster().prepareDeleteRepository(REPO));
final String repositoryName = "test-repo";
createRepository(
repositoryName,
"mock",
Settings.builder()
.put("location", randomRepoPath())
.put("compress", randomBoolean())
.put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
.put("block_on_data", true)
);
String dataStream = "datastream";
DataStreamIT.putComposableIndexTemplate("dst", Collections.singletonList(dataStream));
logger.info("--> indexing some data");
for (int i = 0; i < 100; i++) {
client.prepareIndex(dataStream, "_doc")
.setOpType(DocWriteRequest.OpType.CREATE)
.setId(Integer.toString(i))
.setSource(Collections.singletonMap("@timestamp", "2020-12-12"))
.execute()
.actionGet();
}
refresh();
assertDocCount(dataStream, 100L);
logger.info("--> snapshot");
ActionFuture<CreateSnapshotResponse> future = client.admin()
.cluster()
.prepareCreateSnapshot(repositoryName, SNAPSHOT)
.setIndices(dataStream)
.setWaitForCompletion(true)
.setPartial(false)
.execute();
logger.info("--> wait for block to kick in");
waitForBlockOnAnyDataNode(repositoryName, TimeValue.timeValueMinutes(1));
// non-partial snapshots do not allow delete operations on data streams where snapshot has not been completed
try {
logger.info("--> delete index while non-partial snapshot is running");
client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { dataStream })).actionGet();
fail("Expected deleting index to fail during snapshot");
} catch (SnapshotInProgressException e) {
assertThat(e.getMessage(), containsString("Cannot delete data streams that are being snapshotted: [" + dataStream));
} finally {
logger.info("--> unblock all data nodes");
unblockAllDataNodes(repositoryName);
}
logger.info("--> waiting for snapshot to finish");
CreateSnapshotResponse createSnapshotResponse = future.get();
logger.info("--> snapshot successfully completed");
SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
assertThat(snapshotInfo.state(), equalTo((SnapshotState.SUCCESS)));
assertThat(snapshotInfo.dataStreams(), contains(dataStream));
assertThat(snapshotInfo.indices(), contains(DataStream.getDefaultBackingIndexName(dataStream, 1)));
}
}

118
x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ShardClusterSnapshotRestoreIT.java
View File

@ -1,118 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.datastreams;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.core.action.DeleteDataStreamAction;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.common.collect.List;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
import org.elasticsearch.snapshots.SnapshotInProgressException;
import org.elasticsearch.snapshots.SnapshotInfo;
import org.elasticsearch.snapshots.SnapshotState;
import org.elasticsearch.snapshots.mockstore.MockRepository;
import org.elasticsearch.xpack.datastreams.DataStreamsPlugin;
import org.junit.After;
import java.util.Collection;
import java.util.Collections;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
// The tests in here do a lot of state updates and other writes to disk and are slowed down too much by WindowsFS
@LuceneTestCase.SuppressFileSystems(value = "WindowsFS")
@ESIntegTestCase.ClusterScope(transportClientRatio = 0)
public class ShardClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return List.of(MockRepository.Plugin.class, DataStreamsPlugin.class);
}
@After
public void cleanup() {
AcknowledgedResponse response = client().execute(
DeleteDataStreamAction.INSTANCE,
new DeleteDataStreamAction.Request(new String[] { "*" })
).actionGet();
assertAcked(response);
}
public void testDeleteDataStreamDuringSnapshot() throws Exception {
Client client = client();
createRepository(
"test-repo",
"mock",
Settings.builder()
.put("location", randomRepoPath())
.put("compress", randomBoolean())
.put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
.put("block_on_data", true)
);
String dataStream = "datastream";
DataStreamIT.putComposableIndexTemplate("dst", Collections.singletonList(dataStream));
logger.info("--> indexing some data");
for (int i = 0; i < 100; i++) {
client.prepareIndex(dataStream, "_doc")
.setOpType(DocWriteRequest.OpType.CREATE)
.setId(Integer.toString(i))
.setSource(Collections.singletonMap("@timestamp", "2020-12-12"))
.execute()
.actionGet();
}
refresh();
assertDocCount(dataStream, 100L);
logger.info("--> snapshot");
ActionFuture<CreateSnapshotResponse> future = client.admin()
.cluster()
.prepareCreateSnapshot("test-repo", "test-snap")
.setIndices(dataStream)
.setWaitForCompletion(true)
.setPartial(false)
.execute();
logger.info("--> wait for block to kick in");
waitForBlockOnAnyDataNode("test-repo", TimeValue.timeValueMinutes(1));
// non-partial snapshots do not allow delete operations on data streams where snapshot has not been completed
try {
logger.info("--> delete index while non-partial snapshot is running");
client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { dataStream })).actionGet();
fail("Expected deleting index to fail during snapshot");
} catch (SnapshotInProgressException e) {
assertThat(e.getMessage(), containsString("Cannot delete data streams that are being snapshotted: [" + dataStream));
} finally {
logger.info("--> unblock all data nodes");
unblockAllDataNodes("test-repo");
}
logger.info("--> waiting for snapshot to finish");
CreateSnapshotResponse createSnapshotResponse = future.get();
logger.info("Snapshot successfully completed");
SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
assertThat(snapshotInfo.state(), equalTo((SnapshotState.SUCCESS)));
assertThat(snapshotInfo.dataStreams(), contains(dataStream));
assertThat(snapshotInfo.indices(), contains(DataStream.getDefaultBackingIndexName(dataStream, 1)));
}
}