Cleanup the PR and apply advices from the review
* ESBlobStore tests must move to the test framework if we want to be able to reuse them in the context of plugins. * To be able to identify more easily what are Integration Tests vs Unit Tests, this commit renames `*AzureTestCase` to `*AzureIntegTestCase`. * Move some debug level logs to trace level * Collapse when possible identical catch blocks * `blobNameFromUri()` does not need anymore to get the container name. We just split the URI after 3 `/` and simply get the remaining part. * Added a Unit test for that * As we renamed some existing classes, checkstyle is now complaining about the lines width. * While we are at it, let's replace all calls to `execute().actionGet()` with `get()` * Move `readSettingsFromFile()` in a Util class. Note that this class might be useful for other plugins (S3/EC2/Azure-discovery for instance) so may be we should move it to the test framework? * Replace some part of the code with lambdas
This commit is contained in:
David Pilato
parent
9b247f9828
commit
6772517f4d
|
|
@ -62,7 +62,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
|
|||
|
||||
@Override
|
||||
public boolean blobExists(String blobName) {
|
||||
logger.debug("blobExists({})", blobName);
|
||||
logger.trace("blobExists({})", blobName);
|
||||
try {
|
||||
return blobStore.blobExists(blobStore.container(), buildKey(blobName));
|
||||
} catch (URISyntaxException | StorageException e) {
|
||||
|
|
@ -73,7 +73,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
|
|||
|
||||
@Override
|
||||
public InputStream readBlob(String blobName) throws IOException {
|
||||
logger.debug("readBlob({})", blobName);
|
||||
logger.trace("readBlob({})", blobName);
|
||||
try {
|
||||
return blobStore.getInputStream(blobStore.container(), buildKey(blobName));
|
||||
} catch (StorageException e) {
|
||||
|
|
@ -88,7 +88,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
|
|||
|
||||
@Override
|
||||
public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
|
||||
logger.debug("writeBlob({}, stream, {})", blobName, blobSize);
|
||||
logger.trace("writeBlob({}, stream, {})", blobName, blobSize);
|
||||
try (OutputStream stream = createOutput(blobName)) {
|
||||
Streams.copy(inputStream, stream);
|
||||
}
|
||||
|
|
@ -96,7 +96,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
|
|||
|
||||
@Override
|
||||
public void writeBlob(String blobName, BytesReference bytes) throws IOException {
|
||||
logger.debug("writeBlob({}, bytes)", blobName);
|
||||
logger.trace("writeBlob({}, bytes)", blobName);
|
||||
try (OutputStream stream = createOutput(blobName)) {
|
||||
bytes.writeTo(stream);
|
||||
}
|
||||
|
|
@ -119,7 +119,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
|
|||
|
||||
@Override
|
||||
public void deleteBlob(String blobName) throws IOException {
|
||||
logger.debug("deleteBlob({})", blobName);
|
||||
logger.trace("deleteBlob({})", blobName);
|
||||
try {
|
||||
blobStore.deleteBlob(blobStore.container(), buildKey(blobName));
|
||||
} catch (URISyntaxException | StorageException e) {
|
||||
|
|
@ -130,7 +130,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
|
|||
|
||||
@Override
|
||||
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String prefix) throws IOException {
|
||||
logger.debug("listBlobsByPrefix({})", prefix);
|
||||
logger.trace("listBlobsByPrefix({})", prefix);
|
||||
|
||||
try {
|
||||
return blobStore.listBlobsByPrefix(blobStore.container(), keyPath, prefix);
|
||||
|
|
@ -142,7 +142,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
|
|||
|
||||
@Override
|
||||
public void move(String sourceBlobName, String targetBlobName) throws IOException {
|
||||
logger.debug("move({}, {})", sourceBlobName, targetBlobName);
|
||||
logger.trace("move({}, {})", sourceBlobName, targetBlobName);
|
||||
try {
|
||||
String source = keyPath + sourceBlobName;
|
||||
String target = keyPath + targetBlobName;
|
||||
|
|
@ -150,10 +150,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
|
|||
logger.debug("moving blob [{}] to [{}] in container {{}}", source, target, blobStore.container());
|
||||
|
||||
blobStore.moveBlob(blobStore.container(), source, target);
|
||||
} catch (URISyntaxException e) {
|
||||
logger.warn("can not move blob [{}] to [{}] in container {{}}: {}", sourceBlobName, targetBlobName, blobStore.container(), e.getMessage());
|
||||
throw new IOException(e);
|
||||
} catch (StorageException e) {
|
||||
} catch (URISyntaxException | StorageException e) {
|
||||
logger.warn("can not move blob [{}] to [{}] in container {{}}: {}", sourceBlobName, targetBlobName, blobStore.container(), e.getMessage());
|
||||
throw new IOException(e);
|
||||
}
|
||||
|
|
@ -161,7 +158,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
|
|||
|
||||
@Override
|
||||
public Map<String, BlobMetaData> listBlobs() throws IOException {
|
||||
logger.debug("listBlobs()");
|
||||
logger.trace("listBlobs()");
|
||||
return listBlobsByPrefix(null);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -182,18 +182,32 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
|
|||
CloudBlobClient client = this.getSelectedClient(account, mode);
|
||||
CloudBlobContainer blob_container = client.getContainerReference(container);
|
||||
if (blob_container.exists()) {
|
||||
// We list the blobs using a flat blob listing mode
|
||||
for (ListBlobItem blobItem : blob_container.listBlobs(path, true)) {
|
||||
String blobName = blobNameFromUri(container, blobItem.getUri());
|
||||
String blobName = blobNameFromUri(blobItem.getUri());
|
||||
logger.trace("removing blob [{}] full URI was [{}]", blobName, blobItem.getUri());
|
||||
deleteBlob(account, mode, container, blobName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static String blobNameFromUri(String container, URI uri) {
|
||||
/**
|
||||
* Extract the blob name from a URI like https://myservice.azure.net/container/path/to/myfile
|
||||
* It should remove the container part (first part of the path) and gives path/to/myfile
|
||||
* @param uri URI to parse
|
||||
* @return The blob name relative to the container
|
||||
*/
|
||||
public static String blobNameFromUri(URI uri) {
|
||||
String path = uri.getPath();
|
||||
|
||||
// We remove the container name from the path
|
||||
return path.replaceFirst("/" + container + "/", "");
|
||||
// The 3 magic number cames from the fact we have // in the first part of the URI (protocol)
|
||||
// Then a / after the server address
|
||||
// And we finally split after the container/
|
||||
String[] splits = path.split("/", 3);
|
||||
|
||||
// We return the remaining end of the string
|
||||
return splits[2];
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import java.util.Collection;
|
|||
/**
|
||||
* Base class for Azure tests.
|
||||
*/
|
||||
public abstract class AbstractAzureTestCase extends ESIntegTestCase {
|
||||
public abstract class AbstractAzureIntegTestCase extends ESIntegTestCase {
|
||||
|
||||
@Override
|
||||
protected Collection<Class<? extends Plugin>> nodePlugins() {
|
||||
|
|
@ -38,7 +38,7 @@ import org.junit.Before;
|
|||
import java.net.URISyntaxException;
|
||||
import java.util.Collection;
|
||||
|
||||
public abstract class AbstractAzureRepositoryServiceTestCase extends AbstractAzureTestCase {
|
||||
public abstract class AbstractAzureRepositoryServiceIntegTestCase extends AbstractAzureIntegTestCase {
|
||||
|
||||
public static class TestPlugin extends Plugin {
|
||||
@Override
|
||||
|
|
@ -56,7 +56,7 @@ public abstract class AbstractAzureRepositoryServiceTestCase extends AbstractAzu
|
|||
|
||||
protected String basePath;
|
||||
|
||||
public AbstractAzureRepositoryServiceTestCase(String basePath) {
|
||||
public AbstractAzureRepositoryServiceIntegTestCase(String basePath) {
|
||||
this.basePath = basePath;
|
||||
}
|
||||
|
||||
|
|
@ -70,7 +70,7 @@ public abstract class AbstractAzureRepositoryServiceTestCase extends AbstractAzu
|
|||
}
|
||||
for (String repository : repositories) {
|
||||
try {
|
||||
client().admin().cluster().prepareDeleteRepository(repository).execute().actionGet();
|
||||
client().admin().cluster().prepareDeleteRepository(repository).get();
|
||||
} catch (RepositoryMissingException ex) {
|
||||
// ignore
|
||||
}
|
||||
|
|
@ -19,16 +19,15 @@
|
|||
|
||||
package org.elasticsearch.cloud.azure;
|
||||
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.io.PathUtils;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.settings.SettingsException;
|
||||
import org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
import static org.elasticsearch.cloud.azure.AzureTestUtils.readSettingsFromFile;
|
||||
|
||||
/**
|
||||
* Base class for Azure tests that require credentials.
|
||||
* <p>
|
||||
|
|
@ -36,14 +35,13 @@ import java.util.Collection;
|
|||
* in order to run these tests.
|
||||
*/
|
||||
@ThirdParty
|
||||
public abstract class AbstractAzureWithThirdPartyTestCase extends AbstractAzureTestCase {
|
||||
public abstract class AbstractAzureWithThirdPartyIntegTestCase extends AbstractAzureIntegTestCase {
|
||||
|
||||
@Override
|
||||
protected Settings nodeSettings(int nodeOrdinal) {
|
||||
return Settings.builder()
|
||||
.put(super.nodeSettings(nodeOrdinal))
|
||||
.put(readSettingsFromFile())
|
||||
// .put("path.home", createTempDir())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
|
@ -51,20 +49,4 @@ public abstract class AbstractAzureWithThirdPartyTestCase extends AbstractAzureT
|
|||
protected Collection<Class<? extends Plugin>> nodePlugins() {
|
||||
return pluginList(AzureRepositoryPlugin.class);
|
||||
}
|
||||
|
||||
protected Settings readSettingsFromFile() {
|
||||
Settings.Builder settings = Settings.builder();
|
||||
|
||||
// if explicit, just load it and don't load from env
|
||||
try {
|
||||
if (Strings.hasText(System.getProperty("tests.config"))) {
|
||||
settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
|
||||
} else {
|
||||
throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
|
||||
}
|
||||
} catch (SettingsException exception) {
|
||||
throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
|
||||
}
|
||||
return settings.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.cloud.azure;
|
||||
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.io.PathUtils;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.settings.SettingsException;
|
||||
|
||||
public class AzureTestUtils {
|
||||
/**
|
||||
* Read settings from file when running integration tests with ThirdParty annotation.
|
||||
* elasticsearch.yml file path has to be set with -Dtests.config=/path/to/elasticsearch.yml.
|
||||
* @return Settings from elasticsearch.yml integration test file (for 3rd party tests)
|
||||
*/
|
||||
public static Settings readSettingsFromFile() {
|
||||
Settings.Builder settings = Settings.builder();
|
||||
|
||||
// if explicit, just load it and don't load from env
|
||||
try {
|
||||
if (Strings.hasText(System.getProperty("tests.config"))) {
|
||||
settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
|
||||
} else {
|
||||
throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and " +
|
||||
"-Dtests.config=/path/to/elasticsearch.yml");
|
||||
}
|
||||
} catch (SettingsException exception) {
|
||||
throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
|
||||
}
|
||||
return settings.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -25,9 +25,10 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
|
||||
import static org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl.blobNameFromUri;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
import static org.hamcrest.Matchers.lessThan;
|
||||
import static org.hamcrest.Matchers.nullValue;
|
||||
|
||||
public class AzureStorageServiceTests extends ESTestCase {
|
||||
|
|
@ -169,4 +170,15 @@ public class AzureStorageServiceTests extends ESTestCase {
|
|||
new CloudBlobClient(URI.create("https://" + azureStorageSettings.getName())));
|
||||
}
|
||||
}
|
||||
|
||||
public void testBlobNameFromUri() throws URISyntaxException {
|
||||
String name = blobNameFromUri(new URI("https://myservice.azure.net/container/path/to/myfile"));
|
||||
assertThat(name, is("path/to/myfile"));
|
||||
name = blobNameFromUri(new URI("http://myservice.azure.net/container/path/to/myfile"));
|
||||
assertThat(name, is("path/to/myfile"));
|
||||
name = blobNameFromUri(new URI("http://127.0.0.1/container/path/to/myfile"));
|
||||
assertThat(name, is("path/to/myfile"));
|
||||
name = blobNameFromUri(new URI("https://127.0.0.1/container/path/to/myfile"));
|
||||
assertThat(name, is("path/to/myfile"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,11 +23,8 @@ import com.microsoft.azure.storage.StorageException;
|
|||
import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
|
||||
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
|
||||
import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.blobstore.BlobStore;
|
||||
import org.elasticsearch.common.io.PathUtils;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.settings.SettingsException;
|
||||
import org.elasticsearch.repositories.RepositoryName;
|
||||
import org.elasticsearch.repositories.RepositorySettings;
|
||||
import org.elasticsearch.test.ESBlobStoreTestCase;
|
||||
|
|
@ -36,6 +33,8 @@ import org.elasticsearch.test.ESIntegTestCase;
|
|||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
|
||||
import static org.elasticsearch.cloud.azure.AzureTestUtils.readSettingsFromFile;
|
||||
|
||||
/**
|
||||
* You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/elasticsearch.yml}
|
||||
* in order to run these tests.
|
||||
|
|
@ -56,21 +55,4 @@ public class AzureBlobStoreTests extends ESBlobStoreTestCase {
|
|||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
protected Settings readSettingsFromFile() {
|
||||
Settings.Builder settings = Settings.builder();
|
||||
|
||||
// if explicit, just load it and don't load from env
|
||||
try {
|
||||
if (Strings.hasText(System.getProperty("tests.config"))) {
|
||||
settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
|
||||
} else {
|
||||
throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
|
||||
}
|
||||
} catch (SettingsException exception) {
|
||||
throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
|
||||
}
|
||||
return settings.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResp
|
|||
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
|
||||
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.cloud.azure.AbstractAzureRepositoryServiceTestCase;
|
||||
import org.elasticsearch.cloud.azure.AbstractAzureRepositoryServiceIntegTestCase;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.ByteSizeUnit;
|
||||
|
|
@ -39,8 +39,8 @@ import static org.hamcrest.Matchers.greaterThan;
|
|||
numDataNodes = 1,
|
||||
numClientNodes = 0,
|
||||
transportClientRatio = 0.0)
|
||||
public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositoryServiceTestCase {
|
||||
public AzureSnapshotRestoreServiceTests() {
|
||||
public class AzureSnapshotRestoreServiceIntegTests extends AbstractAzureRepositoryServiceIntegTestCase {
|
||||
public AzureSnapshotRestoreServiceIntegTests() {
|
||||
super("/snapshot-test/repo-" + randomInt());
|
||||
}
|
||||
|
||||
|
|
@ -69,11 +69,14 @@ public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositorySer
|
|||
assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(100L));
|
||||
|
||||
logger.info("--> snapshot");
|
||||
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
|
||||
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
|
||||
.setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
|
||||
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
|
||||
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
|
||||
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
|
||||
equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
|
||||
|
||||
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
|
||||
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(),
|
||||
equalTo(SnapshotState.SUCCESS));
|
||||
|
||||
logger.info("--> delete some data");
|
||||
for (int i = 0; i < 50; i++) {
|
||||
|
|
@ -94,7 +97,8 @@ public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositorySer
|
|||
client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
|
||||
|
||||
logger.info("--> restore all indices from the snapshot");
|
||||
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
|
||||
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
|
||||
.setWaitForCompletion(true).get();
|
||||
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
|
||||
|
||||
ensureGreen();
|
||||
|
|
@ -106,7 +110,8 @@ public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositorySer
|
|||
logger.info("--> delete indices");
|
||||
cluster().wipeIndices("test-idx-1", "test-idx-2");
|
||||
logger.info("--> restore one index after deletion");
|
||||
restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet();
|
||||
restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true)
|
||||
.setIndices("test-idx-*", "-test-idx-2").get();
|
||||
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
|
||||
ensureGreen();
|
||||
assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
|
||||
|
|
@ -28,7 +28,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRes
|
|||
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.client.ClusterAdminClient;
|
||||
import org.elasticsearch.cloud.azure.AbstractAzureWithThirdPartyTestCase;
|
||||
import org.elasticsearch.cloud.azure.AbstractAzureWithThirdPartyIntegTestCase;
|
||||
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
|
||||
import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
|
|
@ -51,19 +51,20 @@ import java.net.URISyntaxException;
|
|||
import java.util.Locale;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.elasticsearch.cloud.azure.AzureTestUtils.readSettingsFromFile;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.greaterThan;
|
||||
|
||||
/**
|
||||
* This test needs Azure to run and -Dtests.thirdparty=true to be set
|
||||
* and -Dtests.config=/path/to/elasticsearch.yml
|
||||
* @see AbstractAzureWithThirdPartyTestCase
|
||||
* @see AbstractAzureWithThirdPartyIntegTestCase
|
||||
*/
|
||||
@ClusterScope(
|
||||
scope = ESIntegTestCase.Scope.SUITE,
|
||||
numDataNodes = 1,
|
||||
transportClientRatio = 0.0)
|
||||
public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCase {
|
||||
public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyIntegTestCase {
|
||||
private String getRepositoryPath() {
|
||||
String testName = "it-" + getTestName();
|
||||
return testName.contains(" ") ? Strings.split(testName, " ")[0] : testName;
|
||||
|
|
@ -127,11 +128,14 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(100L));
|
||||
|
||||
logger.info("--> snapshot");
|
||||
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
|
||||
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
|
||||
.setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
|
||||
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
|
||||
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
|
||||
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
|
||||
equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
|
||||
|
||||
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
|
||||
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots()
|
||||
.get(0).state(), equalTo(SnapshotState.SUCCESS));
|
||||
|
||||
logger.info("--> delete some data");
|
||||
for (int i = 0; i < 50; i++) {
|
||||
|
|
@ -152,7 +156,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
|
||||
|
||||
logger.info("--> restore all indices from the snapshot");
|
||||
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
|
||||
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
|
||||
.setWaitForCompletion(true).get();
|
||||
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
|
||||
|
||||
ensureGreen();
|
||||
|
|
@ -164,7 +169,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
logger.info("--> delete indices");
|
||||
cluster().wipeIndices("test-idx-1", "test-idx-2");
|
||||
logger.info("--> restore one index after deletion");
|
||||
restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet();
|
||||
restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true)
|
||||
.setIndices("test-idx-*", "-test-idx-2").get();
|
||||
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
|
||||
ensureGreen();
|
||||
assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
|
||||
|
|
@ -204,11 +210,14 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
|
||||
|
||||
logger.info("creating snapshot [{}]", snapshot1Name);
|
||||
CreateSnapshotResponse createSnapshotResponse1 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot1Name).setWaitForCompletion(true).setIndices(indexName).get();
|
||||
CreateSnapshotResponse createSnapshotResponse1 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot1Name)
|
||||
.setWaitForCompletion(true).setIndices(indexName).get();
|
||||
assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(), greaterThan(0));
|
||||
assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse1.getSnapshotInfo().totalShards()));
|
||||
assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(),
|
||||
equalTo(createSnapshotResponse1.getSnapshotInfo().totalShards()));
|
||||
|
||||
assertThat(client.admin().cluster().prepareGetSnapshots(repositoryName).setSnapshots(snapshot1Name).get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
|
||||
assertThat(client.admin().cluster().prepareGetSnapshots(repositoryName).setSnapshots(snapshot1Name).get().getSnapshots()
|
||||
.get(0).state(), equalTo(SnapshotState.SUCCESS));
|
||||
|
||||
logger.info("indexing second document");
|
||||
index(indexName, typeName, Integer.toString(2), "foo", "bar " + Integer.toString(2));
|
||||
|
|
@ -216,17 +225,21 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
assertThat(client.prepareSearch(indexName).setSize(0).get().getHits().totalHits(), equalTo(2L));
|
||||
|
||||
logger.info("creating snapshot [{}]", snapshot2Name);
|
||||
CreateSnapshotResponse createSnapshotResponse2 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot2Name).setWaitForCompletion(true).setIndices(indexName).get();
|
||||
CreateSnapshotResponse createSnapshotResponse2 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot2Name)
|
||||
.setWaitForCompletion(true).setIndices(indexName).get();
|
||||
assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0));
|
||||
assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()));
|
||||
assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(),
|
||||
equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()));
|
||||
|
||||
assertThat(client.admin().cluster().prepareGetSnapshots(repositoryName).setSnapshots(snapshot2Name).get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
|
||||
assertThat(client.admin().cluster().prepareGetSnapshots(repositoryName).setSnapshots(snapshot2Name).get().getSnapshots()
|
||||
.get(0).state(), equalTo(SnapshotState.SUCCESS));
|
||||
|
||||
logger.info("closing index [{}]", indexName);
|
||||
client.admin().indices().prepareClose(indexName).get();
|
||||
|
||||
logger.info("attempting restore from snapshot [{}]", snapshot1Name);
|
||||
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot(repositoryName, snapshot1Name).setWaitForCompletion(true).execute().actionGet();
|
||||
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot(repositoryName, snapshot1Name)
|
||||
.setWaitForCompletion(true).get();
|
||||
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
|
||||
ensureGreen();
|
||||
assertThat(client.prepareSearch(indexName).setSize(0).get().getHits().totalHits(), equalTo(1L));
|
||||
|
|
@ -263,23 +276,30 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L));
|
||||
|
||||
logger.info("--> snapshot 1");
|
||||
CreateSnapshotResponse createSnapshotResponse1 = client.admin().cluster().prepareCreateSnapshot("test-repo1", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").get();
|
||||
CreateSnapshotResponse createSnapshotResponse1 = client.admin().cluster().prepareCreateSnapshot("test-repo1", "test-snap")
|
||||
.setWaitForCompletion(true).setIndices("test-idx-1").get();
|
||||
assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(), greaterThan(0));
|
||||
assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse1.getSnapshotInfo().totalShards()));
|
||||
assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(),
|
||||
equalTo(createSnapshotResponse1.getSnapshotInfo().totalShards()));
|
||||
|
||||
logger.info("--> snapshot 2");
|
||||
CreateSnapshotResponse createSnapshotResponse2 = client.admin().cluster().prepareCreateSnapshot("test-repo2", "test-snap").setWaitForCompletion(true).setIndices("test-idx-2").get();
|
||||
CreateSnapshotResponse createSnapshotResponse2 = client.admin().cluster().prepareCreateSnapshot("test-repo2", "test-snap")
|
||||
.setWaitForCompletion(true).setIndices("test-idx-2").get();
|
||||
assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0));
|
||||
assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()));
|
||||
assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(),
|
||||
equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()));
|
||||
|
||||
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo1").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
|
||||
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo2").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
|
||||
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo1").setSnapshots("test-snap").get().getSnapshots().get(0).state(),
|
||||
equalTo(SnapshotState.SUCCESS));
|
||||
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo2").setSnapshots("test-snap").get().getSnapshots().get(0).state(),
|
||||
equalTo(SnapshotState.SUCCESS));
|
||||
|
||||
// Test restore after index deletion
|
||||
logger.info("--> delete indices");
|
||||
cluster().wipeIndices("test-idx-1", "test-idx-2");
|
||||
logger.info("--> restore one index after deletion from snapshot 1");
|
||||
RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin().cluster().prepareRestoreSnapshot("test-repo1", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").execute().actionGet();
|
||||
RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin().cluster().prepareRestoreSnapshot("test-repo1", "test-snap")
|
||||
.setWaitForCompletion(true).setIndices("test-idx-1").get();
|
||||
assertThat(restoreSnapshotResponse1.getRestoreInfo().totalShards(), greaterThan(0));
|
||||
ensureGreen();
|
||||
assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
|
||||
|
|
@ -288,7 +308,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
|
||||
|
||||
logger.info("--> restore other index after deletion from snapshot 2");
|
||||
RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin().cluster().prepareRestoreSnapshot("test-repo2", "test-snap").setWaitForCompletion(true).setIndices("test-idx-2").execute().actionGet();
|
||||
RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin().cluster().prepareRestoreSnapshot("test-repo2", "test-snap")
|
||||
.setWaitForCompletion(true).setIndices("test-idx-2").get();
|
||||
assertThat(restoreSnapshotResponse2.getRestoreInfo().totalShards(), greaterThan(0));
|
||||
ensureGreen();
|
||||
assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L));
|
||||
|
|
@ -324,7 +345,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
assertThat(client.prepareGetSnapshots("test-repo").get().getSnapshots().size(), equalTo(0));
|
||||
|
||||
logger.info("--> snapshot");
|
||||
CreateSnapshotResponse createSnapshotResponse = client.prepareCreateSnapshot("test-repo", "test-snap-26").setWaitForCompletion(true).setIndices("test-idx-*").get();
|
||||
CreateSnapshotResponse createSnapshotResponse = client.prepareCreateSnapshot("test-repo", "test-snap-26")
|
||||
.setWaitForCompletion(true).setIndices("test-idx-*").get();
|
||||
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
|
||||
|
||||
// Get all snapshots - should have one
|
||||
|
|
@ -346,7 +368,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
assertThat(client.prepareGetSnapshots("test-repo").get().getSnapshots().size(), equalTo(0));
|
||||
|
||||
logger.info("--> snapshot");
|
||||
createSnapshotResponse = client.prepareCreateSnapshot("test-repo", "test-snap-26").setWaitForCompletion(true).setIndices("test-idx-*").get();
|
||||
createSnapshotResponse = client.prepareCreateSnapshot("test-repo", "test-snap-26").setWaitForCompletion(true)
|
||||
.setIndices("test-idx-*").get();
|
||||
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
|
||||
|
||||
// Get all snapshots - should have one
|
||||
|
|
@ -407,30 +430,26 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
logger.info("--> creating azure repository with container name [{}]", container);
|
||||
// It could happen that we just removed from a previous test the same container so
|
||||
// we can not create it yet.
|
||||
assertBusy(new Runnable() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
assertBusy(() -> {
|
||||
try {
|
||||
PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("azure").setSettings(Settings.builder()
|
||||
.put(Repository.CONTAINER_SETTING.getKey(), container)
|
||||
.put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
|
||||
.put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
|
||||
).get();
|
||||
client().admin().cluster().prepareDeleteRepository("test-repo").get();
|
||||
try {
|
||||
PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
|
||||
.setType("azure").setSettings(Settings.builder()
|
||||
.put(Repository.CONTAINER_SETTING.getKey(), container)
|
||||
.put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
|
||||
.put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
|
||||
).get();
|
||||
client().admin().cluster().prepareDeleteRepository("test-repo").get();
|
||||
try {
|
||||
logger.info("--> remove container [{}]", container);
|
||||
cleanRepositoryFiles(container);
|
||||
} catch (StorageException | URISyntaxException e) {
|
||||
// We can ignore that as we just try to clean after the test
|
||||
}
|
||||
assertTrue(putRepositoryResponse.isAcknowledged() == correct);
|
||||
} catch (RepositoryVerificationException e) {
|
||||
if (correct) {
|
||||
logger.debug(" -> container is being removed. Let's wait a bit...");
|
||||
fail();
|
||||
}
|
||||
logger.info("--> remove container [{}]", container);
|
||||
cleanRepositoryFiles(container);
|
||||
} catch (StorageException | URISyntaxException e) {
|
||||
// We can ignore that as we just try to clean after the test
|
||||
}
|
||||
assertTrue(putRepositoryResponse.isAcknowledged() == correct);
|
||||
} catch (RepositoryVerificationException e) {
|
||||
if (correct) {
|
||||
logger.debug(" -> container is being removed. Let's wait a bit...");
|
||||
fail();
|
||||
}
|
||||
}
|
||||
}, 5, TimeUnit.MINUTES);
|
||||
|
|
@ -452,7 +471,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
|
||||
logger.info("--> restore non existing snapshot");
|
||||
try {
|
||||
client.admin().cluster().prepareRestoreSnapshot("test-repo", "no-existing-snapshot").setWaitForCompletion(true).execute().actionGet();
|
||||
client.admin().cluster().prepareRestoreSnapshot("test-repo", "no-existing-snapshot").setWaitForCompletion(true).get();
|
||||
fail("Shouldn't be here");
|
||||
} catch (SnapshotMissingException ex) {
|
||||
// Expected
|
||||
|
|
@ -468,21 +487,17 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
|
||||
// It could happen that we run this test really close to a previous one
|
||||
// so we might need some time to be able to create the container
|
||||
assertBusy(new Runnable() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
storageService.createContainer(null, LocationMode.PRIMARY_ONLY, container);
|
||||
logger.debug(" -> container created...");
|
||||
} catch (URISyntaxException e) {
|
||||
// Incorrect URL. This should never happen.
|
||||
fail();
|
||||
} catch (StorageException e) {
|
||||
// It could happen. Let's wait for a while.
|
||||
logger.debug(" -> container is being removed. Let's wait a bit...");
|
||||
fail();
|
||||
}
|
||||
assertBusy(() -> {
|
||||
try {
|
||||
storageService.createContainer(null, LocationMode.PRIMARY_ONLY, container);
|
||||
logger.debug(" -> container created...");
|
||||
} catch (URISyntaxException e) {
|
||||
// Incorrect URL. This should never happen.
|
||||
fail();
|
||||
} catch (StorageException e) {
|
||||
// It could happen. Let's wait for a while.
|
||||
logger.debug(" -> container is being removed. Let's wait a bit...");
|
||||
fail();
|
||||
}
|
||||
}, 30, TimeUnit.SECONDS);
|
||||
storageService.removeContainer(null, LocationMode.PRIMARY_ONLY, container);
|
||||
|
|
@ -510,7 +525,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
|
|||
}
|
||||
for (String repository : repositories) {
|
||||
try {
|
||||
client().admin().cluster().prepareDeleteRepository(repository).execute().actionGet();
|
||||
client().admin().cluster().prepareDeleteRepository(repository).get();
|
||||
} catch (RepositoryMissingException ex) {
|
||||
// ignore
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue