honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove tabs

This commit is contained in:
Robert Muir 2015-05-22 00:08:15 -04:00
parent 0c92e167ae
commit 8dabf10c26
2 changed files with 105 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/main/java/org/elasticsearch/cloud/aws/blobstore/DefaultS3OutputStream.java
View File

@ -44,10 +44,10 @@ import java.util.List;
* <p/> * <p/>
* Quick facts about S3: * Quick facts about S3:
* <p/> * <p/>
* Maximum object size: 5 TB * Maximum object size: 5 TB
* Maximum number of parts per upload: 10,000 * Maximum number of parts per upload: 10,000
* Part numbers: 1 to 10,000 (inclusive) * Part numbers: 1 to 10,000 (inclusive)
* Part size: 5 MB to 5 GB, last part can be < 5 MB * Part size: 5 MB to 5 GB, last part can be < 5 MB
* <p/> * <p/>
* See http://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html * See http://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html
* See http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html * See http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html
@ -120,7 +120,7 @@ public class DefaultS3OutputStream extends S3OutputStream {
} }
protected void doUpload(S3BlobStore blobStore, String bucketName, String blobName, InputStream is, int length, protected void doUpload(S3BlobStore blobStore, String bucketName, String blobName, InputStream is, int length,
boolean serverSideEncryption) throws AmazonS3Exception { boolean serverSideEncryption) throws AmazonS3Exception {
ObjectMetadata md = new ObjectMetadata(); ObjectMetadata md = new ObjectMetadata();
if (serverSideEncryption) { if (serverSideEncryption) {
md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
@ -202,15 +202,15 @@ public class DefaultS3OutputStream extends S3OutputStream {
} }
protected PartETag doUploadMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId, InputStream is, protected PartETag doUploadMultipart(S3BlobStore blobStore, String bucketName, String blobName, String uploadId, InputStream is,
int length, boolean lastPart) throws AmazonS3Exception { int length, boolean lastPart) throws AmazonS3Exception {
UploadPartRequest request = new UploadPartRequest() UploadPartRequest request = new UploadPartRequest()
.withBucketName(bucketName) .withBucketName(bucketName)
.withKey(blobName) .withKey(blobName)
.withUploadId(uploadId) .withUploadId(uploadId)
.withPartNumber(multipartChunks) .withPartNumber(multipartChunks)
.withInputStream(is) .withInputStream(is)
.withPartSize(length) .withPartSize(length)
.withLastPart(lastPart); .withLastPart(lastPart);
UploadPartResult response = blobStore.client().uploadPart(request); UploadPartResult response = blobStore.client().uploadPart(request);
return response.getPartETag(); return response.getPartETag();

186
src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
View File

@ -91,7 +91,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest {
.setType("s3").setSettings(ImmutableSettings.settingsBuilder() .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
.put("base_path", basePath) .put("base_path", basePath)
.put("chunk_size", randomIntBetween(1000, 10000)) .put("chunk_size", randomIntBetween(1000, 10000))
).get(); ).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
createIndex("test-idx-1", "test-idx-2", "test-idx-3"); createIndex("test-idx-1", "test-idx-2", "test-idx-3");
@ -154,94 +154,94 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest {
assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
} }
@Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211") @Test @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211")
public void testEncryption() { public void testEncryption() {
Client client = client(); Client client = client();
logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath);
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("s3").setSettings(ImmutableSettings.settingsBuilder() .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
.put("base_path", basePath) .put("base_path", basePath)
.put("chunk_size", randomIntBetween(1000, 10000)) .put("chunk_size", randomIntBetween(1000, 10000))
.put("server_side_encryption", true) .put("server_side_encryption", true)
).get(); ).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
createIndex("test-idx-1", "test-idx-2", "test-idx-3"); createIndex("test-idx-1", "test-idx-2", "test-idx-3");
ensureGreen(); ensureGreen();
logger.info("--> indexing some data"); logger.info("--> indexing some data");
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i); index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i); index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i); index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
} }
refresh(); refresh();
assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L));
assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L));
assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(100L));
logger.info("--> snapshot"); logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get(); CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0)); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())); assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS)); assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
Settings settings = internalCluster().getInstance(Settings.class); Settings settings = internalCluster().getInstance(Settings.class);
Settings bucket = settings.getByPrefix("repositories.s3."); Settings bucket = settings.getByPrefix("repositories.s3.");
AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client( AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(
null, null,
null, null,
bucket.get("region", settings.get("repositories.s3.region")), bucket.get("region", settings.get("repositories.s3.region")),
bucket.get("access_key", settings.get("cloud.aws.access_key")), bucket.get("access_key", settings.get("cloud.aws.access_key")),
bucket.get("secret_key", settings.get("cloud.aws.secret_key"))); bucket.get("secret_key", settings.get("cloud.aws.secret_key")));
String bucketName = bucket.get("bucket"); String bucketName = bucket.get("bucket");
logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath); logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
List<S3ObjectSummary> summaries = s3Client.listObjects(bucketName, basePath).getObjectSummaries(); List<S3ObjectSummary> summaries = s3Client.listObjects(bucketName, basePath).getObjectSummaries();
for (S3ObjectSummary summary : summaries) { for (S3ObjectSummary summary : summaries) {
assertThat(s3Client.getObjectMetadata(bucketName, summary.getKey()).getSSEAlgorithm(), equalTo("AES256")); assertThat(s3Client.getObjectMetadata(bucketName, summary.getKey()).getSSEAlgorithm(), equalTo("AES256"));
} }
logger.info("--> delete some data"); logger.info("--> delete some data");
for (int i = 0; i < 50; i++) { for (int i = 0; i < 50; i++) {
client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get(); client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get();
} }
for (int i = 50; i < 100; i++) { for (int i = 50; i < 100; i++) {
client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get(); client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get();
} }
for (int i = 0; i < 100; i += 2) { for (int i = 0; i < 100; i += 2) {
client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get(); client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get();
} }
refresh(); refresh();
assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L)); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(50L));
assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(50L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(50L));
assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L));
logger.info("--> close indices"); logger.info("--> close indices");
client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get(); client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
logger.info("--> restore all indices from the snapshot"); logger.info("--> restore all indices from the snapshot");
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet(); RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
ensureGreen(); ensureGreen();
assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L));
assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-2").get().getCount(), equalTo(100L));
assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L)); assertThat(client.prepareCount("test-idx-3").get().getCount(), equalTo(50L));
// Test restore after index deletion // Test restore after index deletion
logger.info("--> delete indices"); logger.info("--> delete indices");
cluster().wipeIndices("test-idx-1", "test-idx-2"); cluster().wipeIndices("test-idx-1", "test-idx-2");
logger.info("--> restore one index after deletion"); logger.info("--> restore one index after deletion");
restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet(); restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
ensureGreen(); ensureGreen();
assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L)); assertThat(client.prepareCount("test-idx-1").get().getCount(), equalTo(100L));
ClusterState clusterState = client.admin().cluster().prepareState().get().getState(); ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true)); assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false)); assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
} }
/** /**
@ -254,9 +254,9 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest {
Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket."); Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.private-bucket.");
logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath);
client.admin().cluster().preparePutRepository("test-repo") client.admin().cluster().preparePutRepository("test-repo")
.setType("s3").setSettings(ImmutableSettings.settingsBuilder() .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
.put("base_path", basePath) .put("base_path", basePath)
.put("bucket", bucketSettings.get("bucket")) .put("bucket", bucketSettings.get("bucket"))
).get(); ).get();
fail("repository verification should have raise an exception!"); fail("repository verification should have raise an exception!");
} }
@ -273,7 +273,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest {
.put("access_key", bucketSettings.get("access_key")) .put("access_key", bucketSettings.get("access_key"))
.put("secret_key", bucketSettings.get("secret_key")) .put("secret_key", bucketSettings.get("secret_key"))
.put("bucket", bucketSettings.get("bucket")) .put("bucket", bucketSettings.get("bucket"))
).get(); ).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
assertRepositoryIsOperational(client, "test-repo"); assertRepositoryIsOperational(client, "test-repo");
@ -291,7 +291,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest {
.put("access_key", bucketSettings.get("access_key")) .put("access_key", bucketSettings.get("access_key"))
.put("secret_key", bucketSettings.get("secret_key")) .put("secret_key", bucketSettings.get("secret_key"))
.put("base_path", basePath) .put("base_path", basePath)
).get(); ).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
assertRepositoryIsOperational(client, "test-repo"); assertRepositoryIsOperational(client, "test-repo");
} }
@ -306,11 +306,11 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest {
Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.remote-bucket."); Settings bucketSettings = internalCluster().getInstance(Settings.class).getByPrefix("repositories.s3.remote-bucket.");
logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", bucketSettings.get("bucket"), basePath);
client.admin().cluster().preparePutRepository("test-repo") client.admin().cluster().preparePutRepository("test-repo")
.setType("s3").setSettings(ImmutableSettings.settingsBuilder() .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
.put("base_path", basePath) .put("base_path", basePath)
.put("bucket", bucketSettings.get("bucket")) .put("bucket", bucketSettings.get("bucket"))
// Below setting intentionally omitted to assert bucket is not available in default region. // Below setting intentionally omitted to assert bucket is not available in default region.
// .put("region", privateBucketSettings.get("region")) // .put("region", privateBucketSettings.get("region"))
).get(); ).get();
fail("repository verification should have raise an exception!"); fail("repository verification should have raise an exception!");
@ -327,7 +327,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest {
.put("base_path", basePath) .put("base_path", basePath)
.put("bucket", bucketSettings.get("bucket")) .put("bucket", bucketSettings.get("bucket"))
.put("region", bucketSettings.get("region")) .put("region", bucketSettings.get("region"))
).get(); ).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
assertRepositoryIsOperational(client, "test-repo"); assertRepositoryIsOperational(client, "test-repo");
@ -342,8 +342,8 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest {
logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath); logger.info("--> creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath);
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo") PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
.setType("s3").setSettings(ImmutableSettings.settingsBuilder() .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
.put("base_path", basePath) .put("base_path", basePath)
).get(); ).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
logger.info("--> restore non existing snapshot"); logger.info("--> restore non existing snapshot");
@ -364,8 +364,8 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest {
logger.info("--> creating s3 repository without any path"); logger.info("--> creating s3 repository without any path");
PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo") PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo")
.setType("s3").setSettings(ImmutableSettings.settingsBuilder() .setType("s3").setSettings(ImmutableSettings.settingsBuilder()
.put("base_path", basePath) .put("base_path", basePath)
).get(); ).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
try { try {
@ -383,7 +383,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest {
} }
} }
private void assertRepositoryIsOperational(Client client, String repository) { private void assertRepositoryIsOperational(Client client, String repository) {
createIndex("test-idx-1"); createIndex("test-idx-1");
ensureGreen(); ensureGreen();
@ -447,7 +447,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest {
settings.getByPrefix("repositories.s3.private-bucket."), settings.getByPrefix("repositories.s3.private-bucket."),
settings.getByPrefix("repositories.s3.remote-bucket."), settings.getByPrefix("repositories.s3.remote-bucket."),
settings.getByPrefix("repositories.s3.external-bucket.") settings.getByPrefix("repositories.s3.external-bucket.")
}; };
for (Settings bucket : buckets) { for (Settings bucket : buckets) {
String endpoint = bucket.get("endpoint", settings.get("repositories.s3.endpoint")); String endpoint = bucket.get("endpoint", settings.get("repositories.s3.endpoint"));
String protocol = bucket.get("protocol", settings.get("repositories.s3.protocol")); String protocol = bucket.get("protocol", settings.get("repositories.s3.protocol"));