From c35093b2a4e8533017bf1bef5d9a33a06b00c635 Mon Sep 17 00:00:00 2001 From: Jun Ohtani Date: Fri, 31 Oct 2014 14:17:45 +0900 Subject: [PATCH 1/9] Update to Lucene 4.10.2 Closes #130 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e38947701b6..150431ae305 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ 2.0.0-SNAPSHOT - 4.10.1 + 4.10.2 1.7.13 onerror true From 4658f0e2dfd20f7bcfa5d8cdb184770af1610bb1 Mon Sep 17 00:00:00 2001 From: Jun Ohtani Date: Fri, 31 Oct 2014 14:32:33 +0900 Subject: [PATCH 2/9] Tests: Fix randomizedtest fail Closes #131 --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 150431ae305..55d7b8ac5c8 100644 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,12 @@ 1.3.RC2 test + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + 2.1.10 + test + org.apache.lucene lucene-test-framework From 338a94dbe650cdc58f6bb6ae1c1308700629606d Mon Sep 17 00:00:00 2001 From: tlrx Date: Tue, 4 Nov 2014 10:02:52 +0100 Subject: [PATCH 3/9] Fix constructors of ZenDiscovery's sub classes Closes #133 --- .../java/org/elasticsearch/discovery/ec2/Ec2Discovery.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java index 64bb57ac9fe..dd137d0c242 100755 --- a/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java +++ b/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java @@ -23,6 +23,7 @@ import org.elasticsearch.cloud.aws.AwsEc2Service; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.node.DiscoveryNodeService; +import org.elasticsearch.cluster.settings.DynamicSettings; import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -45,9 +46,9 @@ public class Ec2Discovery extends ZenDiscovery { public Ec2Discovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, DiscoveryNodeService discoveryNodeService, AwsEc2Service ec2Service, DiscoverySettings discoverySettings, - ElectMasterService electMasterService) { + ElectMasterService electMasterService, DynamicSettings dynamicSettings) { super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, - discoveryNodeService, pingService, electMasterService, discoverySettings); + discoveryNodeService, pingService, electMasterService, discoverySettings, dynamicSettings); if (settings.getAsBoolean("cloud.enabled", true)) { ImmutableList zenPings = pingService.zenPings(); UnicastZenPing unicastZenPing = null; From 3cfb2df686fd539f08b20b1267b4fa0ab8268d77 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 5 Nov 2014 17:49:08 -0500 Subject: [PATCH 4/9] upgrade to lucene 5 snapshot --- pom.xml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 55d7b8ac5c8..182d860f0ef 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,8 @@ 2.0.0-SNAPSHOT - 4.10.2 + 5.0.0 + 5.0.0-snapshot-1636426 1.7.13 onerror true @@ -48,6 +49,10 @@ sonatype http://oss.sonatype.org/content/repositories/releases/ + + Lucene snapshots + https://download.elasticsearch.org/lucenesnapshots/maven/ + @@ -72,7 +77,7 @@ org.apache.lucene lucene-test-framework - ${lucene.version} + ${lucene.maven.version} test @@ -85,7 +90,7 @@ org.apache.lucene lucene-core - ${lucene.version} + ${lucene.maven.version} provided From 4be25988d1da58f68a61d6d9a4dcd7cb06578f9e Mon Sep 17 00:00:00 2001 From: tlrx Date: Fri, 7 Nov 2014 09:52:38 +0100 Subject: [PATCH 5/9] Update S3BlobContainer because BlobContainer changed See elasticsearch/elasticsearch#8366 --- .../cloud/aws/blobstore/S3BlobContainer.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java index d5a10079f6d..d5e231b70ba 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java @@ -19,6 +19,7 @@ package org.elasticsearch.cloud.aws.blobstore; +import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3Object; @@ -68,9 +69,12 @@ public class S3BlobContainer extends AbstractBlobContainer { } @Override - public boolean deleteBlob(String blobName) throws IOException { - blobStore.client().deleteObject(blobStore.bucket(), buildKey(blobName)); - return true; + public void deleteBlob(String blobName) throws IOException { + try { + blobStore.client().deleteObject(blobStore.bucket(), buildKey(blobName)); + } catch (AmazonClientException e) { + throw new IOException("Exception when deleting blob [" + blobName + "]", e); + } } @Override From 65bda6238839b43c89644bd38cc8dab6c5d8a6e0 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Mon, 10 Nov 2014 16:46:24 -0500 Subject: [PATCH 6/9] Upgrade to Lucene 5.0.0-snapshot-1637347 --- pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 182d860f0ef..1c77b5e8749 100644 --- a/pom.xml +++ b/pom.xml @@ -34,7 +34,7 @@ 2.0.0-SNAPSHOT 5.0.0 - 5.0.0-snapshot-1636426 + 5.0.0-snapshot-1637347 1.7.13 onerror true @@ -46,12 +46,12 @@ - sonatype - http://oss.sonatype.org/content/repositories/releases/ + Lucene snapshots + https://download.elasticsearch.org/lucenesnapshots/1637347/ - Lucene snapshots - https://download.elasticsearch.org/lucenesnapshots/maven/ + sonatype + http://oss.sonatype.org/content/repositories/releases/ From ea91adf6a1ed28c1a93a6c5d38a6e02ed295ed07 Mon Sep 17 00:00:00 2001 From: tlrx Date: Tue, 4 Nov 2014 18:33:42 +0100 Subject: [PATCH 7/9] Add retry logic for S3 connection errors when restoring snapshots This commit adds a retry logic when reading blobs from S3. It also adds a retry logic when initializing a multipart upload and sets the internal "max retries" parameter of the Amazon S3 client with the same value as the "max_retry" parameter set for the snapshot repository (so in worst cases with the default value set to 3, 3x3=9 attempts will be made). The internal S3 client uses an exponential back off strategy between each connection exception (mainly IOException). Closes elasticsearch/elasticsearch#8280 --- .../elasticsearch/cloud/aws/AwsS3Service.java | 2 + .../cloud/aws/InternalAwsS3Service.java | 18 ++++++-- .../aws/blobstore/DefaultS3OutputStream.java | 42 +++++++++++-------- .../cloud/aws/blobstore/S3BlobContainer.java | 26 ++++++++---- .../cloud/aws/blobstore/S3BlobStore.java | 22 ++++++---- .../repositories/s3/S3Repository.java | 8 +++- 6 files changed, 79 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java index af0147670a1..fb01a0b9705 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java @@ -29,4 +29,6 @@ public interface AwsS3Service extends LifecycleComponent { AmazonS3 client(); AmazonS3 client(String region, String account, String key); + + AmazonS3 client(String region, String account, String key, Integer maxRetries); } diff --git a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 75efa1601f2..a5828e40e0a 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -60,11 +60,16 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent clientDescriptor = new Tuple(endpoint, account); AmazonS3Client client = clients.get(clientDescriptor); if (client != null) { @@ -111,6 +116,11 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent(); + int retry = 0; + while ((retry <= getNumberOfRetries()) && (multipartId == null)) { + try { + multipartId = doInitialize(getBlobStore(), getBucketName(), getBlobName(), isServerSideEncryption()); + if (multipartId != null) { + multipartChunks = 1; + multiparts = new ArrayList<>(); + } + } catch (AmazonClientException e) { + if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) { + retry++; + } else { + throw e; + } } } } @@ -145,14 +155,14 @@ public class DefaultS3OutputStream extends S3OutputStream { private void uploadMultipart(byte[] bytes, int off, int len, boolean lastPart) throws IOException { try (ByteArrayInputStream is = new ByteArrayInputStream(bytes, off, len)) { int retry = 0; - while (retry < getNumberOfRetries()) { + while (retry <= getNumberOfRetries()) { try { PartETag partETag = doUploadMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId, is, len, lastPart); multiparts.add(partETag); multipartChunks++; return; - } catch (AmazonS3Exception e) { - if (shouldRetry(e) && retry < getNumberOfRetries()) { + } catch (AmazonClientException e) { + if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) { is.reset(); retry++; } else { @@ -182,13 +192,13 @@ public class DefaultS3OutputStream extends S3OutputStream { private void completeMultipart() { int retry = 0; - while (retry < getNumberOfRetries()) { + while (retry <= getNumberOfRetries()) { try { doCompleteMultipart(getBlobStore(), getBucketName(), getBlobName(), multipartId, multiparts); multipartId = null; return; - } catch (AmazonS3Exception e) { - if (shouldRetry(e) && retry < getNumberOfRetries()) { + } catch (AmazonClientException e) { + if (getBlobStore().shouldRetry(e) && retry < getNumberOfRetries()) { retry++; } else { abortMultipart(); @@ -218,8 +228,4 @@ public class DefaultS3OutputStream extends S3OutputStream { throws AmazonS3Exception { blobStore.client().abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, blobName, uploadId)); } - - protected boolean shouldRetry(AmazonS3Exception e) { - return e.getStatusCode() == 400 && "RequestTimeout".equals(e.getErrorCode()); - } } diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java index d5e231b70ba..299633d2a17 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java @@ -79,20 +79,30 @@ public class S3BlobContainer extends AbstractBlobContainer { @Override public InputStream openInput(String blobName) throws IOException { - try { - S3Object s3Object = blobStore.client().getObject(blobStore.bucket(), buildKey(blobName)); - return s3Object.getObjectContent(); - } catch (AmazonS3Exception e) { - if (e.getStatusCode() == 404) { - throw new FileNotFoundException(e.getMessage()); + int retry = 0; + while (retry <= blobStore.numberOfRetries()) { + try { + S3Object s3Object = blobStore.client().getObject(blobStore.bucket(), buildKey(blobName)); + return s3Object.getObjectContent(); + } catch (AmazonClientException e) { + if (blobStore.shouldRetry(e) && (retry < blobStore.numberOfRetries())) { + retry++; + } else { + if (e instanceof AmazonS3Exception) { + if (404 == ((AmazonS3Exception) e).getStatusCode()) { + throw new FileNotFoundException(e.getMessage()); + } + } + throw e; + } } - throw e; } + throw new BlobStoreException("retries exhausted while attempting to access blob object [name:" + blobName + ", bucket:" + blobStore.bucket() +"]"); } @Override public OutputStream createOutput(final String blobName) throws IOException { - // UploadS3OutputStream does buffering internally + // UploadS3OutputStream does buffering & retry logic internally return new DefaultS3OutputStream(blobStore, blobStore.bucket(), buildKey(blobName), blobStore.bufferSizeInBytes(), blobStore.numberOfRetries(), blobStore.serverSideEncryption()); } diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java index 10ce6c7373f..fcb9e67fbce 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java @@ -19,7 +19,9 @@ package org.elasticsearch.cloud.aws.blobstore; +import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.ObjectListing; @@ -55,12 +57,8 @@ public class S3BlobStore extends AbstractComponent implements BlobStore { private final int numberOfRetries; - - public S3BlobStore(Settings settings, AmazonS3 client, String bucket, String region, boolean serverSideEncryption) { - this(settings, client, bucket, region, serverSideEncryption, null); - } - - public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, boolean serverSideEncryption, ByteSizeValue bufferSize) { + public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, boolean serverSideEncryption, + ByteSizeValue bufferSize, int maxRetries) { super(settings); this.client = client; this.bucket = bucket; @@ -72,7 +70,7 @@ public class S3BlobStore extends AbstractComponent implements BlobStore { throw new BlobStoreException("\"Detected a buffer_size for the S3 storage lower than [" + MIN_BUFFER_SIZE + "]"); } - this.numberOfRetries = settings.getAsInt("max_retries", 3); + this.numberOfRetries = maxRetries; if (!client.doesBucketExist(bucket)) { if (region != null) { client.createBucket(bucket, region); @@ -152,6 +150,16 @@ public class S3BlobStore extends AbstractComponent implements BlobStore { } } + protected boolean shouldRetry(AmazonClientException e) { + if (e instanceof AmazonS3Exception) { + AmazonS3Exception s3e = (AmazonS3Exception)e; + if (s3e.getStatusCode() == 400 && "RequestTimeout".equals(s3e.getErrorCode())) { + return true; + } + } + return e.isRetryable(); + } + @Override public void close() { } diff --git a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 4948364e758..f4bcd5bcec5 100644 --- a/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -120,10 +120,14 @@ public class S3Repository extends BlobStoreRepository { boolean serverSideEncryption = repositorySettings.settings().getAsBoolean("server_side_encryption", componentSettings.getAsBoolean("server_side_encryption", false)); ByteSizeValue bufferSize = repositorySettings.settings().getAsBytesSize("buffer_size", componentSettings.getAsBytesSize("buffer_size", null)); - logger.debug("using bucket [{}], region [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}]", bucket, region, chunkSize, serverSideEncryption, bufferSize); - blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key")), bucket, region, serverSideEncryption, bufferSize); + Integer maxRetries = repositorySettings.settings().getAsInt("max_retries", componentSettings.getAsInt("max_retries", 3)); this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", componentSettings.getAsBytesSize("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB))); this.compress = repositorySettings.settings().getAsBoolean("compress", componentSettings.getAsBoolean("compress", false)); + + logger.debug("using bucket [{}], region [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}]", + bucket, region, chunkSize, serverSideEncryption, bufferSize, maxRetries); + + blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key"), maxRetries), bucket, region, serverSideEncryption, bufferSize, maxRetries); String basePath = repositorySettings.settings().get("base_path", null); if (Strings.hasLength(basePath)) { BlobPath path = new BlobPath(); From e757722291b3c22c08ece398b065c3f5efa4f8cd Mon Sep 17 00:00:00 2001 From: tlrx Date: Mon, 10 Nov 2014 12:11:34 +0100 Subject: [PATCH 8/9] Add random read failures in unit tests to test the retry logic added to S3BlobContainer.openInput() --- .../cloud/aws/blobstore/S3BlobContainer.java | 2 +- .../cloud/aws/AbstractAwsTest.java | 3 +- .../elasticsearch/cloud/aws/TestAmazonS3.java | 46 +++++++++++++++++-- .../cloud/aws/TestAwsS3Service.java | 5 ++ 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java index 299633d2a17..d560ceca348 100644 --- a/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java +++ b/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java @@ -90,7 +90,7 @@ public class S3BlobContainer extends AbstractBlobContainer { } else { if (e instanceof AmazonS3Exception) { if (404 == ((AmazonS3Exception) e).getStatusCode()) { - throw new FileNotFoundException(e.getMessage()); + throw new FileNotFoundException("Blob object [" + blobName + "] not found: " + e.getMessage()); } } throw e; diff --git a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java index 3ea6470cb58..537fa058e80 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java +++ b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java @@ -60,7 +60,8 @@ public abstract class AbstractAwsTest extends ElasticsearchIntegrationTest { .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) .put(AwsModule.S3_SERVICE_TYPE_KEY, TestAwsS3Service.class) .put("cloud.aws.test.random", randomInt()) - .put("cloud.aws.test.write_failures", 0.1); + .put("cloud.aws.test.write_failures", 0.1) + .put("cloud.aws.test.read_failures", 0.1); Environment environment = new Environment(); diff --git a/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java b/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java index a3c3e02d28c..cdf167be794 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java +++ b/src/test/java/org/elasticsearch/cloud/aws/TestAmazonS3.java @@ -22,10 +22,10 @@ package org.elasticsearch.cloud.aws; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectResult; +import com.amazonaws.services.s3.model.*; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import java.io.IOException; @@ -44,7 +44,10 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.randomDouble; */ public class TestAmazonS3 extends AmazonS3Wrapper { + protected final ESLogger logger = Loggers.getLogger(getClass()); + private double writeFailureRate = 0.0; + private double readFailureRate = 0.0; private String randomPrefix; @@ -65,6 +68,7 @@ public class TestAmazonS3 extends AmazonS3Wrapper { super(delegate); randomPrefix = componentSettings.get("test.random"); writeFailureRate = componentSettings.getAsDouble("test.write_failures", 0.0); + readFailureRate = componentSettings.getAsDouble("test.read_failures", 0.0); } @Override @@ -80,6 +84,7 @@ public class TestAmazonS3 extends AmazonS3Wrapper { throw new ElasticsearchException("cannot read input stream", ex); } } + logger.info("--> random write failure on putObject method: throwing an exception for [bucket={}, key={}]", bucketName, key); AmazonS3Exception ex = new AmazonS3Exception("Random S3 exception"); ex.setStatusCode(400); ex.setErrorCode("RequestTimeout"); @@ -89,6 +94,41 @@ public class TestAmazonS3 extends AmazonS3Wrapper { } } + @Override + public UploadPartResult uploadPart(UploadPartRequest request) throws AmazonClientException, AmazonServiceException { + if (shouldFail(request.getBucketName(), request.getKey(), writeFailureRate)) { + long length = request.getPartSize(); + long partToRead = (long) (length * randomDouble()); + byte[] buffer = new byte[1024]; + for (long cur = 0; cur < partToRead; cur += buffer.length) { + try (InputStream input = request.getInputStream()){ + input.read(buffer, 0, (int) (partToRead - cur > buffer.length ? buffer.length : partToRead - cur)); + } catch (IOException ex) { + throw new ElasticsearchException("cannot read input stream", ex); + } + } + logger.info("--> random write failure on uploadPart method: throwing an exception for [bucket={}, key={}]", request.getBucketName(), request.getKey()); + AmazonS3Exception ex = new AmazonS3Exception("Random S3 write exception"); + ex.setStatusCode(400); + ex.setErrorCode("RequestTimeout"); + throw ex; + } else { + return super.uploadPart(request); + } + } + + @Override + public S3Object getObject(String bucketName, String key) throws AmazonClientException, AmazonServiceException { + if (shouldFail(bucketName, key, readFailureRate)) { + logger.info("--> random read failure on getObject method: throwing an exception for [bucket={}, key={}]", bucketName, key); + AmazonS3Exception ex = new AmazonS3Exception("Random S3 read exception"); + ex.setStatusCode(404); + throw ex; + } else { + return super.getObject(bucketName, key); + } + } + private boolean shouldFail(String bucketName, String key, double probability) { if (probability > 0.0) { String path = randomPrefix + "-" + bucketName + "+" + key; diff --git a/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java index 6372657d95d..8c4707b806c 100644 --- a/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java +++ b/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java @@ -49,6 +49,11 @@ public class TestAwsS3Service extends InternalAwsS3Service { return cachedWrapper(super.client(region, account, key)); } + @Override + public synchronized AmazonS3 client(String region, String account, String key, Integer maxRetries) { + return cachedWrapper(super.client(region, account, key, maxRetries)); + } + private AmazonS3 cachedWrapper(AmazonS3 client) { TestAmazonS3 wrapper = clients.get(client); if (wrapper == null) { From c8839ee68c552c01ca28f462a27c0351e23f7bb5 Mon Sep 17 00:00:00 2001 From: Andreas Kohn Date: Tue, 11 Nov 2014 15:57:04 +0100 Subject: [PATCH 9/9] Use the new 'install' form when invoking bin/plugin '-install' was deprecated in https://github.com/elasticsearch/elasticsearch/issues/3112, and 'install' is the proper variant from https://github.com/elasticsearch/elasticsearch/pull/7339. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 463088c5514..4e39d812f45 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ for the unicast discovery mechanism and add S3 repositories. In order to install the plugin, run: ```sh -bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.4.0 +bin/plugin install elasticsearch/elasticsearch-cloud-aws/2.4.0 ``` You need to install a version matching your Elasticsearch version: