ECS Task IAM profile credentials ignored in repository-s3 plugin (#31864)

ECS Task IAM profile credentials ignored in repository-s3 plugin (#31864)

Closes #26913

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -142,6 +142,8 @@ class ClusterConfiguration {
     // there are cases when value depends on task that is not executed yet on configuration stage
     Map<String, Object> systemProperties = new HashMap<>()
 
+    Map<String, Object> environmentVariables = new HashMap<>()
+
     Map<String, Object> settings = new HashMap<>()
 
     Map<String, String> keystoreSettings = new HashMap<>()
@@ -164,6 +166,11 @@ class ClusterConfiguration {
         systemProperties.put(property, value)
     }
 
+    @Input
+    void environment(String variable, Object value) {
+        environmentVariables.put(variable, value)
+    }
+
     @Input
     void setting(String name, Object value) {
         settings.put(name, value)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -181,6 +181,7 @@ class NodeInfo {
 
         args.addAll("-E", "node.portsfile=true")
         env = [:]
+        env.putAll(config.environmentVariables)
         for (Map.Entry<String, String> property : System.properties.entrySet()) {
             if (property.key.startsWith('tests.es.')) {
                 args.add("-E")

docs/plugins/repository-s3.asciidoc

@@ -13,8 +13,8 @@ include::install_remove.asciidoc[]
 ==== Getting started with AWS
 
 The plugin provides a repository type named `s3` which may be used when creating a repository.
-The repository defaults to using
-http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html[IAM Role]
+The repository defaults to using https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html[ECS IAM Role] or
+http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html[EC2 IAM Role]
 credentials for authentication. The only mandatory setting is the bucket name:
 
 [source,js]

plugins/repository-s3/build.gradle

@@ -92,11 +92,15 @@ String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
 String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
 String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
 
+String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
+String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
+
 // If all these variables are missing then we are testing against the internal fixture instead, which has the following
 // credentials hard-coded in.
 
 if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath
-        && !s3EC2Bucket && !s3EC2BasePath) {
+        && !s3EC2Bucket && !s3EC2BasePath
+        && !s3ECSBucket && !s3ECSBasePath) {
   s3PermanentAccessKey = 's3_integration_test_permanent_access_key'
   s3PermanentSecretKey = 's3_integration_test_permanent_secret_key'
   s3PermanentBucket = 'permanent-bucket-test'
@@ -105,10 +109,14 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
   s3EC2Bucket = 'ec2-bucket-test'
   s3EC2BasePath = 'integration_test'
 
+  s3ECSBucket = 'ecs-bucket-test'
+  s3ECSBasePath = 'integration_test'
+
   useFixture = true
 
 } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath
-        || !s3EC2Bucket || !s3EC2BasePath) {
+        || !s3EC2Bucket || !s3EC2BasePath
+        || !s3ECSBucket || !s3ECSBasePath) {
   throw new IllegalArgumentException("not all options specified to run against external S3 service")
 }
 
@@ -284,7 +292,8 @@ if (useFixture && minioDistribution) {
   // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
   integTestMinioRunner.systemProperty 'tests.rest.blacklist', [
           'repository_s3/30_repository_temporary_credentials/*',
-          'repository_s3/40_repository_ec2_credentials/*'
+          'repository_s3/40_repository_ec2_credentials/*',
+          'repository_s3/50_repository_ecs_credentials/*'
         ].join(",")
 
   project.check.dependsOn(integTestMinio)
@@ -302,7 +311,8 @@ task s3FixtureProperties {
       "s3Fixture.temporary_bucket_name"  : s3TemporaryBucket,
       "s3Fixture.temporary_key"          : s3TemporaryAccessKey,
       "s3Fixture.temporary_session_token": s3TemporarySessionToken,
-      "s3Fixture.ec2_bucket_name"        : s3EC2Bucket
+      "s3Fixture.ec2_bucket_name"        : s3EC2Bucket,
+      "s3Fixture.ecs_bucket_name"        : s3ECSBucket
   ]
 
   doLast {
@@ -327,7 +337,9 @@ Map<String, Object> expansions = [
   'temporary_bucket': s3TemporaryBucket,
   'temporary_base_path': s3TemporaryBasePath,
   'ec2_bucket': s3EC2Bucket,
-  'ec2_base_path': s3EC2BasePath
+  'ec2_base_path': s3EC2BasePath,
+  'ecs_bucket': s3ECSBucket,
+  'ecs_base_path': s3ECSBasePath
 ]
 
 processTestResources {
@@ -364,6 +376,34 @@ integTestCluster {
   }
 }
 
+integTestRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
+
+///
+RestIntegTestTask integTestECS = project.tasks.create('integTestECS', RestIntegTestTask.class) {
+  description = "Runs tests using the ECS repository."
+}
+
+// The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
+project.afterEvaluate {
+  ClusterConfiguration cluster = project.extensions.getByName('integTestECSCluster') as ClusterConfiguration
+  cluster.dependsOn(project.s3Fixture)
+
+  cluster.setting 's3.client.integration_test_ecs.endpoint', "http://${-> s3Fixture.addressAndPort}"
+
+  Task integTestECSTask = project.tasks.getByName('integTestECS')
+  integTestECSTask.clusterConfig.plugin(project.path)
+  integTestECSTask.clusterConfig.environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI',
+          "http://${-> s3Fixture.addressAndPort}/ecs_credentials_endpoint"
+  integTestECSRunner.systemProperty 'tests.rest.blacklist', [
+          'repository_s3/10_basic/*',
+          'repository_s3/20_repository_permanent_credentials/*',
+          'repository_s3/30_repository_temporary_credentials/*',
+          'repository_s3/40_repository_ec2_credentials/*'
+  ].join(",")
+}
+project.check.dependsOn(integTestECS)
+///
+
 thirdPartyAudit.excludes = [
   // classes are missing
   'javax.servlet.ServletContextEvent', 

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -22,7 +22,7 @@ package org.elasticsearch.repositories.s3;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
 import com.amazonaws.http.IdleConnectionReaper;
 import com.amazonaws.internal.StaticCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
@@ -156,10 +156,11 @@ class S3Service extends AbstractComponent implements Closeable {
     }
 
     static class PrivilegedInstanceProfileCredentialsProvider implements AWSCredentialsProvider {
-        private final InstanceProfileCredentialsProvider credentials;
+        private final AWSCredentialsProvider credentials;
 
         private PrivilegedInstanceProfileCredentialsProvider() {
-            this.credentials = new InstanceProfileCredentialsProvider();
+            // InstanceProfileCredentialsProvider as last item of chain
+            this.credentials = new EC2ContainerCredentialsProviderWrapper();
         }
 
         @Override

plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java

@@ -88,7 +88,10 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
         final Bucket ec2Bucket = new Bucket("s3Fixture.ec2",
             randomAsciiAlphanumOfLength(random, 10), randomAsciiAlphanumOfLength(random, 10));
 
-        this.handlers = defaultHandlers(buckets, ec2Bucket);
+        final Bucket ecsBucket = new Bucket("s3Fixture.ecs",
+            randomAsciiAlphanumOfLength(random, 10), randomAsciiAlphanumOfLength(random, 10));
+
+        this.handlers = defaultHandlers(buckets, ec2Bucket, ecsBucket);
     }
 
     private static String nonAuthPath(Request request) {
@@ -174,7 +177,7 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
     }
 
     /** Builds the default request handlers **/
-    private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> buckets, final Bucket ec2Bucket) {
+    private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> buckets, final Bucket ec2Bucket, final Bucket ecsBucket) {
         final PathTrie<RequestHandler> handlers = new PathTrie<>(RestUtils.REST_DECODER);
 
         // HEAD Object
@@ -400,11 +403,18 @@ public class AmazonS3Fixture extends AbstractHttpFixture {
         handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/latest/meta-data/iam/security-credentials/{profileName}"), (request) -> {
             final String profileName = request.getParam("profileName");
             if (EC2_PROFILE.equals(profileName) == false) {
-                return new Response(RestStatus.NOT_FOUND.getStatus(), new HashMap<>(), "unknown credentials".getBytes(UTF_8));
+                return new Response(RestStatus.NOT_FOUND.getStatus(), new HashMap<>(), "unknown profile".getBytes(UTF_8));
             }
             return credentialResponseFunction.apply(profileName, ec2Bucket.key, ec2Bucket.token);
         });
 
+        // GET
+        //
+        // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html
+        handlers.insert(nonAuthPath(HttpGet.METHOD_NAME, "/ecs_credentials_endpoint"),
+            (request) -> credentialResponseFunction.apply("CPV_ECS", ecsBucket.key, ecsBucket.token));
+
+
         return handlers;
     }
 

plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml

@@ -0,0 +1,243 @@
+# Integration tests for repository-s3
+
+---
+setup:
+
+  # Register repository with ecs credentials
+  - do:
+      snapshot.create_repository:
+        repository: repository_ecs
+        body:
+          type: s3
+          settings:
+            bucket: ${ecs_bucket}
+            client: integration_test_ecs
+            base_path: ${ecs_base_path}
+            canned_acl: private
+            storage_class: standard
+
+---
+"Snapshot and Restore with repository-s3 using ecs credentials":
+
+  # Get repository
+  - do:
+      snapshot.get_repository:
+        repository: repository_ecs
+
+  - match: { repository_ecs.settings.bucket : ${ecs_bucket} }
+  - match: { repository_ecs.settings.client : "integration_test_ecs" }
+  - match: { repository_ecs.settings.base_path : ${ecs_base_path} }
+  - match: { repository_ecs.settings.canned_acl : "private" }
+  - match: { repository_ecs.settings.storage_class : "standard" }
+  - is_false: repository_ecs.settings.access_key
+  - is_false: repository_ecs.settings.secret_key
+  - is_false: repository_ecs.settings.session_token
+
+  # Index documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    1
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    2
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    3
+          - snapshot: one
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  # Create a first snapshot
+  - do:
+      snapshot.create:
+        repository: repository_ecs
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot-one }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.include_global_state: true }
+  - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.status:
+        repository: repository_ecs
+        snapshot: snapshot-one
+
+  - is_true: snapshots
+  - match: { snapshots.0.snapshot: snapshot-one }
+  - match: { snapshots.0.state : SUCCESS }
+
+  # Index more documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    4
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    5
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    6
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    7
+          - snapshot: two
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Create a second snapshot
+  - do:
+      snapshot.create:
+        repository: repository_ecs
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot-two }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.get:
+        repository: repository_ecs
+        snapshot: snapshot-one,snapshot-two
+
+  - is_true: snapshots
+  - match: { snapshots.0.state : SUCCESS }
+  - match: { snapshots.1.state : SUCCESS }
+
+  # Delete the index
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the second snapshot
+  - do:
+      snapshot.restore:
+        repository: repository_ecs
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Delete the index again
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the first snapshot
+  - do:
+      snapshot.restore:
+        repository: repository_ecs
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  # Remove the snapshots
+  - do:
+      snapshot.delete:
+        repository: repository_ecs
+        snapshot: snapshot-two
+
+  - do:
+      snapshot.delete:
+        repository: repository_ecs
+        snapshot: snapshot-one
+
+---
+"Register a repository with a non existing bucket":
+
+  - do:
+      catch: /repository_exception/
+      snapshot.create_repository:
+        repository: repository_ecs
+        body:
+          type: s3
+          settings:
+            bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE
+            client: integration_test_temporary
+
+---
+"Register a repository with a non existing client":
+
+  - do:
+      catch: /repository_exception/
+      snapshot.create_repository:
+        repository: repository_ecs
+        body:
+          type: s3
+          settings:
+            bucket: repository_ecs
+            client: unknown
+
+---
+"Get a non existing snapshot":
+
+  - do:
+      catch: /snapshot_missing_exception/
+      snapshot.get:
+        repository: repository_ecs
+        snapshot: missing
+
+---
+"Delete a non existing snapshot":
+
+  - do:
+      catch: /snapshot_missing_exception/
+      snapshot.delete:
+        repository: repository_ecs
+        snapshot: missing
+
+---
+"Restore a non existing snapshot":
+
+  - do:
+      catch: /snapshot_restore_exception/
+      snapshot.restore:
+        repository: repository_ecs
+        snapshot: missing
+        wait_for_completion: true
+
+---
+teardown:
+
+  # Remove our repository
+  - do:
+     snapshot.delete_repository:
+       repository: repository_ecs