Support use of IRSA for repository-s3 plugin credentials: added YAML Rest test case (#3499)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
This commit is contained in:
Andriy Redko 2022-06-06 13:36:34 -04:00 committed by GitHub
parent 2fbf33555e
commit 6c769d4fca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 496 additions and 13 deletions

View File

@ -131,6 +131,9 @@ String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
String s3EKSBucket = System.getenv("amazon_s3_bucket_eks")
String s3EKSBasePath = System.getenv("amazon_s3_base_path_eks")
boolean s3DisableChunkedEncoding = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()
// If all these variables are missing then we are testing against the internal fixture instead, which has the following
@ -160,13 +163,15 @@ if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3T
throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
}
if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath && !s3EKSBucket && !s3EKSBasePath) {
s3EC2Bucket = 'ec2_bucket'
s3EC2BasePath = 'ec2_base_path'
s3ECSBucket = 'ecs_bucket'
s3ECSBasePath = 'ecs_base_path'
} else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath) {
throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
s3EKSBucket = 'eks_bucket'
s3EKSBasePath = 'eks_base_path'
} else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath || !s3EKSBucket || !s3EKSBasePath) {
throw new IllegalArgumentException("not all options specified to run EC2/ECS/EKS tests are present")
}
processYamlRestTestResources {
@ -179,7 +184,9 @@ processYamlRestTestResources {
'ec2_base_path': s3EC2BasePath,
'ecs_bucket': s3ECSBucket,
'ecs_base_path': s3ECSBasePath,
'disable_chunked_encoding': s3DisableChunkedEncoding,
'eks_bucket': s3EKSBucket,
'eks_base_path': s3EKSBasePath,
'disable_chunked_encoding': s3DisableChunkedEncoding
]
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
@ -198,7 +205,8 @@ yamlRestTest {
[
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
'repository_s3/50_repository_ecs_credentials/*',
'repository_s3/60_repository_eks_credentials/*'
]
).join(",")
}
@ -215,6 +223,7 @@ testClusters.yamlRestTest {
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture')
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-session-token')
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ec2')
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-eks')
normalization {
runtimeClasspath {
@ -223,12 +232,21 @@ testClusters.yamlRestTest {
}
}
keystore 's3.client.integration_test_eks.role_arn', "arn:aws:iam::000000000000:role/test"
keystore 's3.client.integration_test_eks.role_session_name', "s3-test"
keystore 's3.client.integration_test_eks.access_key', "access_key"
keystore 's3.client.integration_test_eks.secret_key', "secret_key"
setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture', '80')}" }, IGNORE_VALUE
setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-session-token', '80')}" }, IGNORE_VALUE
setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE
setting 's3.client.integration_test_eks.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-eks', '80')}" }, IGNORE_VALUE
setting 's3.client.integration_test_eks.region', { "us-east-2" }, IGNORE_VALUE
// to redirect InstanceProfileCredentialsProvider to custom auth point
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE
// to redirect AWSSecurityTokenServiceClient to custom auth point
systemProperty "com.amazonaws.sdk.stsEndpointOverride", { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-eks', '80')}/eks_credentials_endpoint" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-s3 plugin"
}
@ -250,7 +268,8 @@ if (useFixture) {
systemProperty 'tests.rest.denylist', [
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
'repository_s3/50_repository_ecs_credentials/*',
'repository_s3/60_repository_eks_credentials/*'
].join(",")
}
check.dependsOn(yamlRestTestMinio)
@ -277,7 +296,8 @@ if (useFixture) {
'repository_s3/10_basic/*',
'repository_s3/20_repository_permanent_credentials/*',
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*'
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/60_repository_eks_credentials/*'
].join(",")
}
check.dependsOn(yamlRestTestECS)
@ -289,6 +309,41 @@ if (useFixture) {
}
}
// EKS
if (useFixture) {
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-eks')
task yamlRestTestEKS(type: RestIntegTestTask.class) {
description = "Runs tests using the EKS repository."
dependsOn('bundlePlugin')
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME)
setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
systemProperty 'tests.rest.denylist', [
'repository_s3/10_basic/*',
'repository_s3/20_repository_permanent_credentials/*',
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
].join(",")
}
check.dependsOn(yamlRestTestEKS)
testClusters.yamlRestTestEKS {
keystore 's3.client.integration_test_eks.role_arn', "arn:aws:iam::000000000000:role/test"
keystore 's3.client.integration_test_eks.role_session_name', "s3-test"
keystore 's3.client.integration_test_eks.access_key', "access_key"
keystore 's3.client.integration_test_eks.secret_key', "secret_key"
setting 's3.client.integration_test_eks.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-eks', '80')}" }, IGNORE_VALUE
setting 's3.client.integration_test_eks.region', { "us-east-2" }, IGNORE_VALUE
plugin tasks.bundlePlugin.archiveFile
// to redirect AWSSecurityTokenServiceClient to custom auth point
systemProperty "com.amazonaws.sdk.stsEndpointOverride", { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-eks', '80')}/eks_credentials_endpoint" }, IGNORE_VALUE
}
}
// 3rd Party Tests
TaskProvider s3ThirdPartyTest = tasks.register("s3ThirdPartyTest", Test) {
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);

View File

@ -41,6 +41,7 @@ import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
import com.amazonaws.auth.STSAssumeRoleWithWebIdentitySessionCredentialsProvider;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
import com.amazonaws.http.IdleConnectionReaper;
import com.amazonaws.http.SystemPropertyTlsKeyManagersProvider;
import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory;
@ -82,6 +83,8 @@ import static java.util.Collections.emptyMap;
class S3Service implements Closeable {
private static final Logger logger = LogManager.getLogger(S3Service.class);
private static final String STS_ENDPOINT_OVERRIDE_SYSTEM_PROPERTY = "com.amazonaws.sdk.stsEndpointOverride";
private volatile Map<S3ClientSettings, AmazonS3Reference> clientsCache = emptyMap();
/**
@ -280,13 +283,25 @@ class S3Service implements Closeable {
AWSSecurityTokenService securityTokenService = null;
final String region = Strings.hasLength(clientSettings.region) ? clientSettings.region : null;
if (region != null || basicCredentials != null) {
securityTokenService = SocketAccess.doPrivileged(
() -> AWSSecurityTokenServiceClientBuilder.standard()
.withCredentials((basicCredentials != null) ? new AWSStaticCredentialsProvider(basicCredentials) : null)
.withRegion(region)
.build()
);
securityTokenService = SocketAccess.doPrivileged(() -> {
AWSSecurityTokenServiceClientBuilder builder = AWSSecurityTokenServiceClientBuilder.standard();
// Use similar approach to override STS endpoint as SDKGlobalConfiguration.EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY
final String stsEndpoint = System.getProperty(STS_ENDPOINT_OVERRIDE_SYSTEM_PROPERTY);
if (region != null && stsEndpoint != null) {
builder = builder.withEndpointConfiguration(new EndpointConfiguration(stsEndpoint, region));
} else {
builder = builder.withRegion(region);
}
if (basicCredentials != null) {
builder = builder.withCredentials(new AWSStaticCredentialsProvider(basicCredentials));
}
return builder.build();
});
}
if (irsaCredentials.getIdentityTokenFile() == null) {

View File

@ -0,0 +1,268 @@
# Integration tests for repository-s3
---
setup:
# Register repository with eks credentials
- do:
snapshot.create_repository:
repository: repository_eks
body:
type: s3
settings:
bucket: ${eks_bucket}
client: integration_test_eks
base_path: "${eks_base_path}"
canned_acl: private
storage_class: standard
disable_chunked_encoding: ${disable_chunked_encoding}
---
"Snapshot and Restore with repository-s3 using eks credentials":
# Get repository
- do:
snapshot.get_repository:
repository: repository_eks
- match: { repository_eks.settings.bucket : ${eks_bucket} }
- match: { repository_eks.settings.client : "integration_test_eks" }
- match: { repository_eks.settings.base_path : "${eks_base_path}" }
- match: { repository_eks.settings.canned_acl : "private" }
- match: { repository_eks.settings.storage_class : "standard" }
- is_false: repository_eks.settings.access_key
- is_false: repository_eks.settings.secret_key
- is_false: repository_eks.settings.session_token
- is_false: repository_eks.settings.role_arn
- is_false: repository_eks.settings.role_session_name
- is_false: repository_eks.settings.identity_token_file
# Index documents
- do:
bulk:
refresh: true
body:
- index:
_index: docs
_id: 1
- snapshot: one
- index:
_index: docs
_id: 2
- snapshot: one
- index:
_index: docs
_id: 3
- snapshot: one
- do:
count:
index: docs
- match: {count: 3}
# Create a first snapshot
- do:
snapshot.create:
repository: repository_eks
snapshot: snapshot-one
wait_for_completion: true
- match: { snapshot.snapshot: snapshot-one }
- match: { snapshot.state : SUCCESS }
- match: { snapshot.include_global_state: true }
- match: { snapshot.shards.failed : 0 }
- do:
snapshot.status:
repository: repository_eks
snapshot: snapshot-one
- is_true: snapshots
- match: { snapshots.0.snapshot: snapshot-one }
- match: { snapshots.0.state : SUCCESS }
# Index more documents
- do:
bulk:
refresh: true
body:
- index:
_index: docs
_id: 4
- snapshot: two
- index:
_index: docs
_id: 5
- snapshot: two
- index:
_index: docs
_id: 6
- snapshot: two
- index:
_index: docs
_id: 7
- snapshot: two
- do:
count:
index: docs
- match: {count: 7}
# Create a second snapshot
- do:
snapshot.create:
repository: repository_eks
snapshot: snapshot-two
wait_for_completion: true
- match: { snapshot.snapshot: snapshot-two }
- match: { snapshot.state : SUCCESS }
- match: { snapshot.shards.failed : 0 }
- do:
snapshot.get:
repository: repository_eks
snapshot: snapshot-one,snapshot-two
- is_true: snapshots
- match: { snapshots.0.state : SUCCESS }
- match: { snapshots.1.state : SUCCESS }
# Delete the index
- do:
indices.delete:
index: docs
# Restore the second snapshot
- do:
snapshot.restore:
repository: repository_eks
snapshot: snapshot-two
wait_for_completion: true
- do:
count:
index: docs
- match: {count: 7}
# Delete the index again
- do:
indices.delete:
index: docs
# Restore the first snapshot
- do:
snapshot.restore:
repository: repository_eks
snapshot: snapshot-one
wait_for_completion: true
- do:
count:
index: docs
- match: {count: 3}
# Remove the snapshots
- do:
snapshot.delete:
repository: repository_eks
snapshot: snapshot-two
- do:
snapshot.delete:
repository: repository_eks
snapshot: snapshot-one
---
"Register a repository with a non existing bucket":
- do:
catch: /repository_verification_exception/
snapshot.create_repository:
repository: repository_eks
body:
type: s3
settings:
bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE
client: integration_test_eks
---
"Register a repository with a non existing client":
- do:
catch: /illegal_argument_exception/
snapshot.create_repository:
repository: repository_eks
body:
type: s3
settings:
bucket: repository_eks
client: unknown
---
"Register a read-only repository with a non existing bucket":
- do:
catch: /repository_verification_exception/
snapshot.create_repository:
repository: repository_eks
body:
type: s3
settings:
readonly: true
bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE
client: integration_test_eks
---
"Register a read-only repository with a non existing client":
- do:
catch: /illegal_argument_exception/
snapshot.create_repository:
repository: repository_eks
body:
type: s3
settings:
readonly: true
bucket: repository_eks
client: unknown
---
"Get a non existing snapshot":
- do:
catch: /snapshot_missing_exception/
snapshot.get:
repository: repository_eks
snapshot: missing
---
"Delete a non existing snapshot":
- do:
catch: /snapshot_missing_exception/
snapshot.delete:
repository: repository_eks
snapshot: missing
---
"Restore a non existing snapshot":
- do:
catch: /snapshot_restore_exception/
snapshot.restore:
repository: repository_eks
snapshot: missing
wait_for_completion: true
---
teardown:
# Remove our repository
- do:
snapshot.delete_repository:
repository: repository_eks

25
test/fixtures/s3-fixture/Dockerfile.eks vendored Normal file
View File

@ -0,0 +1,25 @@
FROM ubuntu:18.04
RUN apt-get update -qqy
RUN apt-get install -qqy openjdk-11-jre-headless
ARG fixtureClass
ARG port
ARG bucket
ARG basePath
ARG accessKey
ARG roleArn
ARG roleSessionName
ENV S3_FIXTURE_CLASS=${fixtureClass}
ENV S3_FIXTURE_PORT=${port}
ENV S3_FIXTURE_BUCKET=${bucket}
ENV S3_FIXTURE_BASE_PATH=${basePath}
ENV S3_FIXTURE_ACCESS_KEY=${accessKey}
ENV S3_FIXTURE_ROLE_ARN=${roleArn}
ENV S3_FIXTURE_ROLE_SESSION_NAME=${roleSessionName}
ENTRYPOINT exec java -classpath "/fixture/shared/*" \
$S3_FIXTURE_CLASS 0.0.0.0 "$S3_FIXTURE_PORT" "$S3_FIXTURE_BUCKET" "$S3_FIXTURE_BASE_PATH" "$S3_FIXTURE_ACCESS_KEY" "$S3_FIXTURE_ROLE_ARN" "$S3_FIXTURE_ROLE_SESSION_NAME"
EXPOSE $port

View File

@ -92,3 +92,20 @@ services:
- ./testfixtures_shared/shared:/fixture/shared
ports:
- "80"
s3-fixture-with-eks:
build:
context: .
args:
fixtureClass: fixture.s3.S3HttpFixtureWithEKS
port: 80
bucket: "eks_bucket"
basePath: "eks_base_path"
accessKey: "eks_access_key"
roleArn: "eks_role_arn"
roleSessionName: "eks_role_session_name"
dockerfile: Dockerfile.eks
volumes:
- ./testfixtures_shared/shared:/fixture/shared
ports:
- "80"

View File

@ -0,0 +1,103 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
/*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/
package fixture.s3;
import com.sun.net.httpserver.HttpHandler;
import org.opensearch.rest.RestStatus;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Objects;
public class S3HttpFixtureWithEKS extends S3HttpFixture {
private S3HttpFixtureWithEKS(final String[] args) throws Exception {
super(args);
}
@Override
protected HttpHandler createHandler(final String[] args) {
final String accessKey = Objects.requireNonNull(args[4]);
final String eksRoleArn = Objects.requireNonNull(args[5]);
final HttpHandler delegate = super.createHandler(args);
return exchange -> {
// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html
if ("POST".equals(exchange.getRequestMethod()) && exchange.getRequestURI().getPath().startsWith("/eks_credentials_endpoint")) {
final byte[] response = buildCredentialResponse(eksRoleArn, accessKey).getBytes(StandardCharsets.UTF_8);
exchange.getResponseHeaders().add("Content-Type", "application/xml");
exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
exchange.getResponseBody().write(response);
exchange.close();
return;
}
delegate.handle(exchange);
};
}
protected String buildCredentialResponse(final String roleArn, final String accessKey) {
// See please: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
return "<AssumeRoleWithWebIdentityResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\">\n"
+ " <AssumeRoleWithWebIdentityResult>\n"
+ " <SubjectFromWebIdentityToken>amzn1.account.AF6RHO7KZU5XRVQJGXK6HB56KR2A</SubjectFromWebIdentityToken>\n"
+ " <Audience>client.5498841531868486423.1548@apps.example.com</Audience>\n"
+ " <AssumedRoleUser>\n"
+ " <Arn>" + roleArn + "</Arn>\n"
+ " <AssumedRoleId>AROACLKWSDQRAOEXAMPLE:s3</AssumedRoleId>\n"
+ " </AssumedRoleUser>\n"
+ " <Credentials>\n"
+ " <SessionToken>AQoDYXdzEE0a8ANXXXXXXXXNO1ewxE5TijQyp+IEXAMPLE</SessionToken>\n"
+ " <SecretAccessKey>wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY</SecretAccessKey>\n"
+ " <Expiration>" + LocalDateTime.now().plusMonths(1).atZone(ZoneId.of("UTC")).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + "</Expiration>\n"
+ " <AccessKeyId>" + accessKey + "</AccessKeyId>\n"
+ " </Credentials>\n"
+ " <SourceIdentity>SourceIdentityValue</SourceIdentity>\n"
+ " <Provider>www.amazon.com</Provider>\n"
+ " </AssumeRoleWithWebIdentityResult>\n"
+ " <ResponseMetadata>\n"
+ " <RequestId>ad4156e9-bce1-11e2-82e6-6b6efEXAMPLE</RequestId>\n"
+ " </ResponseMetadata>\n"
+ "</AssumeRoleWithWebIdentityResponse>";
}
public static void main(final String[] args) throws Exception {
if (args == null || args.length < 6) {
throw new IllegalArgumentException("S3HttpFixtureWithEKS expects 6 arguments " +
"[address, port, bucket, base path, role arn, role session name]");
}
final S3HttpFixtureWithEKS fixture = new S3HttpFixtureWithEKS(args);
fixture.start();
}
}