From f6697aa82b69af54dca6ac6b0eddacd0a74ede49 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 12 Nov 2019 15:28:15 +0000 Subject: [PATCH] HADOOP-16477. S3A delegation token tests fail if fs.s3a.encryption.key set. Contributed by Steve Loughran. Change-Id: I843989f32472bbdefbd4fa504b26c7a614ab1cee --- .../hadoop/fs/s3a/WriteOperationHelper.java | 4 +- .../hadoop/fs/s3a/auth/RolePolicies.java | 6 +- .../tools/hadoop-aws/assumed_roles.md | 61 +++++++++++++- .../fs/s3a/AbstractTestS3AEncryption.java | 80 ++++++++++++++++--- .../s3a/ITestS3AAWSCredentialsProvider.java | 18 ++++- .../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 46 +++++------ .../ITestS3AEncryptionSSEKMSDefaultKey.java | 7 +- ...TestS3AEncryptionSSEKMSUserDefinedKey.java | 20 +++-- .../hadoop/fs/s3a/ITestS3AMiscOperations.java | 15 +++- .../apache/hadoop/fs/s3a/S3ATestUtils.java | 5 ++ .../hadoop/fs/s3a/auth/RoleTestUtils.java | 8 +- .../auth/delegation/AbstractDelegationIT.java | 2 + .../ITestSessionDelegationInFileystem.java | 13 ++- .../s3a/impl/ITestPartialRenamesDeletes.java | 8 +- 14 files changed, 224 insertions(+), 69 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java index 570f0373ff7..602732b6d32 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java @@ -457,7 +457,7 @@ public class WriteOperationHelper { @Retries.RetryTranslated public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws IOException { - return retry("put", + return retry("Writing Object", putObjectRequest.getKey(), true, () -> owner.putObjectDirect(putObjectRequest)); } @@ -472,7 +472,7 @@ public class WriteOperationHelper { public UploadResult uploadObject(PutObjectRequest putObjectRequest) throws IOException { // no retry; rely on xfer manager logic - return retry("put", + return retry("Writing Object", putObjectRequest.getKey(), true, () -> owner.executePut(putObjectRequest, null)); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java index 610dbcc6765..d2e7eb1a657 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/RolePolicies.java @@ -55,7 +55,7 @@ public final class RolePolicies { /** * Arn for all KMS keys: {@value}. */ - public static final String KMS_ALL_KEYS = "arn:aws:kms:*"; + public static final String KMS_ALL_KEYS = "*"; /** * This is used by S3 to generate a per-object encryption key and @@ -68,7 +68,7 @@ public final class RolePolicies { * Actions needed to read and write SSE-KMS data. */ private static final String[] KMS_KEY_RW = - new String[]{KMS_DECRYPT, KMS_GENERATE_DATA_KEY}; + new String[]{KMS_DECRYPT, KMS_GENERATE_DATA_KEY, KMS_ENCRYPT}; /** * Actions needed to read SSE-KMS data. @@ -81,7 +81,7 @@ public final class RolePolicies { * SSE-KMS. */ public static final Statement STATEMENT_ALLOW_SSE_KMS_RW = - statement(true, KMS_ALL_KEYS, KMS_KEY_RW); + statement(true, KMS_ALL_KEYS, KMS_ALL_OPERATIONS); /** * Statement to allow read access to KMS keys, so the ability diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md index f08f40f27b0..96bd64811aa 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md @@ -46,9 +46,25 @@ have access to the appropriate KMS keys. Trying to learn how IAM Assumed Roles work by debugging stack traces from the S3A client is "suboptimal". -### How the S3A connector support IAM Assumed Roles. +### How the S3A connector supports IAM Assumed Roles. -To use assumed roles, the client must be configured to use the + +The S3A connector support IAM Assumed Roles in two ways: + +1. Using the full credentials on the client to request credentials for a specific + role -credentials which are then used for all the store operations. + This can be used to verify that a specific role has the access permissions + you need, or to "su" into a role which has permissions that's the full + accounts does not directly qualify for -such as access to a KMS key. +2. Using the full credentials to request role credentials which are then + propagated into a launched application as delegation tokens. + This extends the previous use as it allows the jobs to be submitted to a + shared cluster with the permissions of the requested role, rather than + those of the VMs/Containers of the deployed cluster. + +For Delegation Token integration, see (Delegation Tokens)[delegation_tokens.html] + +To for Assumed Role authentication, the client must be configured to use the *Assumed Role Credential Provider*, `org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider`, in the configuration option `fs.s3a.aws.credentials.provider`. @@ -298,7 +314,7 @@ Without these permissions, tables cannot be created, destroyed or have their IO changed through the `s3guard set-capacity` call. The `dynamodb:Scan` permission is needed for `s3guard prune` -The `dynamodb:CreateTable` permission is needed by a client it tries to +The `dynamodb:CreateTable` permission is needed by a client when it tries to create the DynamoDB table on startup, that is `fs.s3a.s3guard.ddb.table.create` is `true` and the table does not already exist. @@ -758,14 +774,51 @@ Make sure that all the read and write permissions are allowed for any bucket/pat to which data is being written to, and read permissions for all buckets read from. +### `AccessDeniedException` When working with KMS-encrypted data + If the bucket is using SSE-KMS to encrypt data: 1. The caller must have the `kms:Decrypt` permission to read the data. -1. The caller needs `kms:Decrypt` and `kms:GenerateDataKey`. +1. The caller needs `kms:Decrypt` and `kms:GenerateDataKey` to write data. Without permissions, the request fails *and there is no explicit message indicating that this is an encryption-key issue*. +This problem is most obvious when you fail when writing data in a "Writing Object" operation. + +If the client does have write access to the bucket, verify that the caller has +`kms:GenerateDataKey` permissions for the encryption key in use. + +``` +java.nio.file.AccessDeniedException: test/testDTFileSystemClient: Writing Object on test/testDTFileSystemClient: + com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied (Service: Amazon S3; Status Code: 403; + Error Code: AccessDenied; Request ID: E86544FF1D029857) + + at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:243) + at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:111) + at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$4(Invoker.java:314) + at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:406) + at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:310) + at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:285) + at org.apache.hadoop.fs.s3a.WriteOperationHelper.retry(WriteOperationHelper.java:150) + at org.apache.hadoop.fs.s3a.WriteOperationHelper.putObject(WriteOperationHelper.java:460) + at org.apache.hadoop.fs.s3a.S3ABlockOutputStream.lambda$putObject$0(S3ABlockOutputStream.java:438) + at org.apache.hadoop.util.SemaphoredDelegatingExecutor$CallableWithPermitRelease.call(SemaphoredDelegatingExecutor.java:219) + at org.apache.hadoop.util.SemaphoredDelegatingExecutor$CallableWithPermitRelease.call(SemaphoredDelegatingExecutor.java:219) + at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:125) + at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:57) + at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:78) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) + at java.lang.Thread.run(Thread.java:748) +Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied (Service: Amazon S3; Status Code: 403; + Error Code: AccessDenied; Request ID: E86544FF1D029857) +``` + +Note: the ability to read encrypted data in the store does not guarantee that the caller can encrypt new data. +It is a separate permission. + + ### `AccessDeniedException` + `AmazonDynamoDBException` ``` diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java index 108f102c67f..ea7aa3de3fd 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java @@ -18,18 +18,25 @@ package org.apache.hadoop.fs.s3a; +import java.io.IOException; + import com.amazonaws.services.s3.model.ObjectMetadata; +import org.junit.Test; + import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.net.util.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.junit.Test; - -import java.io.IOException; +import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; import static org.apache.hadoop.fs.contract.ContractTestUtils.*; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; +import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionTestsDisabled; +import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; /** * Test whether or not encryption works by turning it on. Some checks @@ -38,11 +45,18 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; */ public abstract class AbstractTestS3AEncryption extends AbstractS3ATestBase { + protected static final String AWS_KMS_SSE_ALGORITHM = "aws:kms"; + + protected static final String SSE_C_ALGORITHM = "AES256"; + @Override protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); S3ATestUtils.disableFilesystemCaching(conf); - conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM, + removeBaseAndBucketOverrides(conf, + SERVER_SIDE_ENCRYPTION_ALGORITHM, + SERVER_SIDE_ENCRYPTION_KEY); + conf.set(SERVER_SIDE_ENCRYPTION_ALGORITHM, getSSEAlgorithm().getMethod()); return conf; } @@ -51,8 +65,34 @@ public abstract class AbstractTestS3AEncryption extends AbstractS3ATestBase { 0, 1, 2, 3, 4, 5, 254, 255, 256, 257, 2 ^ 12 - 1 }; + protected void requireEncryptedFileSystem() { + skipIfEncryptionTestsDisabled(getFileSystem().getConf()); + } + + @Override + public void setup() throws Exception { + super.setup(); + requireEncryptedFileSystem(); + } + + /** + * This examines how encryption settings propagate better. + * If the settings are actually in a JCEKS file, then the + * test override will fail; this is here to help debug the problem. + */ + @Test + public void testEncryptionSettingPropagation() throws Throwable { + S3AFileSystem fs = getFileSystem(); + S3AEncryptionMethods algorithm = getEncryptionAlgorithm( + fs.getBucket(), fs.getConf()); + assertEquals("Configuration has wrong encryption algorithm", + getSSEAlgorithm(), algorithm); + } + @Test public void testEncryption() throws Throwable { + requireEncryptedFileSystem(); + validateEncrytionSecrets(getFileSystem().getEncryptionSecrets()); for (int size: SIZES) { validateEncryptionForFilesize(size); } @@ -60,10 +100,11 @@ public abstract class AbstractTestS3AEncryption extends AbstractS3ATestBase { @Test public void testEncryptionOverRename() throws Throwable { - skipIfEncryptionTestsDisabled(getConfiguration()); Path src = path(createFilename(1024)); byte[] data = dataset(1024, 'a', 'z'); S3AFileSystem fs = getFileSystem(); + EncryptionSecrets secrets = fs.getEncryptionSecrets(); + validateEncrytionSecrets(secrets); writeDataset(fs, src, data, data.length, 1024 * 1024, true); ContractTestUtils.verifyFileContents(fs, src, data); Path dest = path(src.getName() + "-copy"); @@ -72,8 +113,19 @@ public abstract class AbstractTestS3AEncryption extends AbstractS3ATestBase { assertEncrypted(dest); } + /** + * Verify that the filesystem encryption secrets match expected. + * This makes sure that the settings have propagated properly. + * @param secrets encryption secrets of the filesystem. + */ + protected void validateEncrytionSecrets(final EncryptionSecrets secrets) { + assertNotNull("No encryption secrets for filesystem", secrets); + S3AEncryptionMethods sseAlgorithm = getSSEAlgorithm(); + assertEquals("Filesystem has wrong encryption algorithm", + sseAlgorithm, secrets.getEncryptionMethod()); + } + protected void validateEncryptionForFilesize(int len) throws IOException { - skipIfEncryptionTestsDisabled(getConfiguration()); describe("Create an encrypted file of size " + len); String src = createFilename(len); Path path = writeThenReadFile(src, len); @@ -98,15 +150,17 @@ public abstract class AbstractTestS3AEncryption extends AbstractS3ATestBase { ObjectMetadata md = getFileSystem().getObjectMetadata(path); switch(getSSEAlgorithm()) { case SSE_C: - assertEquals("AES256", md.getSSECustomerAlgorithm()); + assertNull("Metadata algorithm should have been null", + md.getSSEAlgorithm()); + assertEquals("Wrong SSE-C algorithm", SSE_C_ALGORITHM, md.getSSECustomerAlgorithm()); String md5Key = convertKeyToMd5(); - assertEquals(md5Key, md.getSSECustomerKeyMd5()); + assertEquals("getSSECustomerKeyMd5() wrong", md5Key, md.getSSECustomerKeyMd5()); break; case SSE_KMS: - assertEquals("aws:kms", md.getSSEAlgorithm()); + assertEquals(AWS_KMS_SSE_ALGORITHM, md.getSSEAlgorithm()); //S3 will return full arn of the key, so specify global arn in properties assertEquals(this.getConfiguration(). - getTrimmed(Constants.SERVER_SIDE_ENCRYPTION_KEY), + getTrimmed(SERVER_SIDE_ENCRYPTION_KEY), md.getSSEAwsKmsKeyId()); break; default: @@ -123,8 +177,8 @@ public abstract class AbstractTestS3AEncryption extends AbstractS3ATestBase { * key */ private String convertKeyToMd5() { - String base64Key = getConfiguration().getTrimmed( - Constants.SERVER_SIDE_ENCRYPTION_KEY + String base64Key = getFileSystem().getConf().getTrimmed( + SERVER_SIDE_ENCRYPTION_KEY ); byte[] key = Base64.decodeBase64(base64Key); byte[] md5 = DigestUtils.md5(key); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java index 9e8a871ef7a..30f1f71eeea 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs.s3a; import java.io.IOException; import java.nio.file.AccessDeniedException; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -38,9 +39,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.hadoop.fs.s3a.Constants.*; -import static org.apache.hadoop.fs.s3a.S3ATestConstants.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getCSVTestPath; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.S3AUtils.*; +import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING; import static org.junit.Assert.*; /** @@ -51,11 +53,11 @@ public class ITestS3AAWSCredentialsProvider { LoggerFactory.getLogger(ITestS3AAWSCredentialsProvider.class); @Rule - public Timeout testTimeout = new Timeout(1 * 60 * 1000); + public Timeout testTimeout = new Timeout(60_1000, TimeUnit.MILLISECONDS); @Test public void testBadConfiguration() throws IOException { - Configuration conf = new Configuration(); + Configuration conf = createConf(); conf.set(AWS_CREDENTIALS_PROVIDER, "no.such.class"); try { createFailingFS(conf); @@ -93,7 +95,7 @@ public class ITestS3AAWSCredentialsProvider { @Test public void testBadCredentialsConstructor() throws Exception { - Configuration conf = new Configuration(); + Configuration conf = createConf(); conf.set(AWS_CREDENTIALS_PROVIDER, BadCredentialsProviderConstructor.class.getName()); try { @@ -103,6 +105,14 @@ public class ITestS3AAWSCredentialsProvider { } } + protected Configuration createConf() { + Configuration conf = new Configuration(); + removeBaseAndBucketOverrides(conf, + DELEGATION_TOKEN_BINDING, + AWS_CREDENTIALS_PROVIDER); + return conf; + } + /** * Create a filesystem, expect it to fail by raising an IOException. * Raises an assertion exception if in fact the FS does get instantiated. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java index bac1bb13beb..f086a08201c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java @@ -67,6 +67,12 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { return conf; } + @Override + public void setup() throws Exception { + super.setup(); + assumeEnabled(); + } + @Override public void teardown() throws Exception { super.teardown(); @@ -85,9 +91,6 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { @Test public void testCreateFileAndReadWithDifferentEncryptionKey() throws Exception { - assumeEnabled(); - skipIfEncryptionTestsDisabled(getConfiguration()); - intercept(AccessDeniedException.class, SERVICE_AMAZON_S3_STATUS_CODE_403, () -> { @@ -117,9 +120,7 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { */ @Test public void testCreateSubdirWithDifferentKey() throws Exception { - assumeEnabled(); - skipIfEncryptionTestsDisabled(getConfiguration()); - assumeS3GuardState(false, getConfiguration()); + requireUnguardedFilesystem(); intercept(AccessDeniedException.class, SERVICE_AMAZON_S3_STATUS_CODE_403, @@ -145,9 +146,6 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { */ @Test public void testCreateFileThenMoveWithDifferentSSECKey() throws Exception { - assumeEnabled(); - skipIfEncryptionTestsDisabled(getConfiguration()); - intercept(AccessDeniedException.class, SERVICE_AMAZON_S3_STATUS_CODE_403, () -> { @@ -169,9 +167,6 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { */ @Test public void testRenameFile() throws Exception { - assumeEnabled(); - skipIfEncryptionTestsDisabled(getConfiguration()); - Path src = path("original-path.txt"); writeThenReadFile(src, TEST_FILE_LEN); Path newPath = path("different-path.txt"); @@ -188,9 +183,7 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { */ @Test public void testListEncryptedDir() throws Exception { - assumeEnabled(); - skipIfEncryptionTestsDisabled(getConfiguration()); - assumeS3GuardState(false, getConfiguration()); + requireUnguardedFilesystem(); Path pathABC = path("testListEncryptedDir/a/b/c/"); Path pathAB = pathABC.getParent(); @@ -235,9 +228,7 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { */ @Test public void testListStatusEncryptedDir() throws Exception { - assumeEnabled(); - skipIfEncryptionTestsDisabled(getConfiguration()); - assumeS3GuardState(false, getConfiguration()); + requireUnguardedFilesystem(); Path pathABC = path("testListStatusEncryptedDir/a/b/c/"); Path pathAB = pathABC.getParent(); @@ -282,11 +273,9 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { */ @Test public void testListStatusEncryptedFile() throws Exception { - assumeEnabled(); - skipIfEncryptionTestsDisabled(getConfiguration()); - assumeS3GuardState(false, getConfiguration()); + requireUnguardedFilesystem(); Path pathABC = path("testListStatusEncryptedFile/a/b/c/"); - assertTrue(getFileSystem().mkdirs(pathABC)); + assertTrue("mkdirs failed", getFileSystem().mkdirs(pathABC)); Path fileToStat = new Path(pathABC, "fileToStat.txt"); writeThenReadFile(fileToStat, TEST_FILE_LEN); @@ -301,6 +290,15 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { }); } + /** + * Skip the test case if S3Guard is enabled; generally this is because + * list and GetFileStatus calls can succeed even with different keys. + */ + protected void requireUnguardedFilesystem() { + assume("Filesystem has a metastore", + !getFileSystem().hasMetadataStore()); + } + /** * It is possible to delete directories without the proper encryption key and @@ -310,9 +308,7 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption { */ @Test public void testDeleteEncryptedObjectWithDifferentKey() throws Exception { - assumeEnabled(); - skipIfEncryptionTestsDisabled(getConfiguration()); - assumeS3GuardState(false, getConfiguration()); + requireUnguardedFilesystem(); Path pathABC = path("testDeleteEncryptedObjectWithDifferentKey/a/b/c/"); Path pathAB = pathABC.getParent(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java index 8b68fcfb880..039eb282dc3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java @@ -18,14 +18,15 @@ package org.apache.hadoop.fs.s3a; -import static org.hamcrest.CoreMatchers.containsString; - import java.io.IOException; import com.amazonaws.services.s3.model.ObjectMetadata; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import static org.hamcrest.CoreMatchers.containsString; + /** * Concrete class that extends {@link AbstractTestS3AEncryption} * and tests SSE-KMS encryption when no KMS encryption key is provided and AWS @@ -51,7 +52,7 @@ public class ITestS3AEncryptionSSEKMSDefaultKey @Override protected void assertEncrypted(Path path) throws IOException { ObjectMetadata md = getFileSystem().getObjectMetadata(path); - assertEquals("aws:kms", md.getSSEAlgorithm()); + assertEquals("SSE Algorithm", AWS_KMS_SSE_ALGORITHM, md.getSSEAlgorithm()); assertThat(md.getSSEAwsKmsKeyId(), containsString("arn:aws:kms:")); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java index a8a78f62825..67c403e0b6d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java @@ -18,11 +18,13 @@ package org.apache.hadoop.fs.s3a; -import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; - import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; +import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.SSE_KMS; + /** * Concrete class that extends {@link AbstractTestS3AEncryption} * and tests SSE-KMS encryption. This requires the SERVER_SIDE_ENCRYPTION_KEY @@ -33,16 +35,20 @@ public class ITestS3AEncryptionSSEKMSUserDefinedKey @Override protected Configuration createConfiguration() { - Configuration conf = super.createConfiguration(); - if(StringUtils.isBlank(conf.get(Constants.SERVER_SIDE_ENCRYPTION_KEY))){ - skip(Constants.SERVER_SIDE_ENCRYPTION_KEY+ " is not set for " + - S3AEncryptionMethods.SSE_KMS.getMethod()); + // get the KMS key for this test. + Configuration c = new Configuration(); + String kmsKey = c.get(SERVER_SIDE_ENCRYPTION_KEY); + if (StringUtils.isBlank(kmsKey)){ + skip(SERVER_SIDE_ENCRYPTION_KEY+ " is not set for " + + SSE_KMS.getMethod()); } + Configuration conf = super.createConfiguration(); + conf.set(SERVER_SIDE_ENCRYPTION_KEY, kmsKey); return conf; } @Override protected S3AEncryptionMethods getSSEAlgorithm() { - return S3AEncryptionMethods.SSE_KMS; + return SSE_KMS; } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java index 8f7f1beb444..cab568e4614 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java @@ -28,6 +28,7 @@ import com.amazonaws.services.s3.model.PutObjectRequest; import org.junit.Assume; import org.junit.Test; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileAlreadyExistsException; @@ -39,6 +40,9 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapab import static org.apache.hadoop.fs.contract.ContractTestUtils.assertLacksPathCapabilities; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; +import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; /** * Tests of the S3A FileSystem which don't have a specific home and can share @@ -56,6 +60,15 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase { enableChecksums(true); } + @Override + protected Configuration createConfiguration() { + final Configuration conf = super.createConfiguration(); + removeBaseAndBucketOverrides(conf, + SERVER_SIDE_ENCRYPTION_ALGORITHM, + SERVER_SIDE_ENCRYPTION_KEY); + return conf; + } + /** * Turn checksums on. * Relies on the FS not caching the configuration option @@ -149,7 +162,7 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase { CommonPathCapabilities.FS_CHECKSUMS); assertNotNull("Null file 1 checksum", checksum1); assertNotEquals("file 1 checksum", 0, checksum1.getLength()); - assertEquals("checksums", checksum1, + assertEquals("checksums of empty files", checksum1, fs.getFileChecksum(touchFile("file2"), 0)); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index c8e0d368793..db871a5f917 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -771,6 +771,11 @@ public final class S3ATestUtils { LOG.debug("Removing option {}; was {}", target, v); conf.unset(target); } + String extended = bucketPrefix + option; + if (conf.get(extended) != null) { + LOG.debug("Removing option {}", extended); + conf.unset(extended); + } } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java index 8942d70daa6..7163c5f4f53 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java @@ -35,13 +35,14 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants; import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.auth.RoleModel.*; import static org.apache.hadoop.fs.s3a.auth.RolePolicies.*; +import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -153,11 +154,14 @@ public final class RoleTestUtils { final Configuration srcConf, final String roleARN) { Configuration conf = new Configuration(srcConf); + removeBaseAndBucketOverrides(conf, + DELEGATION_TOKEN_BINDING, + ASSUMED_ROLE_ARN, + AWS_CREDENTIALS_PROVIDER); conf.set(AWS_CREDENTIALS_PROVIDER, AssumedRoleCredentialProvider.NAME); conf.set(ASSUMED_ROLE_ARN, roleARN); conf.set(ASSUMED_ROLE_SESSION_NAME, "test"); conf.set(ASSUMED_ROLE_SESSION_DURATION, "15m"); - conf.unset(DelegationConstants.DELEGATION_TOKEN_BINDING); disableFilesystemCaching(conf); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationIT.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationIT.java index e00a20e773a..d9c18d9df2d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationIT.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationIT.java @@ -148,6 +148,8 @@ public abstract class AbstractDelegationIT extends AbstractS3ATestBase { * @param binding binding to use */ protected void enableDelegationTokens(Configuration conf, String binding) { + removeBaseAndBucketOverrides(conf, + DELEGATION_TOKEN_BINDING); LOG.info("Enabling delegation token support for {}", binding); conf.set(DELEGATION_TOKEN_BINDING, binding); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java index 66a1fc4ea8c..f6c908ea2f7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java @@ -139,6 +139,10 @@ public class ITestSessionDelegationInFileystem extends AbstractDelegationIT { // disable if assume role opts are off assumeSessionTestsEnabled(conf); disableFilesystemCaching(conf); + removeBaseAndBucketOverrides(conf, + DELEGATION_TOKEN_BINDING, + SERVER_SIDE_ENCRYPTION_ALGORITHM, + SERVER_SIDE_ENCRYPTION_KEY); conf.set(HADOOP_SECURITY_AUTHENTICATION, UserGroupInformation.AuthenticationMethod.KERBEROS.name()); enableDelegationTokens(conf, getDelegationBinding()); @@ -332,6 +336,7 @@ public class ITestSessionDelegationInFileystem extends AbstractDelegationIT { removeBaseAndBucketOverrides(bucket, conf, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN, SERVER_SIDE_ENCRYPTION_ALGORITHM, + SERVER_SIDE_ENCRYPTION_KEY, DELEGATION_TOKEN_ROLE_ARN, DELEGATION_TOKEN_ENDPOINT); // this is done to make sure you cannot create an STS session no @@ -347,8 +352,10 @@ public class ITestSessionDelegationInFileystem extends AbstractDelegationIT { LOG.info("Delegated filesystem is: {}", delegatedFS); assertBoundToDT(delegatedFS, tokenKind); if (encryptionTestEnabled()) { + assertNotNull("Encryption propagation failed", + delegatedFS.getServerSideEncryptionAlgorithm()); assertEquals("Encryption propagation failed", - S3AEncryptionMethods.SSE_S3, + fs.getServerSideEncryptionAlgorithm(), delegatedFS.getServerSideEncryptionAlgorithm()); } verifyRestrictedPermissions(delegatedFS); @@ -380,8 +387,10 @@ public class ITestSessionDelegationInFileystem extends AbstractDelegationIT { try (S3AFileSystem secondDelegate = newS3AInstance(uri, conf)) { assertBoundToDT(secondDelegate, tokenKind); if (encryptionTestEnabled()) { + assertNotNull("Encryption propagation failed", + secondDelegate.getServerSideEncryptionAlgorithm()); assertEquals("Encryption propagation failed", - S3AEncryptionMethods.SSE_S3, + fs.getServerSideEncryptionAlgorithm(), secondDelegate.getServerSideEncryptionAlgorithm()); } ContractTestUtils.assertDeleted(secondDelegate, testPath, true); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java index 8530cb87960..69ca553b9ce 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestPartialRenamesDeletes.java @@ -66,6 +66,7 @@ import static org.apache.hadoop.fs.s3a.auth.RolePolicies.*; import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.bindRolePolicyStatements; import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.forbidden; import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.newAssumedRoleConfig; +import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING; import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit; import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletion; import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.extractUndeletedPaths; @@ -256,6 +257,7 @@ public class ITestPartialRenamesDeletes extends AbstractS3ATestBase { assumedRoleConfig = createAssumedRoleConfig(); bindRolePolicyStatements(assumedRoleConfig, STATEMENT_S3GUARD_CLIENT, + STATEMENT_ALLOW_SSE_KMS_RW, STATEMENT_ALL_BUCKET_READ_ACCESS, // root: r-x new Statement(Effects.Allow) // dest: rwx .addActions(S3_PATH_RW_OPERATIONS) @@ -311,9 +313,9 @@ public class ITestPartialRenamesDeletes extends AbstractS3ATestBase { private Configuration createAssumedRoleConfig(String roleARN) { Configuration conf = newAssumedRoleConfig(getContract().getConf(), roleARN); - String bucketName = getTestBucketName(conf); - - removeBucketOverrides(bucketName, conf, ENABLE_MULTI_DELETE); + removeBaseAndBucketOverrides(conf, + DELEGATION_TOKEN_BINDING, + ENABLE_MULTI_DELETE); conf.setBoolean(ENABLE_MULTI_DELETE, multiDelete); return conf; }