HADOOP-16319. S3A Etag tests fail with default encryption enabled on bucket.

Contributed by Ben Roling.

ETag values are unpredictable with some S3 encryption algorithms.

Skip ITestS3AMiscOperations tests which make assertions about etags
when default encryption on a bucket is enabled.

When testing with an AWS an account which lacks the privilege
for a call to getBucketEncryption(), we don't skip the tests.
In the event of failure, developers get to expand the
permissions of the account or relax default encryption settings.
This commit is contained in:
Steve Loughran 2020-03-17 13:31:48 +00:00 committed by GitHub
parent 1975479285
commit 8d6373483e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 47 additions and 1 deletions

View File

@ -168,6 +168,13 @@ You can also force all the tests to run with a specific SSE encryption method
by configuring the property `fs.s3a.server-side-encryption-algorithm` in the s3a by configuring the property `fs.s3a.server-side-encryption-algorithm` in the s3a
contract file. contract file.
### <a name="default_encyption"></a> Default Encryption
Buckets can be configured with [default encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html)
on the AWS side. Some S3AFileSystem tests are skipped when default encryption is
enabled due to unpredictability in how [ETags](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html)
are generated.
## <a name="running"></a> Running the Tests ## <a name="running"></a> Running the Tests
After completing the configuration, execute the test run through Maven. After completing the configuration, execute the test run through Maven.

View File

@ -23,7 +23,10 @@
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GetBucketEncryptionResult;
import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.PutObjectRequest;
import org.junit.Assume; import org.junit.Assume;
@ -44,6 +47,7 @@
import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM; import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM;
import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY; import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.hamcrest.Matchers.nullValue;
/** /**
* Tests of the S3A FileSystem which don't have a specific home and can share * Tests of the S3A FileSystem which don't have a specific home and can share
@ -151,10 +155,12 @@ Path mkFile(String name, byte[] data) throws IOException {
/** /**
* The assumption here is that 0-byte files uploaded in a single PUT * The assumption here is that 0-byte files uploaded in a single PUT
* always have the same checksum, including stores with encryption. * always have the same checksum, including stores with encryption.
* This will be skipped if the bucket has S3 default encryption enabled.
* @throws Throwable on a failure * @throws Throwable on a failure
*/ */
@Test @Test
public void testEmptyFileChecksums() throws Throwable { public void testEmptyFileChecksums() throws Throwable {
assumeNoDefaultEncryption();
final S3AFileSystem fs = getFileSystem(); final S3AFileSystem fs = getFileSystem();
Path file1 = touchFile("file1"); Path file1 = touchFile("file1");
EtagChecksum checksum1 = fs.getFileChecksum(file1, 0); EtagChecksum checksum1 = fs.getFileChecksum(file1, 0);
@ -167,6 +173,20 @@ public void testEmptyFileChecksums() throws Throwable {
fs.getFileChecksum(touchFile("file2"), 0)); fs.getFileChecksum(touchFile("file2"), 0));
} }
/**
* Skip a test if we can get the default encryption on a bucket and it is
* non-null.
*/
private void assumeNoDefaultEncryption() throws IOException {
try {
Assume.assumeThat(getDefaultEncryption(), nullValue());
} catch (AccessDeniedException e) {
// if the user can't check the default encryption, assume that it is
// null and keep going
LOG.warn("User does not have permission to call getBucketEncryption()");
}
}
/** /**
* Make sure that when checksums are disabled, the caller * Make sure that when checksums are disabled, the caller
* gets null back. * gets null back.
@ -207,12 +227,13 @@ public void testNonEmptyFileChecksums() throws Throwable {
/** /**
* Verify that on an unencrypted store, the checksum of two non-empty * Verify that on an unencrypted store, the checksum of two non-empty
* (single PUT) files is the same if the data is the same. * (single PUT) files is the same if the data is the same.
* This will fail if the bucket has S3 default encryption enabled. * This will be skipped if the bucket has S3 default encryption enabled.
* @throws Throwable failure * @throws Throwable failure
*/ */
@Test @Test
public void testNonEmptyFileChecksumsUnencrypted() throws Throwable { public void testNonEmptyFileChecksumsUnencrypted() throws Throwable {
Assume.assumeTrue(encryptionAlgorithm().equals(S3AEncryptionMethods.NONE)); Assume.assumeTrue(encryptionAlgorithm().equals(S3AEncryptionMethods.NONE));
assumeNoDefaultEncryption();
final S3AFileSystem fs = getFileSystem(); final S3AFileSystem fs = getFileSystem();
final EtagChecksum checksum1 = final EtagChecksum checksum1 =
fs.getFileChecksum(mkFile("file5", HELLO), 0); fs.getFileChecksum(mkFile("file5", HELLO), 0);
@ -256,6 +277,7 @@ public void testS3AToStringUnitialized() throws Throwable {
} }
/** /**
<<<<<<< ours
* Verify that paths with a trailing "/" are fixed up. * Verify that paths with a trailing "/" are fixed up.
*/ */
@Test @Test
@ -368,4 +390,21 @@ private static <T> T verifyNoTrailingSlash(String role, T o) {
s.endsWith("/")); s.endsWith("/"));
return o; return o;
} }
/**
* Gets default encryption settings for the bucket or returns null if default
* encryption is disabled.
*/
private GetBucketEncryptionResult getDefaultEncryption() throws IOException {
S3AFileSystem fs = getFileSystem();
AmazonS3 s3 = fs.getAmazonS3ClientForTesting("check default encryption");
try {
return Invoker.once("getBucketEncryption()",
fs.getBucket(),
() -> s3.getBucketEncryption(fs.getBucket()));
} catch (FileNotFoundException e) {
return null;
}
}
} }