HADOOP-16319. S3A Etag tests fail with default encryption enabled on bucket.
Contributed by Ben Roling. ETag values are unpredictable with some S3 encryption algorithms. Skip ITestS3AMiscOperations tests which make assertions about etags when default encryption on a bucket is enabled. When testing with an AWS an account which lacks the privilege for a call to getBucketEncryption(), we don't skip the tests. In the event of failure, developers get to expand the permissions of the account or relax default encryption settings.
This commit is contained in:
parent
1975479285
commit
8d6373483e
|
@ -168,6 +168,13 @@ You can also force all the tests to run with a specific SSE encryption method
|
||||||
by configuring the property `fs.s3a.server-side-encryption-algorithm` in the s3a
|
by configuring the property `fs.s3a.server-side-encryption-algorithm` in the s3a
|
||||||
contract file.
|
contract file.
|
||||||
|
|
||||||
|
### <a name="default_encyption"></a> Default Encryption
|
||||||
|
|
||||||
|
Buckets can be configured with [default encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html)
|
||||||
|
on the AWS side. Some S3AFileSystem tests are skipped when default encryption is
|
||||||
|
enabled due to unpredictability in how [ETags](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html)
|
||||||
|
are generated.
|
||||||
|
|
||||||
## <a name="running"></a> Running the Tests
|
## <a name="running"></a> Running the Tests
|
||||||
|
|
||||||
After completing the configuration, execute the test run through Maven.
|
After completing the configuration, execute the test run through Maven.
|
||||||
|
|
|
@ -23,7 +23,10 @@ import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.AccessDeniedException;
|
||||||
|
|
||||||
|
import com.amazonaws.services.s3.AmazonS3;
|
||||||
|
import com.amazonaws.services.s3.model.GetBucketEncryptionResult;
|
||||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||||
import com.amazonaws.services.s3.model.PutObjectRequest;
|
import com.amazonaws.services.s3.model.PutObjectRequest;
|
||||||
import org.junit.Assume;
|
import org.junit.Assume;
|
||||||
|
@ -44,6 +47,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM;
|
import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY;
|
import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY;
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
|
||||||
|
import static org.hamcrest.Matchers.nullValue;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tests of the S3A FileSystem which don't have a specific home and can share
|
* Tests of the S3A FileSystem which don't have a specific home and can share
|
||||||
|
@ -151,10 +155,12 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
|
||||||
/**
|
/**
|
||||||
* The assumption here is that 0-byte files uploaded in a single PUT
|
* The assumption here is that 0-byte files uploaded in a single PUT
|
||||||
* always have the same checksum, including stores with encryption.
|
* always have the same checksum, including stores with encryption.
|
||||||
|
* This will be skipped if the bucket has S3 default encryption enabled.
|
||||||
* @throws Throwable on a failure
|
* @throws Throwable on a failure
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testEmptyFileChecksums() throws Throwable {
|
public void testEmptyFileChecksums() throws Throwable {
|
||||||
|
assumeNoDefaultEncryption();
|
||||||
final S3AFileSystem fs = getFileSystem();
|
final S3AFileSystem fs = getFileSystem();
|
||||||
Path file1 = touchFile("file1");
|
Path file1 = touchFile("file1");
|
||||||
EtagChecksum checksum1 = fs.getFileChecksum(file1, 0);
|
EtagChecksum checksum1 = fs.getFileChecksum(file1, 0);
|
||||||
|
@ -167,6 +173,20 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
|
||||||
fs.getFileChecksum(touchFile("file2"), 0));
|
fs.getFileChecksum(touchFile("file2"), 0));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Skip a test if we can get the default encryption on a bucket and it is
|
||||||
|
* non-null.
|
||||||
|
*/
|
||||||
|
private void assumeNoDefaultEncryption() throws IOException {
|
||||||
|
try {
|
||||||
|
Assume.assumeThat(getDefaultEncryption(), nullValue());
|
||||||
|
} catch (AccessDeniedException e) {
|
||||||
|
// if the user can't check the default encryption, assume that it is
|
||||||
|
// null and keep going
|
||||||
|
LOG.warn("User does not have permission to call getBucketEncryption()");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Make sure that when checksums are disabled, the caller
|
* Make sure that when checksums are disabled, the caller
|
||||||
* gets null back.
|
* gets null back.
|
||||||
|
@ -207,12 +227,13 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
|
||||||
/**
|
/**
|
||||||
* Verify that on an unencrypted store, the checksum of two non-empty
|
* Verify that on an unencrypted store, the checksum of two non-empty
|
||||||
* (single PUT) files is the same if the data is the same.
|
* (single PUT) files is the same if the data is the same.
|
||||||
* This will fail if the bucket has S3 default encryption enabled.
|
* This will be skipped if the bucket has S3 default encryption enabled.
|
||||||
* @throws Throwable failure
|
* @throws Throwable failure
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testNonEmptyFileChecksumsUnencrypted() throws Throwable {
|
public void testNonEmptyFileChecksumsUnencrypted() throws Throwable {
|
||||||
Assume.assumeTrue(encryptionAlgorithm().equals(S3AEncryptionMethods.NONE));
|
Assume.assumeTrue(encryptionAlgorithm().equals(S3AEncryptionMethods.NONE));
|
||||||
|
assumeNoDefaultEncryption();
|
||||||
final S3AFileSystem fs = getFileSystem();
|
final S3AFileSystem fs = getFileSystem();
|
||||||
final EtagChecksum checksum1 =
|
final EtagChecksum checksum1 =
|
||||||
fs.getFileChecksum(mkFile("file5", HELLO), 0);
|
fs.getFileChecksum(mkFile("file5", HELLO), 0);
|
||||||
|
@ -256,6 +277,7 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
<<<<<<< ours
|
||||||
* Verify that paths with a trailing "/" are fixed up.
|
* Verify that paths with a trailing "/" are fixed up.
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
|
@ -368,4 +390,21 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
|
||||||
s.endsWith("/"));
|
s.endsWith("/"));
|
||||||
return o;
|
return o;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets default encryption settings for the bucket or returns null if default
|
||||||
|
* encryption is disabled.
|
||||||
|
*/
|
||||||
|
private GetBucketEncryptionResult getDefaultEncryption() throws IOException {
|
||||||
|
S3AFileSystem fs = getFileSystem();
|
||||||
|
AmazonS3 s3 = fs.getAmazonS3ClientForTesting("check default encryption");
|
||||||
|
try {
|
||||||
|
return Invoker.once("getBucketEncryption()",
|
||||||
|
fs.getBucket(),
|
||||||
|
() -> s3.getBucketEncryption(fs.getBucket()));
|
||||||
|
} catch (FileNotFoundException e) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue