diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 3be260943f1..e0de1d2458c 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -52,6 +52,9 @@ false + + + unset @@ -126,6 +129,8 @@ ${fs.s3a.directory.marker.retention} ${fs.s3a.directory.marker.audit} + + ${fs.s3a.prefetch.enabled} @@ -165,6 +170,8 @@ ${fs.s3a.directory.marker.retention} ${test.integration.timeout} + + ${fs.s3a.prefetch.enabled} @@ -214,6 +221,8 @@ ${fs.s3a.directory.marker.retention} ${fs.s3a.directory.marker.audit} + + ${fs.s3a.prefetch.enabled} @@ -270,6 +279,8 @@ ${fs.s3a.directory.marker.retention} ${fs.s3a.directory.marker.audit} + + ${fs.s3a.prefetch.enabled} ${fs.s3a.scale.test.timeout} @@ -331,6 +342,19 @@ + + + prefetch + + + prefetch + + + + true + + + diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index bbe1d0f8081..d2ed9ede017 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -617,6 +617,20 @@ your `core-site.xml` file, so that trying to use S3 select fails fast with a meaningful error ("S3 Select not supported") rather than a generic Bad Request exception. +### Enabling prefetch for all tests + +The tests are run with prefetch if the `prefetch` property is set in the +maven build. This can be combined with the scale tests as well. + +```bash +mvn verify -Dprefetch + +mvn verify -Dparallel-tests -Dprefetch -DtestsThreadCount=8 + +mvn verify -Dparallel-tests -Dprefetch -Dscale -DtestsThreadCount=8 +``` + + ### Testing Requester Pays By default, the requester pays tests will look for a bucket that exists on Amazon S3 diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 384142d899a..4940eeb2e8f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -575,6 +575,10 @@ public final class S3ATestUtils { DEFAULT_DIRECTORY_MARKER_POLICY); conf.set(DIRECTORY_MARKER_POLICY, directoryRetention); + boolean prefetchEnabled = + getTestPropertyBool(conf, PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT); + conf.setBoolean(PREFETCH_ENABLED_KEY, prefetchEnabled); + return conf; }