Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-17385. ITestS3ADeleteCost.testDirMarkersFileCreation failure (#2473). Contributed by Steve Loughran 

The addition of deprecated S3A configuration options in HADOOP-17318
triggered a reload of default (xml resource) configurations, which breaks
tests which fail if there's a per-bucket setting inconsistent with test
setup.

Creating an S3AFS instance before creating the Configuration() instance
for test runs gets that reload out the way before test setup takes
place.

Along with the fix, extra changes in the failing test suite to fail
fast when marker policy isn't as expected, and to log FS state better.

Rather than create and discard an instance, add a new static method
to S3AFS and invoke it in test setup. This forces the load

Change-Id: Id52b1c46912c6fedd2ae270e2b1eb2222a360329
This commit is contained in:
Steve Loughran 2020-11-26 12:50:33 +00:00 committed by GitHub
parent cf43a7eaae
commit 67dc0928c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 26 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -4954,6 +4954,15 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
return new MarkerToolOperationsImpl(operationCallbacks);
}
/**
* This is purely for testing, as it force initializes all static
* initializers. See HADOOP-17385 for details.
*/
@InterfaceAudience.Private
public static void initializeClass() {
LOG.debug("Initialize S3A class");
}
/**
* The implementation of context accessors.
*/

4
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.fs.contract.s3a;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
/**
@ -53,6 +54,9 @@ public class S3AContract extends AbstractBondedFSContract {
*/
public S3AContract(Configuration conf, boolean addContractResource) {
super(conf);
// Force deprecated key load through the
// static initializers. See: HADOOP-17385
S3AFileSystem.initializeClass();
//insert the base features
if (addContractResource) {
addConfResource(CONTRACT_XML);

3
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
View File

@ -60,6 +60,9 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase
// filesystems which add default configuration resources to do it before
// our tests start adding/removing options. See HADOOP-16626.
FileSystem.getLocal(new Configuration());
// Force deprecated key load through the
// static initializers. See: HADOOP-17385
S3AFileSystem.initializeClass();
super.setup();
}

9
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java
View File

@ -35,6 +35,7 @@ import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.Statistic;
import org.apache.hadoop.fs.s3a.Tristate;
import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy;
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
import static org.apache.hadoop.fs.s3a.Constants.*;
@ -141,6 +142,14 @@ public class AbstractS3ACostTest extends AbstractS3ATestBase {
isDeleting = !isKeeping;
// check that the FS has the expected state
DirectoryPolicy markerPolicy = fs.getDirectoryMarkerPolicy();
Assertions.assertThat(markerPolicy.getMarkerPolicy())
.describedAs("Marker policy for filesystem %s", fs)
.isEqualTo(isKeepingMarkers()
? DirectoryPolicy.MarkerPolicy.Keep
: DirectoryPolicy.MarkerPolicy.Delete);
// insert new metrics so as to keep the list sorted
costValidator = OperationCostValidator.builder(getFileSystem())
.withMetrics(

1
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java
View File

@ -267,6 +267,7 @@ public class ITestS3ADeleteCost extends AbstractS3ACostTest {
verifyMetrics(() -> {
file(new Path(srcDir, "source.txt"));
LOG.info("Metrics: {}\n{}", getMetricSummary(), getFileSystem());
return "after touch(fs, srcFilePath) " + getMetricSummary();
},
with(DIRECTORIES_CREATED, 0),