Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-17398. Skipping network I/O in S3A getFileStatus(/) breaks some tests (#2493) 

Follow-on to HADOOP-17323.

Contributed by Mukund Thakur.
This commit is contained in:
Mukund Thakur 2020-11-27 01:55:32 +05:30 committed by GitHub
parent 67dc0928c1
commit 03b4e98971
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 22 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/MarkerTool.java
View File

@ -282,16 +282,24 @@ public final class MarkerTool extends S3GuardTool {
}
FileSystem fs = path.getFileSystem(getConf());
boolean nonAuth = command.getOpt(OPT_NONAUTH);
ScanResult result = execute(
new ScanArgsBuilder()
.withSourceFS(fs)
.withPath(path)
.withDoPurge(clean)
.withMinMarkerCount(expectedMin)
.withMaxMarkerCount(expectedMax)
.withLimit(limit)
.withNonAuth(nonAuth)
.build());
ScanResult result;
try {
result = execute(
new ScanArgsBuilder()
.withSourceFS(fs)
.withPath(path)
.withDoPurge(clean)
.withMinMarkerCount(expectedMin)
.withMaxMarkerCount(expectedMax)
.withLimit(limit)
.withNonAuth(nonAuth)
.build());
} catch (UnknownStoreException ex) {
// bucket doesn't exist.
// replace the stack trace with an error code.
throw new ExitUtil.ExitException(EXIT_NOT_FOUND,
ex.toString(), ex);
}
if (verbose) {
dumpFileSystemStatistics(out);
}

6
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java
View File

@ -62,8 +62,10 @@ public class ITestS3ABucketExistence extends AbstractS3ATestBase {
Path root = new Path(uri);
expectUnknownStore(
() -> fs.getFileStatus(root));
//See HADOOP-17323.
assertTrue("root path should always exist", fs.exists(root));
assertTrue("getFileStatus on root should always return a directory",
fs.getFileStatus(root).isDirectory());
expectUnknownStore(
() -> fs.listStatus(root));