Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-16632 Speculating & Partitioned S3A magic committers can leave pending files under __magic (#1599) 

Contributed by Steve Loughran.

This downgrade the checks for leftover __magic entries from fail to warn now the parallel
test runs make speculation more likely. 

Change-Id: Ia4df2e90f82a06dbae69f3fdaadcbb0e0d713b38
This commit is contained in:
Steve Loughran 2019-11-19 13:54:33 +00:00 committed by GitHub
parent 30b2096933
commit 9fbfe6c8f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java
View File

@ -628,16 +628,18 @@ public class ITestS3ACommitterMRJob extends AbstractYarnClusterITest {
S3AFileSystem fs = getRemoteFS(); S3AFileSystem fs = getRemoteFS();
// log the contents // log the contents
lsR(fs, destPath, true); lsR(fs, destPath, true);
intercept(FileNotFoundException.class, () -> { // and look for the magic directory
// HADOOP-16632 shows how partitioned/speculative tasks can leave
// data here and it is not an error. So just log and continue
try {
final FileStatus st = fs.getFileStatus(magicDir); final FileStatus st = fs.getFileStatus(magicDir);
StringBuilder result = new StringBuilder("Found magic dir which should" LOG.warn("Found magic dir which should"
+ " have been deleted at ").append(st).append('\n'); + " have been deleted at {}", st);
result.append(" [");
applyLocatedFiles(fs.listFiles(magicDir, true), applyLocatedFiles(fs.listFiles(magicDir, true),
(status) -> result.append(" ").append(status.getPath()).append('\n')); (status) -> LOG.warn("{}", status));
result.append("]"); } catch (FileNotFoundException ignored) {
return result.toString(); // expected
}); }
} }
} }