From 0b008dc21c3c8c9dbb211d0d176db5936d853485 Mon Sep 17 00:00:00 2001 From: Jian He Date: Wed, 20 Apr 2016 19:02:10 -0700 Subject: [PATCH] MAPREDUCE-6680. JHS UserLogDir scan algorithm sometime could skip directory with update in CloudFS (Azure FileSystem, S3, etc. Contributed by Junping Du (cherry picked from commit 1e48eefe5800975ea0c4295c9911ae3f572ed37d) --- .../mapreduce/v2/hs/HistoryFileManager.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 01fb9d50b32..0b22d13f1d1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -306,10 +306,21 @@ public class HistoryFileManager extends AbstractService { */ private class UserLogDir { long modTime = 0; - + private long scanTime = 0; + public synchronized void scanIfNeeded(FileStatus fs) { long newModTime = fs.getModificationTime(); - if (modTime != newModTime) { + // MAPREDUCE-6680: In some Cloud FileSystem, like Azure FS or S3, file's + // modification time is truncated into seconds. In that case, + // modTime == newModTime doesn't means no file update in the directory, + // so we need to have additional check. + // Note: modTime (X second Y millisecond) could be casted to X second or + // X+1 second. + if (modTime != newModTime + || (scanTime/1000) == (modTime/1000) + || (scanTime/1000 + 1) == (modTime/1000)) { + // reset scanTime before scanning happens + scanTime = System.currentTimeMillis(); Path p = fs.getPath(); try { scanIntermediateDirectory(p); @@ -323,10 +334,12 @@ public class HistoryFileManager extends AbstractService { if (LOG.isDebugEnabled()) { LOG.debug("Scan not needed of " + fs.getPath()); } + // reset scanTime + scanTime = System.currentTimeMillis(); } } } - + public class HistoryFileInfo { private Path historyFile; private Path confFile;