diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 4166e6aa952..f9488fbb332 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -349,6 +349,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6333. TestEvents,TestAMWebServicesTasks,TestAppController are broken due to MAPREDUCE-6297. (Siqi Li via gera) + MAPREDUCE-6252. JobHistoryServer should not fail when encountering a + missing directory. (Craig Welch via devaraj) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 65f8a4febfc..69f814dc5f6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -740,17 +740,22 @@ public class HistoryFileManager extends AbstractService { } } - private static List scanDirectory(Path path, FileContext fc, + @VisibleForTesting + protected static List scanDirectory(Path path, FileContext fc, PathFilter pathFilter) throws IOException { path = fc.makeQualified(path); List jhStatusList = new ArrayList(); - RemoteIterator fileStatusIter = fc.listStatus(path); - while (fileStatusIter.hasNext()) { - FileStatus fileStatus = fileStatusIter.next(); - Path filePath = fileStatus.getPath(); - if (fileStatus.isFile() && pathFilter.accept(filePath)) { - jhStatusList.add(fileStatus); + try { + RemoteIterator fileStatusIter = fc.listStatus(path); + while (fileStatusIter.hasNext()) { + FileStatus fileStatus = fileStatusIter.next(); + Path filePath = fileStatus.getPath(); + if (fileStatus.isFile() && pathFilter.accept(filePath)) { + jhStatusList.add(fileStatus); + } } + } catch (FileNotFoundException fe) { + LOG.error("Error while scanning directory " + path, fe); } return jhStatusList; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java index e2e943a3d65..1c5cc5ce741 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java @@ -21,13 +21,17 @@ package org.apache.hadoop.mapreduce.v2.hs; import java.io.File; import java.io.FileOutputStream; +import java.io.FileNotFoundException; import java.util.UUID; +import java.util.List; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -45,6 +49,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; +import static org.mockito.Mockito.*; + public class TestHistoryFileManager { private static MiniDFSCluster dfsCluster = null; private static MiniDFSCluster dfsCluster2 = null; @@ -199,4 +205,20 @@ public class TestHistoryFileManager { testCreateHistoryDirs(dfsCluster.getConfiguration(0), clock); } + @Test + public void testScanDirectory() throws Exception { + + Path p = new Path("any"); + FileContext fc = mock(FileContext.class); + when(fc.makeQualified(p)).thenReturn(p); + when(fc.listStatus(p)).thenThrow(new FileNotFoundException()); + + List lfs = HistoryFileManager.scanDirectory(p, fc, null); + + //primarily, succcess is that an exception was not thrown. Also nice to + //check this + Assert.assertNotNull(lfs); + + } + }