diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt index 8b10741ed4e..36c162482b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt @@ -180,3 +180,5 @@ HDFS-2733. Document HA configuration and CLI. (atm) HDFS-2794. Active NN may purge edit log files before standby NN has a chance to read them (todd) HDFS-2901. Improvements for SBN web UI - not show under-replicated/missing blocks. (Brandon Li via jitendra) + +HDFS-2905. HA: Standby NN NPE when shared edits dir is deleted. (Bikas Saha via jitendra) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java index 90bf1a77def..1eca2797b44 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java @@ -135,8 +135,7 @@ class FileJournalManager implements JournalManager { */ List getRemoteEditLogs(long firstTxId) throws IOException { File currentDir = sd.getCurrentDir(); - List allLogFiles = matchEditLogs( - FileUtil.listFiles(currentDir)); + List allLogFiles = matchEditLogs(currentDir); List ret = Lists.newArrayListWithCapacity( allLogFiles.size()); @@ -155,6 +154,20 @@ class FileJournalManager implements JournalManager { return ret; } + /** + * returns matching edit logs via the log directory. Simple helper function + * that lists the files in the logDir and calls matchEditLogs(File[]) + * + * @param logDir + * directory to match edit logs in + * @return matched edit logs + * @throws IOException + * IOException thrown for invalid logDir + */ + static List matchEditLogs(File logDir) throws IOException { + return matchEditLogs(FileUtil.listFiles(logDir)); + } + static List matchEditLogs(File[] filesInStorage) { List ret = Lists.newArrayList(); for (File f : filesInStorage) { @@ -278,7 +291,7 @@ class FileJournalManager implements JournalManager { synchronized public void recoverUnfinalizedSegments() throws IOException { File currentDir = sd.getCurrentDir(); LOG.info("Recovering unfinalized segments in " + currentDir); - List allLogFiles = matchEditLogs(currentDir.listFiles()); + List allLogFiles = matchEditLogs(currentDir); for (EditLogFile elf : allLogFiles) { if (elf.getFile().equals(currentInProgress)) { @@ -318,7 +331,7 @@ class FileJournalManager implements JournalManager { private List getLogFiles(long fromTxId) throws IOException { File currentDir = sd.getCurrentDir(); - List allLogFiles = matchEditLogs(currentDir.listFiles()); + List allLogFiles = matchEditLogs(currentDir); List logFiles = Lists.newArrayList(); for (EditLogFile elf : allLogFiles) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java index f0b8a6d2b30..665e088cb80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java @@ -440,7 +440,7 @@ public abstract class FSImageTestUtil { throws IOException { File currentDir = sd.getCurrentDir(); List foundEditLogs - = Lists.newArrayList(FileJournalManager.matchEditLogs(currentDir.listFiles())); + = Lists.newArrayList(FileJournalManager.matchEditLogs(currentDir)); return Collections.max(foundEditLogs, EditLogFile.COMPARE_BY_START_TXID); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java index b862727b0e6..def29365776 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java @@ -315,6 +315,15 @@ public class TestFileJournalManager { "", getLogsAsString(fjm, 9999)); } + /** + * tests that passing an invalid dir to matchEditLogs throws IOException + */ + @Test(expected = IOException.class) + public void testMatchEditLogInvalidDirThrowsIOException() throws IOException { + File badDir = new File("does not exist"); + FileJournalManager.matchEditLogs(badDir); + } + /** * Make sure that we starting reading the correct op when we request a stream * with a txid in the middle of an edit log file.