Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-4739. NN can miscalculate the number of extra edit log segments to retain. Contributed by Aaron T. Myers. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1471769 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Aaron Myers 2013-04-24 23:29:30 +00:00
parent a2f97fd9cb
commit e6b5b01561
3 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -585,6 +585,9 @@ Release 2.0.5-beta - UNRELEASED
HDFS-4737. JVM path embedded in fuse binaries. (Sean Mackrory via atm) HDFS-4737. JVM path embedded in fuse binaries. (Sean Mackrory via atm)
HDFS-4739. NN can miscalculate the number of extra edit log segments to
retain. (atm)
Release 2.0.4-alpha - UNRELEASED Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java
View File

@ -119,10 +119,16 @@ public class NNStorageRetentionManager {
} }
}); });
// Remove from consideration any edit logs that are in fact required.
while (editLogs.size() > 0 &&
editLogs.get(editLogs.size() - 1).getFirstTxId() >= minimumRequiredTxId) {
editLogs.remove(editLogs.size() - 1);
}
// Next, adjust the number of transactions to retain if doing so would mean // Next, adjust the number of transactions to retain if doing so would mean
// keeping too many segments around. // keeping too many segments around.
while (editLogs.size() > maxExtraEditsSegmentsToRetain) { while (editLogs.size() > maxExtraEditsSegmentsToRetain) {
purgeLogsFrom = editLogs.get(0).getFirstTxId(); purgeLogsFrom = editLogs.get(0).getLastTxId() + 1;
editLogs.remove(0); editLogs.remove(0);
} }

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java
View File

@ -217,11 +217,12 @@ public class TestNNStorageRetentionManager {
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(176, 200), true); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(176, 200), true);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 225), true); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 225), true);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(226, 240), true); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(226, 240), true);
// Only retain 2 extra segments. The 301-400 segment is considered required, // Only retain 2 extra segments. The 301-350 and 351-400 segments are
// not extra. // considered required, not extra.
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(241, 275), false); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(241, 275), false);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(276, 300), false); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(276, 300), false);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 350), false);
tc.addLog("/foo2/current/" + getFinalizedEditsFileName(351, 400), false);
tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false); tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false);
runTest(tc); runTest(tc);
} }