From e6b5b015613973d5960c9aeae4866e587d3ecf1d Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 24 Apr 2013 23:29:30 +0000 Subject: [PATCH] HDFS-4739. NN can miscalculate the number of extra edit log segments to retain. Contributed by Aaron T. Myers. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1471769 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/namenode/NNStorageRetentionManager.java | 8 +++++++- .../server/namenode/TestNNStorageRetentionManager.java | 7 ++++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 066cf56fd5c..5bc0b87ecbc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -585,6 +585,9 @@ Release 2.0.5-beta - UNRELEASED HDFS-4737. JVM path embedded in fuse binaries. (Sean Mackrory via atm) + HDFS-4739. NN can miscalculate the number of extra edit log segments to + retain. (atm) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java index 8c51a99a36a..75d1bd82fec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java @@ -118,11 +118,17 @@ public class NNStorageRetentionManager { .result(); } }); + + // Remove from consideration any edit logs that are in fact required. + while (editLogs.size() > 0 && + editLogs.get(editLogs.size() - 1).getFirstTxId() >= minimumRequiredTxId) { + editLogs.remove(editLogs.size() - 1); + } // Next, adjust the number of transactions to retain if doing so would mean // keeping too many segments around. while (editLogs.size() > maxExtraEditsSegmentsToRetain) { - purgeLogsFrom = editLogs.get(0).getFirstTxId(); + purgeLogsFrom = editLogs.get(0).getLastTxId() + 1; editLogs.remove(0); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java index 4c7662efdba..161a9f1fe84 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java @@ -217,11 +217,12 @@ public class TestNNStorageRetentionManager { tc.addLog("/foo2/current/" + getFinalizedEditsFileName(176, 200), true); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 225), true); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(226, 240), true); - // Only retain 2 extra segments. The 301-400 segment is considered required, - // not extra. + // Only retain 2 extra segments. The 301-350 and 351-400 segments are + // considered required, not extra. tc.addLog("/foo2/current/" + getFinalizedEditsFileName(241, 275), false); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(276, 300), false); - tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 400), false); + tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 350), false); + tc.addLog("/foo2/current/" + getFinalizedEditsFileName(351, 400), false); tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false); runTest(tc); }