Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14276. [SBN read] Reduce tailing overhead. Contributed by Wei-Chiu Chuang.

This commit is contained in:
Ayush Saxena 2019-08-22 02:25:06 +05:30
parent 8fc6567b94
commit 0f598aed13
2 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
View File

@ -468,9 +468,11 @@ public class EditLogTailer {
// There's no point in triggering a log roll if the Standby hasn't // There's no point in triggering a log roll if the Standby hasn't
// read any more transactions since the last time a roll was // read any more transactions since the last time a roll was
// triggered. // triggered.
boolean triggeredLogRoll = false;
if (tooLongSinceLastLoad() && if (tooLongSinceLastLoad() &&
lastRollTriggerTxId < lastLoadedTxnId) { lastRollTriggerTxId < lastLoadedTxnId) {
triggerActiveLogRoll(); triggerActiveLogRoll();
triggeredLogRoll = true;
} }
/** /**
* Check again in case someone calls {@link EditLogTailer#stop} while * Check again in case someone calls {@link EditLogTailer#stop} while
@ -496,7 +498,9 @@ public class EditLogTailer {
Time.monotonicNow() - startTime); Time.monotonicNow() - startTime);
} }
//Update NameDirSize Metric //Update NameDirSize Metric
namesystem.getFSImage().getStorage().updateNameDirSize(); if (triggeredLogRoll) {
namesystem.getFSImage().getStorage().updateNameDirSize();
}
} catch (EditLogInputException elie) { } catch (EditLogInputException elie) {
LOG.warn("Error while reading edits from disk. Will try again.", elie); LOG.warn("Error while reading edits from disk. Will try again.", elie);
} catch (InterruptedException ie) { } catch (InterruptedException ie) {

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
View File

@ -671,6 +671,7 @@ public class TestNameNodeMXBean {
public void testNNDirectorySize() throws Exception{ public void testNNDirectorySize() throws Exception{
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 0);
MiniDFSCluster cluster = null; MiniDFSCluster cluster = null;
for (int i = 0; i < 5; i++) { for (int i = 0; i < 5; i++) {
try{ try{
@ -700,8 +701,6 @@ public class TestNameNodeMXBean {
FSNamesystem nn0 = cluster.getNamesystem(0); FSNamesystem nn0 = cluster.getNamesystem(0);
FSNamesystem nn1 = cluster.getNamesystem(1); FSNamesystem nn1 = cluster.getNamesystem(1);
checkNNDirSize(cluster.getNameDirs(0), nn0.getNameDirSize());
checkNNDirSize(cluster.getNameDirs(1), nn1.getNameDirSize());
cluster.transitionToActive(0); cluster.transitionToActive(0);
fs = cluster.getFileSystem(0); fs = cluster.getFileSystem(0);
DFSTestUtil.createFile(fs, new Path("/file"), 0, (short) 1, 0L); DFSTestUtil.createFile(fs, new Path("/file"), 0, (short) 1, 0L);