Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14859. Prevent unnecessary evaluation of costly operation getNumLiveDataNodes when dfs.namenode.safemode.min.datanodes is not zero. Contributed by Srinivasu Majeti.

This commit is contained in:
Ayush Saxena 2019-10-08 14:14:14 +05:30
parent 7ab02a67bc
commit a3599c7e70
2 changed files with 97 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
View File

@ -567,12 +567,18 @@ class BlockManagerSafeMode {
assert namesystem.hasWriteLock(); assert namesystem.hasWriteLock();
// Calculating the number of live datanodes is time-consuming // Calculating the number of live datanodes is time-consuming
// in large clusters. Skip it when datanodeThreshold is zero. // in large clusters. Skip it when datanodeThreshold is zero.
int datanodeNum = 0; // We need to evaluate getNumLiveDataNodes only when
if (datanodeThreshold > 0) { // (blockSafe >= blockThreshold) is true and hence moving evaluation
datanodeNum = blockManager.getDatanodeManager().getNumLiveDataNodes(); // of datanodeNum conditional to isBlockThresholdMet as well
}
synchronized (this) { synchronized (this) {
return blockSafe >= blockThreshold && datanodeNum >= datanodeThreshold; boolean isBlockThresholdMet = (blockSafe >= blockThreshold);
boolean isDatanodeThresholdMet = true;
if (isBlockThresholdMet && datanodeThreshold > 0) {
int datanodeNum = blockManager.getDatanodeManager().
getNumLiveDataNodes();
isDatanodeThresholdMet = (datanodeNum >= datanodeThreshold);
}
return isBlockThresholdMet && isDatanodeThresholdMet;
} }
} }

86
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
View File

@ -365,6 +365,87 @@ public class TestBlockManagerSafeMode {
assertFalse(bmSafeMode.isInSafeMode()); assertFalse(bmSafeMode.isInSafeMode());
} }
/**
* Test block manager won't leave safe mode if datanode threshold is not met
* only if datanodeThreshold is configured > 0.
*/
@Test(timeout = 30000)
public void testDatanodeThreshodShouldBeMetOnlyIfConfigured()
throws Exception {
bmSafeMode.activate(BLOCK_TOTAL);
//Blocks received is set to threshold
setBlockSafe(BLOCK_THRESHOLD);
//datanodeThreshold is configured to 1 but not all DNs registered .
// Expecting safe mode .
when(dn.getNumLiveDataNodes()).thenReturn(1);
setDatanodeThreshold(1);
bmSafeMode.checkSafeMode();
assertTrue(bmSafeMode.isInSafeMode());
//datanodeThreshold is configured to 1 and all DNs registered .
// Not expecting safe mode .
when(dn.getNumLiveDataNodes()).thenReturn(DATANODE_NUM);
setDatanodeThreshold(1);
bmSafeMode.checkSafeMode();
waitForExtensionPeriod();
assertFalse(bmSafeMode.isInSafeMode());
//datanodeThreshold is configured to 0 but not all DNs registered .
// Not Expecting safe mode .
bmSafeMode.activate(BLOCK_TOTAL);
setBlockSafe(BLOCK_THRESHOLD);
when(dn.getNumLiveDataNodes()).thenReturn(1);
setDatanodeThreshold(0);
bmSafeMode.checkSafeMode();
assertFalse(bmSafeMode.isInSafeMode());
//datanodeThreshold is configured to 0 and all DNs registered .
// Not Expecting safe mode .
bmSafeMode.activate(BLOCK_TOTAL);
setBlockSafe(BLOCK_THRESHOLD);
when(dn.getNumLiveDataNodes()).thenReturn(DATANODE_NUM);
setDatanodeThreshold(0);
bmSafeMode.checkSafeMode();
assertFalse(bmSafeMode.isInSafeMode());
//Blocks received set to below threshold and all combinations
//of datanodeThreshold should result in safe mode.
//datanodeThreshold is configured to 1 but not all DNs registered .
// Expecting safe mode .
bmSafeMode.activate(BLOCK_TOTAL);
setBlockSafe(BLOCK_THRESHOLD-1);
setSafeModeStatus(BMSafeModeStatus.PENDING_THRESHOLD);
when(dn.getNumLiveDataNodes()).thenReturn(1);
setDatanodeThreshold(1);
bmSafeMode.checkSafeMode();
assertTrue(bmSafeMode.isInSafeMode());
//datanodeThreshold is configured to 1 and all DNs registered .
// Expecting safe mode .
when(dn.getNumLiveDataNodes()).thenReturn(DATANODE_NUM);
setDatanodeThreshold(1);
bmSafeMode.checkSafeMode();
assertTrue(bmSafeMode.isInSafeMode());
//datanodeThreshold is configured to 0 but not all DNs registered .
// Expecting safe mode .
when(dn.getNumLiveDataNodes()).thenReturn(1);
setDatanodeThreshold(0);
bmSafeMode.checkSafeMode();
assertTrue(bmSafeMode.isInSafeMode());
//datanodeThreshold is configured to 0 and all DNs registered .
// Expecting safe mode .
when(dn.getNumLiveDataNodes()).thenReturn(DATANODE_NUM);
setDatanodeThreshold(0);
bmSafeMode.checkSafeMode();
assertTrue(bmSafeMode.isInSafeMode());
}
/** /**
* Test block manager won't leave safe mode if there are blocks with * Test block manager won't leave safe mode if there are blocks with
* generation stamp (GS) in future. * generation stamp (GS) in future.
@ -575,6 +656,11 @@ public class TestBlockManagerSafeMode {
Whitebox.setInternalState(bmSafeMode, "blockSafe", blockSafe); Whitebox.setInternalState(bmSafeMode, "blockSafe", blockSafe);
} }
private void setDatanodeThreshold(int dnSafeModeThreshold) {
Whitebox.setInternalState(bmSafeMode, "datanodeThreshold",
dnSafeModeThreshold);
}
private long getblockSafe() { private long getblockSafe() {
return (long)Whitebox.getInternalState(bmSafeMode, "blockSafe"); return (long)Whitebox.getInternalState(bmSafeMode, "blockSafe");
} }