Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-10694. processReport() should print blockReportId in each log message. Contributed by Yuanbo Liu. 

(cherry picked from commit 10e84c6a6e)
This commit is contained in:
Yuanbo Liu 2016-08-10 10:48:42 -07:00 committed by Konstantin V Shvachko
parent 3d401206cf
commit 44162e555a
1 changed files with 24 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -1954,6 +1954,8 @@ public boolean processReport(final DatanodeID nodeID,
final long endTime; final long endTime;
DatanodeDescriptor node; DatanodeDescriptor node;
Collection<Block> invalidatedBlocks = Collections.emptyList(); Collection<Block> invalidatedBlocks = Collections.emptyList();
String strBlockReportId =
context != null ? Long.toHexString(context.getReportId()) : "";
try { try {
node = datanodeManager.getDatanode(nodeID); node = datanodeManager.getDatanode(nodeID);
@ -1972,9 +1974,10 @@ public boolean processReport(final DatanodeID nodeID,
} }
if (namesystem.isInStartupSafeMode() if (namesystem.isInStartupSafeMode()
&& storageInfo.getBlockReportCount() > 0) { && storageInfo.getBlockReportCount() > 0) {
blockLog.info("BLOCK* processReport: " blockLog.info("BLOCK* processReport 0x{}: "
+ "discarded non-initial block report from {}" + "discarded non-initial block report from {}"
+ " because namenode still in startup phase", nodeID); + " because namenode still in startup phase",
strBlockReportId, nodeID);
blockReportLeaseManager.removeLease(node); blockReportLeaseManager.removeLease(node);
return !node.hasStaleStorages(); return !node.hasStaleStorages();
} }
@ -1988,12 +1991,14 @@ public boolean processReport(final DatanodeID nodeID,
if (storageInfo.getBlockReportCount() == 0) { if (storageInfo.getBlockReportCount() == 0) {
// The first block report can be processed a lot more efficiently than // The first block report can be processed a lot more efficiently than
// ordinary block reports. This shortens restart times. // ordinary block reports. This shortens restart times.
LOG.info("Processing first storage report for " + blockLog.info("BLOCK* processReport 0x{}: Processing first "
storageInfo.getStorageID() + " from datanode " + + "storage report for {} from datanode {}",
strBlockReportId,
storageInfo.getStorageID(),
nodeID.getDatanodeUuid()); nodeID.getDatanodeUuid());
processFirstBlockReport(storageInfo, newReport); processFirstBlockReport(storageInfo, newReport);
} else { } else {
invalidatedBlocks = processReport(storageInfo, newReport); invalidatedBlocks = processReport(storageInfo, newReport, context);
} }
storageInfo.receivedBlockReport(); storageInfo.receivedBlockReport();
@ -2029,8 +2034,8 @@ public boolean processReport(final DatanodeID nodeID,
} }
for (Block b : invalidatedBlocks) { for (Block b : invalidatedBlocks) {
blockLog.debug("BLOCK* processReport: {} on node {} size {} does not " + blockLog.debug("BLOCK* processReport 0x{}: {} on node {} size {} does not"
"belong to any file", b, node, b.getNumBytes()); + " belong to any file", strBlockReportId, b, node, b.getNumBytes());
} }
// Log the block report processing stats from Namenode perspective // Log the block report processing stats from Namenode perspective
@ -2038,10 +2043,10 @@ public boolean processReport(final DatanodeID nodeID,
if (metrics != null) { if (metrics != null) {
metrics.addBlockReport((int) (endTime - startTime)); metrics.addBlockReport((int) (endTime - startTime));
} }
blockLog.info("BLOCK* processReport: from storage {} node {}, " + blockLog.info("BLOCK* processReport 0x{}: from storage {} node {}, " +
"blocks: {}, hasStaleStorage: {}, processing time: {} msecs, " + "blocks: {}, hasStaleStorage: {}, processing time: {} msecs, " +
"invalidatedBlocks: {}", storage.getStorageID(), nodeID, "invalidatedBlocks: {}", strBlockReportId, storage.getStorageID(),
newReport.getNumberOfBlocks(), nodeID, newReport.getNumberOfBlocks(),
node.hasStaleStorages(), (endTime - startTime), node.hasStaleStorages(), (endTime - startTime),
invalidatedBlocks.size()); invalidatedBlocks.size());
return !node.hasStaleStorages(); return !node.hasStaleStorages();
@ -2149,7 +2154,8 @@ void rescanPostponedMisreplicatedBlocks() {
private Collection<Block> processReport( private Collection<Block> processReport(
final DatanodeStorageInfo storageInfo, final DatanodeStorageInfo storageInfo,
final BlockListAsLongs report) throws IOException { final BlockListAsLongs report,
BlockReportContext context) throws IOException {
// Normal case: // Normal case:
// Modify the (block-->datanode) map, according to the difference // Modify the (block-->datanode) map, according to the difference
// between the old and new block report. // between the old and new block report.
@ -2161,6 +2167,11 @@ private Collection<Block> processReport(
Collection<StatefulBlockInfo> toUC = new LinkedList<StatefulBlockInfo>(); Collection<StatefulBlockInfo> toUC = new LinkedList<StatefulBlockInfo>();
reportDiff(storageInfo, report, reportDiff(storageInfo, report,
toAdd, toRemove, toInvalidate, toCorrupt, toUC); toAdd, toRemove, toInvalidate, toCorrupt, toUC);
String strBlockReportId = "";
if (context != null) {
strBlockReportId = Long.toHexString(context.getReportId());
}
DatanodeDescriptor node = storageInfo.getDatanodeDescriptor(); DatanodeDescriptor node = storageInfo.getDatanodeDescriptor();
// Process the blocks on each queue // Process the blocks on each queue
@ -2176,8 +2187,8 @@ private Collection<Block> processReport(
numBlocksLogged++; numBlocksLogged++;
} }
if (numBlocksLogged > maxNumBlocksToLog) { if (numBlocksLogged > maxNumBlocksToLog) {
blockLog.info("BLOCK* processReport: logged info for {} of {} " + blockLog.info("BLOCK* processReport 0x{}: logged info for {} of {} " +
"reported.", maxNumBlocksToLog, numBlocksLogged); "reported.", strBlockReportId, maxNumBlocksToLog, numBlocksLogged);
} }
for (Block b : toInvalidate) { for (Block b : toInvalidate) {
addToInvalidates(b, node); addToInvalidates(b, node);