HDFS-7579. Improve log reporting during block report rpc failure. Contributed by Charles Lamb.

(cherry picked from commit 7e2d9a3242)
(cherry picked from commit f0acb7c2a2)
(cherry picked from commit 33fb7b45195a89b6464e4a1cb3fbf6bbad2bcecb)
This commit is contained in:
cnauroth 2015-01-08 15:12:56 -08:00 committed by Vinod Kumar Vavilapalli
parent 3f8da2a9eb
commit e6b52fc3e9
3 changed files with 57 additions and 30 deletions

View File

@ -14,6 +14,9 @@ Release 2.6.1 - UNRELEASED
HDFS-7531. Improve the concurrent access on FsVolumeList (Lei Xu via Colin HDFS-7531. Improve the concurrent access on FsVolumeList (Lei Xu via Colin
P. McCabe) P. McCabe)
HDFS-7579. Improve log reporting during block report rpc failure.
(Charles Lamb via cnauroth)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

View File

@ -22,7 +22,10 @@ import static org.apache.hadoop.util.Time.now;
import java.io.IOException; import java.io.IOException;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import java.net.SocketTimeoutException; import java.net.SocketTimeoutException;
import java.util.*; import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
@ -457,7 +460,7 @@ class BPServiceActor implements Runnable {
return null; return null;
} }
ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>(); final ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>();
// Flush any block information that precedes the block report. Otherwise // Flush any block information that precedes the block report. Otherwise
// we have a chance that we will miss the delHint information // we have a chance that we will miss the delHint information
@ -484,40 +487,54 @@ class BPServiceActor implements Runnable {
} }
// Send the reports to the NN. // Send the reports to the NN.
int numReportsSent; int numReportsSent = 0;
int numRPCs = 0;
boolean success = false;
long brSendStartTime = now(); long brSendStartTime = now();
try {
if (totalBlockCount < dnConf.blockReportSplitThreshold) { if (totalBlockCount < dnConf.blockReportSplitThreshold) {
// Below split threshold, send all reports in a single message. // Below split threshold, send all reports in a single message.
numReportsSent = 1; DatanodeCommand cmd = bpNamenode.blockReport(
DatanodeCommand cmd = bpRegistration, bpos.getBlockPoolId(), reports);
bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), reports); numRPCs = 1;
numReportsSent = reports.length;
if (cmd != null) { if (cmd != null) {
cmds.add(cmd); cmds.add(cmd);
} }
} else { } else {
// Send one block report per message. // Send one block report per message.
numReportsSent = i;
for (StorageBlockReport report : reports) { for (StorageBlockReport report : reports) {
StorageBlockReport singleReport[] = { report }; StorageBlockReport singleReport[] = { report };
DatanodeCommand cmd = bpNamenode.blockReport( DatanodeCommand cmd = bpNamenode.blockReport(
bpRegistration, bpos.getBlockPoolId(), singleReport); bpRegistration, bpos.getBlockPoolId(), singleReport);
numReportsSent++;
numRPCs++;
if (cmd != null) { if (cmd != null) {
cmds.add(cmd); cmds.add(cmd);
} }
} }
} }
success = true;
} finally {
// Log the block report processing stats from Datanode perspective // Log the block report processing stats from Datanode perspective
long brSendCost = now() - brSendStartTime; long brSendCost = now() - brSendStartTime;
long brCreateCost = brSendStartTime - brCreateStartTime; long brCreateCost = brSendStartTime - brCreateStartTime;
dn.getMetrics().addBlockReport(brSendCost); dn.getMetrics().addBlockReport(brSendCost);
LOG.info("Sent " + numReportsSent + " blockreports " + totalBlockCount + final int nCmds = cmds.size();
" blocks total. Took " + brCreateCost + LOG.info((success ? "S" : "Uns") +
"uccessfully sent " + numReportsSent +
" of " + reports.length +
" blockreports for " + totalBlockCount +
" total blocks using " + numRPCs +
" RPCs. This took " + brCreateCost +
" msec to generate and " + brSendCost + " msec to generate and " + brSendCost +
" msecs for RPC and NN processing." + " msecs for RPC and NN processing." +
" Got back commands " + " Got back " +
(cmds.size() == 0 ? "none" : Joiner.on("; ").join(cmds))); ((nCmds == 0) ? "no commands" :
((nCmds == 1) ? "one command: " + cmds.get(0) :
(nCmds + " commands: " + Joiner.on("; ").join(cmds)))) +
".");
}
scheduleNextBlockReport(startTime); scheduleNextBlockReport(startTime);
return cmds.size() == 0 ? null : cmds; return cmds.size() == 0 ? null : cmds;
} }
@ -967,7 +984,6 @@ class BPServiceActor implements Runnable {
/** /**
* Add pending incremental block report for a single block. * Add pending incremental block report for a single block.
* @param blockID
* @param blockInfo * @param blockInfo
*/ */
void putBlockInfo(ReceivedDeletedBlockInfo blockInfo) { void putBlockInfo(ReceivedDeletedBlockInfo blockInfo) {

View File

@ -52,4 +52,12 @@ public abstract class ServerCommand {
public int getAction() { public int getAction() {
return this.action; return this.action;
} }
public String toString() {
final StringBuilder sb = new StringBuilder();
sb.append(getClass().getSimpleName());
sb.append("/");
sb.append(action);
return sb.toString();
}
} }