HDFS-7579. Improve log reporting during block report rpc failure. Contributed by Charles Lamb.
(cherry picked from commit7e2d9a3242
) (cherry picked from commitf0acb7c2a2
) (cherry picked from commit 33fb7b45195a89b6464e4a1cb3fbf6bbad2bcecb)
This commit is contained in:
parent
3f8da2a9eb
commit
e6b52fc3e9
|
@ -14,6 +14,9 @@ Release 2.6.1 - UNRELEASED
|
||||||
HDFS-7531. Improve the concurrent access on FsVolumeList (Lei Xu via Colin
|
HDFS-7531. Improve the concurrent access on FsVolumeList (Lei Xu via Colin
|
||||||
P. McCabe)
|
P. McCabe)
|
||||||
|
|
||||||
|
HDFS-7579. Improve log reporting during block report rpc failure.
|
||||||
|
(Charles Lamb via cnauroth)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
BUG FIXES
|
BUG FIXES
|
||||||
|
|
|
@ -22,7 +22,10 @@ import static org.apache.hadoop.util.Time.now;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.InetSocketAddress;
|
import java.net.InetSocketAddress;
|
||||||
import java.net.SocketTimeoutException;
|
import java.net.SocketTimeoutException;
|
||||||
import java.util.*;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import com.google.common.base.Joiner;
|
import com.google.common.base.Joiner;
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
|
@ -457,7 +460,7 @@ class BPServiceActor implements Runnable {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>();
|
final ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>();
|
||||||
|
|
||||||
// Flush any block information that precedes the block report. Otherwise
|
// Flush any block information that precedes the block report. Otherwise
|
||||||
// we have a chance that we will miss the delHint information
|
// we have a chance that we will miss the delHint information
|
||||||
|
@ -484,40 +487,54 @@ class BPServiceActor implements Runnable {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send the reports to the NN.
|
// Send the reports to the NN.
|
||||||
int numReportsSent;
|
int numReportsSent = 0;
|
||||||
|
int numRPCs = 0;
|
||||||
|
boolean success = false;
|
||||||
long brSendStartTime = now();
|
long brSendStartTime = now();
|
||||||
|
try {
|
||||||
if (totalBlockCount < dnConf.blockReportSplitThreshold) {
|
if (totalBlockCount < dnConf.blockReportSplitThreshold) {
|
||||||
// Below split threshold, send all reports in a single message.
|
// Below split threshold, send all reports in a single message.
|
||||||
numReportsSent = 1;
|
DatanodeCommand cmd = bpNamenode.blockReport(
|
||||||
DatanodeCommand cmd =
|
bpRegistration, bpos.getBlockPoolId(), reports);
|
||||||
bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), reports);
|
numRPCs = 1;
|
||||||
|
numReportsSent = reports.length;
|
||||||
if (cmd != null) {
|
if (cmd != null) {
|
||||||
cmds.add(cmd);
|
cmds.add(cmd);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Send one block report per message.
|
// Send one block report per message.
|
||||||
numReportsSent = i;
|
|
||||||
for (StorageBlockReport report : reports) {
|
for (StorageBlockReport report : reports) {
|
||||||
StorageBlockReport singleReport[] = { report };
|
StorageBlockReport singleReport[] = { report };
|
||||||
DatanodeCommand cmd = bpNamenode.blockReport(
|
DatanodeCommand cmd = bpNamenode.blockReport(
|
||||||
bpRegistration, bpos.getBlockPoolId(), singleReport);
|
bpRegistration, bpos.getBlockPoolId(), singleReport);
|
||||||
|
numReportsSent++;
|
||||||
|
numRPCs++;
|
||||||
if (cmd != null) {
|
if (cmd != null) {
|
||||||
cmds.add(cmd);
|
cmds.add(cmd);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
success = true;
|
||||||
|
} finally {
|
||||||
// Log the block report processing stats from Datanode perspective
|
// Log the block report processing stats from Datanode perspective
|
||||||
long brSendCost = now() - brSendStartTime;
|
long brSendCost = now() - brSendStartTime;
|
||||||
long brCreateCost = brSendStartTime - brCreateStartTime;
|
long brCreateCost = brSendStartTime - brCreateStartTime;
|
||||||
dn.getMetrics().addBlockReport(brSendCost);
|
dn.getMetrics().addBlockReport(brSendCost);
|
||||||
LOG.info("Sent " + numReportsSent + " blockreports " + totalBlockCount +
|
final int nCmds = cmds.size();
|
||||||
" blocks total. Took " + brCreateCost +
|
LOG.info((success ? "S" : "Uns") +
|
||||||
|
"uccessfully sent " + numReportsSent +
|
||||||
|
" of " + reports.length +
|
||||||
|
" blockreports for " + totalBlockCount +
|
||||||
|
" total blocks using " + numRPCs +
|
||||||
|
" RPCs. This took " + brCreateCost +
|
||||||
" msec to generate and " + brSendCost +
|
" msec to generate and " + brSendCost +
|
||||||
" msecs for RPC and NN processing." +
|
" msecs for RPC and NN processing." +
|
||||||
" Got back commands " +
|
" Got back " +
|
||||||
(cmds.size() == 0 ? "none" : Joiner.on("; ").join(cmds)));
|
((nCmds == 0) ? "no commands" :
|
||||||
|
((nCmds == 1) ? "one command: " + cmds.get(0) :
|
||||||
|
(nCmds + " commands: " + Joiner.on("; ").join(cmds)))) +
|
||||||
|
".");
|
||||||
|
}
|
||||||
scheduleNextBlockReport(startTime);
|
scheduleNextBlockReport(startTime);
|
||||||
return cmds.size() == 0 ? null : cmds;
|
return cmds.size() == 0 ? null : cmds;
|
||||||
}
|
}
|
||||||
|
@ -967,7 +984,6 @@ class BPServiceActor implements Runnable {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Add pending incremental block report for a single block.
|
* Add pending incremental block report for a single block.
|
||||||
* @param blockID
|
|
||||||
* @param blockInfo
|
* @param blockInfo
|
||||||
*/
|
*/
|
||||||
void putBlockInfo(ReceivedDeletedBlockInfo blockInfo) {
|
void putBlockInfo(ReceivedDeletedBlockInfo blockInfo) {
|
||||||
|
|
|
@ -52,4 +52,12 @@ public abstract class ServerCommand {
|
||||||
public int getAction() {
|
public int getAction() {
|
||||||
return this.action;
|
return this.action;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String toString() {
|
||||||
|
final StringBuilder sb = new StringBuilder();
|
||||||
|
sb.append(getClass().getSimpleName());
|
||||||
|
sb.append("/");
|
||||||
|
sb.append(action);
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue