diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e9244a4128f..470d78e1ee9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -14,6 +14,9 @@ Release 2.6.1 - UNRELEASED HDFS-7531. Improve the concurrent access on FsVolumeList (Lei Xu via Colin P. McCabe) + HDFS-7579. Improve log reporting during block report rpc failure. + (Charles Lamb via cnauroth) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 29616986259..6bdb68a2d32 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -22,7 +22,10 @@ import static org.apache.hadoop.util.Time.now; import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketTimeoutException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; import com.google.common.base.Joiner; import org.apache.commons.logging.Log; @@ -457,7 +460,7 @@ class BPServiceActor implements Runnable { return null; } - ArrayList cmds = new ArrayList(); + final ArrayList cmds = new ArrayList(); // Flush any block information that precedes the block report. Otherwise // we have a chance that we will miss the delHint information @@ -484,40 +487,54 @@ class BPServiceActor implements Runnable { } // Send the reports to the NN. - int numReportsSent; + int numReportsSent = 0; + int numRPCs = 0; + boolean success = false; long brSendStartTime = now(); - if (totalBlockCount < dnConf.blockReportSplitThreshold) { - // Below split threshold, send all reports in a single message. - numReportsSent = 1; - DatanodeCommand cmd = - bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), reports); - if (cmd != null) { - cmds.add(cmd); - } - } else { - // Send one block report per message. - numReportsSent = i; - for (StorageBlockReport report : reports) { - StorageBlockReport singleReport[] = { report }; + try { + if (totalBlockCount < dnConf.blockReportSplitThreshold) { + // Below split threshold, send all reports in a single message. DatanodeCommand cmd = bpNamenode.blockReport( - bpRegistration, bpos.getBlockPoolId(), singleReport); + bpRegistration, bpos.getBlockPoolId(), reports); + numRPCs = 1; + numReportsSent = reports.length; if (cmd != null) { cmds.add(cmd); } + } else { + // Send one block report per message. + for (StorageBlockReport report : reports) { + StorageBlockReport singleReport[] = { report }; + DatanodeCommand cmd = bpNamenode.blockReport( + bpRegistration, bpos.getBlockPoolId(), singleReport); + numReportsSent++; + numRPCs++; + if (cmd != null) { + cmds.add(cmd); + } + } } + success = true; + } finally { + // Log the block report processing stats from Datanode perspective + long brSendCost = now() - brSendStartTime; + long brCreateCost = brSendStartTime - brCreateStartTime; + dn.getMetrics().addBlockReport(brSendCost); + final int nCmds = cmds.size(); + LOG.info((success ? "S" : "Uns") + + "uccessfully sent " + numReportsSent + + " of " + reports.length + + " blockreports for " + totalBlockCount + + " total blocks using " + numRPCs + + " RPCs. This took " + brCreateCost + + " msec to generate and " + brSendCost + + " msecs for RPC and NN processing." + + " Got back " + + ((nCmds == 0) ? "no commands" : + ((nCmds == 1) ? "one command: " + cmds.get(0) : + (nCmds + " commands: " + Joiner.on("; ").join(cmds)))) + + "."); } - - // Log the block report processing stats from Datanode perspective - long brSendCost = now() - brSendStartTime; - long brCreateCost = brSendStartTime - brCreateStartTime; - dn.getMetrics().addBlockReport(brSendCost); - LOG.info("Sent " + numReportsSent + " blockreports " + totalBlockCount + - " blocks total. Took " + brCreateCost + - " msec to generate and " + brSendCost + - " msecs for RPC and NN processing. " + - " Got back commands " + - (cmds.size() == 0 ? "none" : Joiner.on("; ").join(cmds))); - scheduleNextBlockReport(startTime); return cmds.size() == 0 ? null : cmds; } @@ -967,7 +984,6 @@ class BPServiceActor implements Runnable { /** * Add pending incremental block report for a single block. - * @param blockID * @param blockInfo */ void putBlockInfo(ReceivedDeletedBlockInfo blockInfo) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java index 8d7544d768b..eed9a6e3801 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java @@ -52,4 +52,12 @@ public abstract class ServerCommand { public int getAction() { return this.action; } + + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append("/"); + sb.append(action); + return sb.toString(); + } }