Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-3520. Add transfer rate logging to TransferFsImage. Contributed by Eli Collins

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1349118 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2012-06-12 05:01:50 +00:00
parent abfd7dba32
commit 8e1730243b
2 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -68,6 +68,8 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3052. Change INodeFile and INodeFileUnderConstruction to package HDFS-3052. Change INodeFile and INodeFileUnderConstruction to package
private. (szetszwo) private. (szetszwo)
HDFS-3520. Add transfer rate logging to TransferFsImage. (eli)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-2982. Startup performance suffers when there are many edit log HDFS-2982. Startup performance suffers when there are many edit log

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
View File

@ -32,11 +32,11 @@
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.hdfs.util.DataTransferThrottler;
import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.security.UserGroupInformation;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
@ -207,6 +207,7 @@ static MD5Hash getFileClient(String nnHostPort,
// //
// open connection to remote server // open connection to remote server
// //
long startTime = Util.monotonicNow();
URL url = new URL(str); URL url = new URL(str);
HttpURLConnection connection = (HttpURLConnection) HttpURLConnection connection = (HttpURLConnection)
@ -312,6 +313,11 @@ static MD5Hash getFileClient(String nnHostPort,
advertisedSize); advertisedSize);
} }
} }
double xferSec = Math.max(
((float)(Util.monotonicNow() - startTime)) / 1000.0, 0.001);
long xferKb = received / 1024;
LOG.info(String.format("Transfer took %.2fs at %.2f KB/s",
xferSec, xferKb / xferSec));
if (digester != null) { if (digester != null) {
MD5Hash computedDigest = new MD5Hash(digester.digest()); MD5Hash computedDigest = new MD5Hash(digester.digest());