Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-9601. NNThroughputBenchmark.BlockReportStats should handle NotReplicatedYetException on adding block (iwasakims)

This commit is contained in:
Masatake Iwasaki 2016-01-22 12:28:38 +09:00 committed by Konstantin V Shvachko
parent a79f570b6c
commit 5fd4bd1d8c
2 changed files with 36 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -1713,6 +1713,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9748. Avoid duplication in pendingReplications when
addExpectedReplicasToPending is called twice. (Walter Su via jing9)
HDFS-9601. NNThroughputBenchmark.BlockReportStats should handle
NotReplicatedYetException on adding block (iwasakims)
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES

34
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
View File

@ -68,6 +68,7 @@ import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.security.Groups;
@ -1181,7 +1182,7 @@ public class NNThroughputBenchmark implements Tool {
throws IOException {
ExtendedBlock prevBlock = null;
for(int jdx = 0; jdx < blocksPerFile; jdx++) {
LocatedBlock loc = clientProto.addBlock(fileName, clientName,
LocatedBlock loc = addBlock(fileName, clientName,
prevBlock, null, HdfsConstants.GRANDFATHER_INODE_ID, null);
prevBlock = loc.getBlock();
for(DatanodeInfo dnInfo : loc.getLocations()) {
@ -1195,10 +1196,41 @@ public class NNThroughputBenchmark implements Tool {
dataNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration,
bpid, report);
}
// IBRs are asynchronously processed by NameNode. The next
// ClientProtocol#addBlock() may throw NotReplicatedYetException.
}
return prevBlock;
}
/**
* Retry ClientProtocol.addBlock() if it throws NotReplicatedYetException.
* Because addBlock() also commits the previous block,
* it fails if enough IBRs are not processed by NameNode.
*/
private LocatedBlock addBlock(String src, String clientName,
ExtendedBlock previous, DatanodeInfo[] excludeNodes, long fileId,
String[] favoredNodes) throws IOException {
for (int i = 0; i < 30; i++) {
try {
return clientProto.addBlock(src, clientName,
previous, excludeNodes, fileId, favoredNodes);
} catch (NotReplicatedYetException|RemoteException e) {
if (e instanceof RemoteException) {
String className = ((RemoteException) e).getClassName();
if (!className.equals(NotReplicatedYetException.class.getName())) {
throw e;
}
}
try {
Thread.sleep(100);
} catch (InterruptedException ie) {
LOG.warn("interrupted while retrying addBlock.", ie);
}
}
}
throw new IOException("failed to add block.");
}
/**
* Does not require the argument
*/