Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-9601. NNThroughputBenchmark.BlockReportStats should handle NotReplicatedYetException on adding block (iwasakims)

This commit is contained in:
Masatake Iwasaki 2016-01-22 12:28:38 +09:00
parent 4d9efaf3b0
commit 245c3728ef
2 changed files with 36 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -54,6 +54,9 @@ Release 2.9.0 - UNRELEASED
BUG FIXES BUG FIXES
HDFS-9601. NNThroughputBenchmark.BlockReportStats should handle
NotReplicatedYetException on adding block (iwasakims)
HDFS-9621. Consolidate FSDirStatAndListingOp#createFileStatus to let its HDFS-9621. Consolidate FSDirStatAndListingOp#createFileStatus to let its
INodesInPath parameter always include the target INode. (jing9) INodesInPath parameter always include the target INode. (jing9)

34
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
View File

@ -68,6 +68,7 @@ import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.DNS; import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.Groups;
@ -1181,7 +1182,7 @@ public class NNThroughputBenchmark implements Tool {
throws IOException { throws IOException {
ExtendedBlock prevBlock = null; ExtendedBlock prevBlock = null;
for(int jdx = 0; jdx < blocksPerFile; jdx++) { for(int jdx = 0; jdx < blocksPerFile; jdx++) {
LocatedBlock loc = clientProto.addBlock(fileName, clientName, LocatedBlock loc = addBlock(fileName, clientName,
prevBlock, null, HdfsConstants.GRANDFATHER_INODE_ID, null); prevBlock, null, HdfsConstants.GRANDFATHER_INODE_ID, null);
prevBlock = loc.getBlock(); prevBlock = loc.getBlock();
for(DatanodeInfo dnInfo : loc.getLocations()) { for(DatanodeInfo dnInfo : loc.getLocations()) {
@ -1195,10 +1196,41 @@ public class NNThroughputBenchmark implements Tool {
dataNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration, dataNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration,
bpid, report); bpid, report);
} }
// IBRs are asynchronously processed by NameNode. The next
// ClientProtocol#addBlock() may throw NotReplicatedYetException.
} }
return prevBlock; return prevBlock;
} }
/**
* Retry ClientProtocol.addBlock() if it throws NotReplicatedYetException.
* Because addBlock() also commits the previous block,
* it fails if enough IBRs are not processed by NameNode.
*/
private LocatedBlock addBlock(String src, String clientName,
ExtendedBlock previous, DatanodeInfo[] excludeNodes, long fileId,
String[] favoredNodes) throws IOException {
for (int i = 0; i < 30; i++) {
try {
return clientProto.addBlock(src, clientName,
previous, excludeNodes, fileId, favoredNodes);
} catch (NotReplicatedYetException|RemoteException e) {
if (e instanceof RemoteException) {
String className = ((RemoteException) e).getClassName();
if (!className.equals(NotReplicatedYetException.class.getName())) {
throw e;
}
}
try {
Thread.sleep(100);
} catch (InterruptedException ie) {
LOG.warn("interrupted while retrying addBlock.", ie);
}
}
}
throw new IOException("failed to add block.");
}
/** /**
* Does not require the argument * Does not require the argument
*/ */