Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-5077. NPE in FSNamesystem.commitBlockSynchronization(). Contributed by Plamen Jeliazkov. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1518851 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Konstantin Shvachko 2013-08-29 23:08:02 +00:00
parent eef32121d1
commit eb484bb562
3 changed files with 39 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -401,6 +401,9 @@ Release 2.1.1-beta - UNRELEASED
HDFS-5132. Deadlock in NameNode between SafeModeMonitor#run and HDFS-5132. Deadlock in NameNode between SafeModeMonitor#run and
DatanodeManager#handleHeartbeat. (kihwal) DatanodeManager#handleHeartbeat. (kihwal)
HDFS-5077. NPE in FSNamesystem.commitBlockSynchronization().
(Plamen Jeliazkov via shv)
Release 2.1.0-beta - 2013-08-22 Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

27
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -174,7 +174,6 @@ import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream; import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease; import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
@ -3772,24 +3771,32 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
// find the DatanodeDescriptor objects // find the DatanodeDescriptor objects
// There should be no locations in the blockManager till now because the // There should be no locations in the blockManager till now because the
// file is underConstruction // file is underConstruction
DatanodeDescriptor[] descriptors = null; List<DatanodeDescriptor> targetList =
new ArrayList<DatanodeDescriptor>(newtargets.length);
if (newtargets.length > 0) { if (newtargets.length > 0) {
descriptors = new DatanodeDescriptor[newtargets.length]; for (DatanodeID newtarget : newtargets) {
for(int i = 0; i < newtargets.length; i++) { // try to get targetNode
descriptors[i] = blockManager.getDatanodeManager().getDatanode( DatanodeDescriptor targetNode =
newtargets[i]); blockManager.getDatanodeManager().getDatanode(newtarget);
if (targetNode != null)
targetList.add(targetNode);
else if (LOG.isDebugEnabled()) {
LOG.debug("DatanodeDescriptor (=" + newtarget + ") not found");
} }
} }
if ((closeFile) && (descriptors != null)) { }
if ((closeFile) && !targetList.isEmpty()) {
// the file is getting closed. Insert block locations into blockManager. // the file is getting closed. Insert block locations into blockManager.
// Otherwise fsck will report these blocks as MISSING, especially if the // Otherwise fsck will report these blocks as MISSING, especially if the
// blocksReceived from Datanodes take a long time to arrive. // blocksReceived from Datanodes take a long time to arrive.
for (int i = 0; i < descriptors.length; i++) { for (DatanodeDescriptor targetNode : targetList) {
descriptors[i].addBlock(storedBlock); targetNode.addBlock(storedBlock);
} }
} }
// add pipeline locations into the INodeUnderConstruction // add pipeline locations into the INodeUnderConstruction
pendingFile.setLastBlock(storedBlock, descriptors); DatanodeDescriptor[] targetArray =
new DatanodeDescriptor[targetList.size()];
pendingFile.setLastBlock(storedBlock, targetList.toArray(targetArray));
} }
if (closeFile) { if (closeFile) {

19
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
View File

@ -169,4 +169,23 @@ public class TestCommitBlockSynchronization {
namesystemSpy.commitBlockSynchronization( namesystemSpy.commitBlockSynchronization(
lastBlock, genStamp, length, true, false, newTargets, null); lastBlock, genStamp, length, true, false, newTargets, null);
} }
@Test
public void testCommitBlockSynchronizationWithCloseAndNonExistantTarget()
throws IOException {
INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class);
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = makeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[]{
new DatanodeID("0.0.0.0", "nonexistantHost", "1", 0, 0, 0)};
ExtendedBlock lastBlock = new ExtendedBlock();
namesystemSpy.commitBlockSynchronization(
lastBlock, genStamp, length, true,
false, newTargets, null);
// Repeat the call to make sure it returns true
namesystemSpy.commitBlockSynchronization(
lastBlock, genStamp, length, true, false, newTargets, null);
}
} }