svn merge -c 1532857 from trunk for HDFS-5283. Under construction blocks only inside snapshots should not be counted in safemode threshhold.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1532858 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
9590b4b93d
commit
b2e641e31f
|
@ -105,6 +105,9 @@ Release 2.3.0 - UNRELEASED
|
||||||
HDFS-5352. Server#initLog() doesn't close InputStream in httpfs. (Ted Yu via
|
HDFS-5352. Server#initLog() doesn't close InputStream in httpfs. (Ted Yu via
|
||||||
jing9)
|
jing9)
|
||||||
|
|
||||||
|
HDFS-5283. Under construction blocks only inside snapshots should not be
|
||||||
|
counted in safemode threshhold. (Vinay via szetszwo)
|
||||||
|
|
||||||
Release 2.2.1 - UNRELEASED
|
Release 2.2.1 - UNRELEASED
|
||||||
|
|
||||||
INCOMPATIBLE CHANGES
|
INCOMPATIBLE CHANGES
|
||||||
|
|
|
@ -1785,6 +1785,14 @@ public class BlockManager {
|
||||||
if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
|
if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
|
||||||
((BlockInfoUnderConstruction)storedBlock).addReplicaIfNotPresent(
|
((BlockInfoUnderConstruction)storedBlock).addReplicaIfNotPresent(
|
||||||
node, iblk, reportedState);
|
node, iblk, reportedState);
|
||||||
|
// OpenFileBlocks only inside snapshots also will be added to safemode
|
||||||
|
// threshold. So we need to update such blocks to safemode
|
||||||
|
// refer HDFS-5283
|
||||||
|
BlockInfoUnderConstruction blockUC = (BlockInfoUnderConstruction) storedBlock;
|
||||||
|
if (namesystem.isInSnapshot(blockUC)) {
|
||||||
|
int numOfReplicas = blockUC.getNumExpectedLocations();
|
||||||
|
namesystem.incrementSafeBlockCount(numOfReplicas);
|
||||||
|
}
|
||||||
//and fall through to next clause
|
//and fall through to next clause
|
||||||
}
|
}
|
||||||
//add replica if appropriate
|
//add replica if appropriate
|
||||||
|
|
|
@ -165,6 +165,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
|
||||||
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
|
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
|
||||||
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
|
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
|
||||||
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
|
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
|
||||||
|
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
|
||||||
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
|
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
|
||||||
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
|
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
|
||||||
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
|
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
|
||||||
|
@ -3700,6 +3701,39 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
||||||
BlockInfo getStoredBlock(Block block) {
|
BlockInfo getStoredBlock(Block block) {
|
||||||
return blockManager.getStoredBlock(block);
|
return blockManager.getStoredBlock(block);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) {
|
||||||
|
assert hasReadOrWriteLock();
|
||||||
|
final BlockCollection bc = blockUC.getBlockCollection();
|
||||||
|
if (bc == null || !(bc instanceof INodeFileUnderConstruction)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
INodeFileUnderConstruction inodeUC = (INodeFileUnderConstruction) blockUC
|
||||||
|
.getBlockCollection();
|
||||||
|
String fullName = inodeUC.getName();
|
||||||
|
try {
|
||||||
|
if (fullName != null && fullName.startsWith(Path.SEPARATOR)
|
||||||
|
&& dir.getINode(fullName) == inodeUC) {
|
||||||
|
// If file exists in normal path then no need to look in snapshot
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} catch (UnresolvedLinkException e) {
|
||||||
|
LOG.error("Error while resolving the link : " + fullName, e);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
/*
|
||||||
|
* 1. if bc is an instance of INodeFileUnderConstructionWithSnapshot, and
|
||||||
|
* bc is not in the current fsdirectory tree, bc must represent a snapshot
|
||||||
|
* file.
|
||||||
|
* 2. if fullName is not an absolute path, bc cannot be existent in the
|
||||||
|
* current fsdirectory tree.
|
||||||
|
* 3. if bc is not the current node associated with fullName, bc must be a
|
||||||
|
* snapshot inode.
|
||||||
|
*/
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
void commitBlockSynchronization(ExtendedBlock lastblock,
|
void commitBlockSynchronization(ExtendedBlock lastblock,
|
||||||
long newgenerationstamp, long newlength,
|
long newgenerationstamp, long newlength,
|
||||||
|
|
|
@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.hdfs.protocol.Block;
|
import org.apache.hadoop.hdfs.protocol.Block;
|
||||||
|
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
|
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
|
||||||
import org.apache.hadoop.hdfs.util.RwLock;
|
import org.apache.hadoop.hdfs.util.RwLock;
|
||||||
import org.apache.hadoop.ipc.StandbyException;
|
import org.apache.hadoop.ipc.StandbyException;
|
||||||
|
@ -43,4 +44,6 @@ public interface Namesystem extends RwLock, SafeMode {
|
||||||
public void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal);
|
public void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal);
|
||||||
|
|
||||||
public void checkOperation(OperationCategory read) throws StandbyException;
|
public void checkOperation(OperationCategory read) throws StandbyException;
|
||||||
|
|
||||||
|
public boolean isInSnapshot(BlockInfoUnderConstruction blockUC);
|
||||||
}
|
}
|
|
@ -1014,4 +1014,8 @@ public class DFSTestUtil {
|
||||||
cluster.getNameNodeRpc(nnIndex), filePath, 0L, bytes.length);
|
cluster.getNameNodeRpc(nnIndex), filePath, 0L, bytes.length);
|
||||||
} while (locatedBlocks.isUnderConstruction());
|
} while (locatedBlocks.isUnderConstruction());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static void abortStream(DFSOutputStream out) throws IOException {
|
||||||
|
out.abort();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue