From b2e641e31f8fe1045c227ba1448616114448d27a Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 16 Oct 2013 18:39:24 +0000 Subject: [PATCH] svn merge -c 1532857 from trunk for HDFS-5283. Under construction blocks only inside snapshots should not be counted in safemode threshhold. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1532858 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../server/blockmanagement/BlockManager.java | 8 +++++ .../hdfs/server/namenode/FSNamesystem.java | 34 +++++++++++++++++++ .../hdfs/server/namenode/Namesystem.java | 3 ++ .../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 +++ 5 files changed, 52 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1ae9b423945..da33967170a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -105,6 +105,9 @@ Release 2.3.0 - UNRELEASED HDFS-5352. Server#initLog() doesn't close InputStream in httpfs. (Ted Yu via jing9) + HDFS-5283. Under construction blocks only inside snapshots should not be + counted in safemode threshhold. (Vinay via szetszwo) + Release 2.2.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 70451c37a0c..9cf7f05db4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1785,6 +1785,14 @@ private void processFirstBlockReport(final DatanodeDescriptor node, if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) { ((BlockInfoUnderConstruction)storedBlock).addReplicaIfNotPresent( node, iblk, reportedState); + // OpenFileBlocks only inside snapshots also will be added to safemode + // threshold. So we need to update such blocks to safemode + // refer HDFS-5283 + BlockInfoUnderConstruction blockUC = (BlockInfoUnderConstruction) storedBlock; + if (namesystem.isInSnapshot(blockUC)) { + int numOfReplicas = blockUC.getNumExpectedLocations(); + namesystem.incrementSafeBlockCount(numOfReplicas); + } //and fall through to next clause } //add replica if appropriate diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 378d4d171a4..0b049ecddfe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -165,6 +165,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; @@ -3700,6 +3701,39 @@ private void finalizeINodeFileUnderConstruction(String src, BlockInfo getStoredBlock(Block block) { return blockManager.getStoredBlock(block); } + + @Override + public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) { + assert hasReadOrWriteLock(); + final BlockCollection bc = blockUC.getBlockCollection(); + if (bc == null || !(bc instanceof INodeFileUnderConstruction)) { + return false; + } + + INodeFileUnderConstruction inodeUC = (INodeFileUnderConstruction) blockUC + .getBlockCollection(); + String fullName = inodeUC.getName(); + try { + if (fullName != null && fullName.startsWith(Path.SEPARATOR) + && dir.getINode(fullName) == inodeUC) { + // If file exists in normal path then no need to look in snapshot + return false; + } + } catch (UnresolvedLinkException e) { + LOG.error("Error while resolving the link : " + fullName, e); + return false; + } + /* + * 1. if bc is an instance of INodeFileUnderConstructionWithSnapshot, and + * bc is not in the current fsdirectory tree, bc must represent a snapshot + * file. + * 2. if fullName is not an absolute path, bc cannot be existent in the + * current fsdirectory tree. + * 3. if bc is not the current node associated with fullName, bc must be a + * snapshot inode. + */ + return true; + } void commitBlockSynchronization(ExtendedBlock lastblock, long newgenerationstamp, long newlength, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java index 9aa0bd5b470..40c4765f91c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java @@ -19,6 +19,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; import org.apache.hadoop.hdfs.util.RwLock; import org.apache.hadoop.ipc.StandbyException; @@ -43,4 +44,6 @@ public interface Namesystem extends RwLock, SafeMode { public void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal); public void checkOperation(OperationCategory read) throws StandbyException; + + public boolean isInSnapshot(BlockInfoUnderConstruction blockUC); } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 1a1b756fcc3..eda3175067a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -1014,4 +1014,8 @@ public static void runOperations(MiniDFSCluster cluster, cluster.getNameNodeRpc(nnIndex), filePath, 0L, bytes.length); } while (locatedBlocks.isUnderConstruction()); } + + public static void abortStream(DFSOutputStream out) throws IOException { + out.abort(); + } }