From ffe347151e0af39eb3e6a50863c74e35f52c52aa Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 16 Oct 2013 18:38:31 +0000 Subject: [PATCH] HDFS-5283. Under construction blocks only inside snapshots should not be counted in safemode threshhold. Contributed by Vinay git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1532857 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../server/blockmanagement/BlockManager.java | 8 +++++ .../hdfs/server/namenode/FSNamesystem.java | 34 +++++++++++++++++++ .../hdfs/server/namenode/Namesystem.java | 3 ++ .../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 +++ 5 files changed, 52 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ac4814ed2c2..589cb5b15d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -349,6 +349,9 @@ Release 2.3.0 - UNRELEASED HDFS-5352. Server#initLog() doesn't close InputStream in httpfs. (Ted Yu via jing9) + HDFS-5283. Under construction blocks only inside snapshots should not be + counted in safemode threshhold. (Vinay via szetszwo) + Release 2.2.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 2d46ad3f6e7..4be05270eed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1780,6 +1780,14 @@ public class BlockManager { if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) { ((BlockInfoUnderConstruction)storedBlock).addReplicaIfNotPresent( node, iblk, reportedState); + // OpenFileBlocks only inside snapshots also will be added to safemode + // threshold. So we need to update such blocks to safemode + // refer HDFS-5283 + BlockInfoUnderConstruction blockUC = (BlockInfoUnderConstruction) storedBlock; + if (namesystem.isInSnapshot(blockUC)) { + int numOfReplicas = blockUC.getNumExpectedLocations(); + namesystem.incrementSafeBlockCount(numOfReplicas); + } //and fall through to next clause } //add replica if appropriate diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index d4d3ae5c5ae..febcdae39f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -165,6 +165,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; @@ -3717,6 +3718,39 @@ public class FSNamesystem implements Namesystem, FSClusterStats, BlockInfo getStoredBlock(Block block) { return blockManager.getStoredBlock(block); } + + @Override + public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) { + assert hasReadOrWriteLock(); + final BlockCollection bc = blockUC.getBlockCollection(); + if (bc == null || !(bc instanceof INodeFileUnderConstruction)) { + return false; + } + + INodeFileUnderConstruction inodeUC = (INodeFileUnderConstruction) blockUC + .getBlockCollection(); + String fullName = inodeUC.getName(); + try { + if (fullName != null && fullName.startsWith(Path.SEPARATOR) + && dir.getINode(fullName) == inodeUC) { + // If file exists in normal path then no need to look in snapshot + return false; + } + } catch (UnresolvedLinkException e) { + LOG.error("Error while resolving the link : " + fullName, e); + return false; + } + /* + * 1. if bc is an instance of INodeFileUnderConstructionWithSnapshot, and + * bc is not in the current fsdirectory tree, bc must represent a snapshot + * file. + * 2. if fullName is not an absolute path, bc cannot be existent in the + * current fsdirectory tree. + * 3. if bc is not the current node associated with fullName, bc must be a + * snapshot inode. + */ + return true; + } void commitBlockSynchronization(ExtendedBlock lastblock, long newgenerationstamp, long newlength, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java index 9aa0bd5b470..40c4765f91c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; import org.apache.hadoop.hdfs.util.RwLock; import org.apache.hadoop.ipc.StandbyException; @@ -43,4 +44,6 @@ public interface Namesystem extends RwLock, SafeMode { public void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal); public void checkOperation(OperationCategory read) throws StandbyException; + + public boolean isInSnapshot(BlockInfoUnderConstruction blockUC); } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 5f770023ca0..ce6721a88ce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -994,4 +994,8 @@ public class DFSTestUtil { cluster.getNameNodeRpc(nnIndex), filePath, 0L, bytes.length); } while (locatedBlocks.isUnderConstruction()); } + + public static void abortStream(DFSOutputStream out) throws IOException { + out.abort(); + } }