From 36c5fe9961a905385282d1a05ced08c83684dd02 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 3 Jan 2013 05:12:40 +0000 Subject: [PATCH] HDFS-4346. Add SequentialNumber as a base class for INodeId and GenerationStamp. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1428167 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/server/common/GenerationStamp.java | 65 ++---------------- .../hdfs/server/namenode/FSNamesystem.java | 31 +++++---- .../hadoop/hdfs/server/namenode/INodeId.java | 67 +------------------ .../server/blockmanagement/TestBlockInfo.java | 2 +- .../TestComputeInvalidateWork.java | 2 +- .../TestDatanodeDescriptor.java | 2 +- .../TestHeartbeatHandling.java | 5 +- .../hdfs/server/namenode/CreateEditsLog.java | 2 +- 9 files changed, 36 insertions(+), 143 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index dc7e02c7b21..862c9f6c75d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -174,6 +174,9 @@ Trunk (Unreleased) HDFS-4334. Add a unique id to INode. (Brandon Li via szetszwo) + HDFS-4346. Add SequentialNumber as a base class for INodeId and + GenerationStamp. (szetszwo) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/GenerationStamp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/GenerationStamp.java index 55a622a4ed1..d09c446742f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/GenerationStamp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/GenerationStamp.java @@ -17,19 +17,18 @@ */ package org.apache.hadoop.hdfs.server.common; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.util.SequentialNumber; /**************************************************************** * A GenerationStamp is a Hadoop FS primitive, identified by a long. ****************************************************************/ @InterfaceAudience.Private -public class GenerationStamp implements Comparable { +public class GenerationStamp extends SequentialNumber { /** - * The first valid generation stamp. + * The last reserved generation stamp. */ - public static final long FIRST_VALID_STAMP = 1000L; + public static final long LAST_RESERVED_STAMP = 1000L; /** * Generation stamp of blocks that pre-date the introduction @@ -37,62 +36,10 @@ public class GenerationStamp implements Comparable { */ public static final long GRANDFATHER_GENERATION_STAMP = 0; - private AtomicLong genstamp = new AtomicLong(); - /** - * Create a new instance, initialized to FIRST_VALID_STAMP. + * Create a new instance, initialized to {@link #LAST_RESERVED_STAMP}. */ public GenerationStamp() { - this(GenerationStamp.FIRST_VALID_STAMP); - } - - /** - * Create a new instance, initialized to the specified value. - */ - GenerationStamp(long stamp) { - genstamp.set(stamp); - } - - /** - * Returns the current generation stamp - */ - public long getStamp() { - return genstamp.get(); - } - - /** - * Sets the current generation stamp - */ - public void setStamp(long stamp) { - genstamp.set(stamp); - } - - /** - * First increments the counter and then returns the stamp - */ - public long nextStamp() { - return genstamp.incrementAndGet(); - } - - @Override // Comparable - public int compareTo(GenerationStamp that) { - long stamp1 = this.genstamp.get(); - long stamp2 = that.genstamp.get(); - return stamp1 < stamp2 ? -1 : - stamp1 > stamp2 ? 1 : 0; - } - - @Override // Object - public boolean equals(Object o) { - if (!(o instanceof GenerationStamp)) { - return false; - } - return compareTo((GenerationStamp)o) == 0; - } - - @Override // Object - public int hashCode() { - long stamp = genstamp.get(); - return (int) (stamp^(stamp>>>32)); + super(LAST_RESERVED_STAMP); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 54bef0ac837..a9ef8c78fae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -378,25 +378,29 @@ private void logAuditEvent(boolean succeeded, private INodeId inodeId; /** - * Set the last allocated inode id when fsimage is loaded or editlog is - * applied. - * @throws IOException + * Set the last allocated inode id when fsimage or editlog is loaded. */ public void resetLastInodeId(long newValue) throws IOException { - inodeId.resetLastInodeId(newValue); + try { + inodeId.skipTo(newValue); + } catch(IllegalStateException ise) { + throw new IOException(ise); + } } /** Should only be used for tests to reset to any value */ void resetLastInodeIdWithoutChecking(long newValue) { - inodeId.resetLastInodeIdWithoutChecking(newValue); + inodeId.setCurrentValue(newValue); } + /** @return the last inode ID. */ public long getLastInodeId() { - return inodeId.getLastInodeId(); + return inodeId.getCurrentValue(); } + /** Allocate a new inode ID. */ public long allocateNewInodeId() { - return inodeId.allocateNewInodeId(); + return inodeId.nextValue(); } /** @@ -405,9 +409,9 @@ public long allocateNewInodeId() { void clear() { dir.reset(); dtSecretManager.reset(); - generationStamp.setStamp(GenerationStamp.FIRST_VALID_STAMP); + generationStamp.setCurrentValue(GenerationStamp.LAST_RESERVED_STAMP); leaseManager.removeAllLeases(); - inodeId.resetLastInodeIdWithoutChecking(INodeId.LAST_RESERVED_ID); + inodeId.setCurrentValue(INodeId.LAST_RESERVED_ID); } @VisibleForTesting @@ -2537,8 +2541,7 @@ private Block allocateBlock(String src, INodesInPath inodesInPath, b.setBlockId(DFSUtil.getRandom().nextLong()); } // Increment the generation stamp for every new block. - nextGenerationStamp(); - b.setGenerationStamp(getGenerationStamp()); + b.setGenerationStamp(nextGenerationStamp()); b = dir.addBlock(src, inodesInPath, b, targets); NameNode.stateChangeLog.info("BLOCK* allocateBlock: " + src + ". " + blockPoolId + " " + b); @@ -4762,14 +4765,14 @@ public int getNumStaleDataNodes() { * Sets the generation stamp for this filesystem */ void setGenerationStamp(long stamp) { - generationStamp.setStamp(stamp); + generationStamp.setCurrentValue(stamp); } /** * Gets the generation stamp for this filesystem */ long getGenerationStamp() { - return generationStamp.getStamp(); + return generationStamp.getCurrentValue(); } /** @@ -4781,7 +4784,7 @@ private long nextGenerationStamp() throws SafeModeException { throw new SafeModeException( "Cannot get next generation stamp", safeMode); } - long gs = generationStamp.nextStamp(); + final long gs = generationStamp.nextValue(); getEditLog().logGenerationStamp(gs); // NB: callers sync the log return gs; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java index e6f682d0bc4..11157b2eb3d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java @@ -17,16 +17,14 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.io.IOException; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.util.SequentialNumber; /** * An id which uniquely identifies an inode */ @InterfaceAudience.Private -class INodeId implements Comparable { +class INodeId extends SequentialNumber { /** * The last reserved inode id. Reserve id 1 to 1000 for potential future * usage. The id won't be recycled and is not expected to wrap around in a @@ -40,66 +38,7 @@ class INodeId implements Comparable { */ public static final long GRANDFATHER_INODE_ID = 0; - private AtomicLong lastInodeId = new AtomicLong(); - - /** - * Create a new instance, initialized to LAST_RESERVED_ID. - */ INodeId() { - lastInodeId.set(INodeId.LAST_RESERVED_ID); - } - - /** - * Set the last allocated inode id when fsimage is loaded or editlog is - * applied. - * @throws IOException - */ - void resetLastInodeId(long newValue) throws IOException { - if (newValue < getLastInodeId()) { - throw new IOException( - "Can't reset lastInodeId to be less than its current value " - + getLastInodeId() + ", newValue=" + newValue); - } - - lastInodeId.set(newValue); - } - - void resetLastInodeIdWithoutChecking(long newValue) { - lastInodeId.set(newValue); - } - - long getLastInodeId() { - return lastInodeId.get(); - } - - /** - * First increment the counter and then get the id. - */ - long allocateNewInodeId() { - return lastInodeId.incrementAndGet(); - } - - @Override - // Comparable - public int compareTo(INodeId that) { - long id1 = this.getLastInodeId(); - long id2 = that.getLastInodeId(); - return id1 < id2 ? -1 : id1 > id2 ? 1 : 0; - } - - @Override - // Object - public boolean equals(Object o) { - if (!(o instanceof INodeId)) { - return false; - } - return compareTo((INodeId) o) == 0; - } - - @Override - // Object - public int hashCode() { - long id = getLastInodeId(); - return (int) (id ^ (id >>> 32)); + super(LAST_RESERVED_ID); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java index b8eb6855fbb..8b7111f010e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java @@ -56,7 +56,7 @@ public void testBlockListMoveToHead() throws Exception { LOG.info("Building block list..."); for (int i = 0; i < MAX_BLOCKS; i++) { - blockList.add(new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP)); + blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP)); blockInfoList.add(new BlockInfo(blockList.get(i), 3)); dd.addBlock(blockInfoList.get(i)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java index b22383469f6..d0edd481567 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java @@ -54,7 +54,7 @@ public void testCompInvalidate() throws Exception { for (int i=0; i blockList = new ArrayList(MAX_BLOCKS); for (int i=0; i blockList = new ArrayList(MAX_INVALIDATE_BLOCKS); for (int i=0; i