diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 11d003870e3..12e8bd3331a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -41,6 +41,9 @@ Release 2.6.1 - UNRELEASED HDFS-7263. Snapshot read can reveal future bytes for appended files. (Tao Luo via shv) + HDFS-7225. Remove stale block invalidation work when DN re-registers with + different UUID. (Zhe Zhang and Andrew Wang) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 37df2236c8f..9f8f4758062 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1111,6 +1111,18 @@ public class BlockManager { } } + /** + * Remove all block invalidation tasks under this datanode UUID; + * used when a datanode registers with a new UUID and the old one + * is wiped. + */ + void removeFromInvalidates(final DatanodeInfo datanode) { + if (!namesystem.isPopulatingReplQueues()) { + return; + } + invalidateBlocks.remove(datanode); + } + /** * Mark the block belonging to datanode as corrupt * @param blk Block to be marked as corrupt @@ -3393,7 +3405,14 @@ public class BlockManager { return 0; } try { - toInvalidate = invalidateBlocks.invalidateWork(datanodeManager.getDatanode(dn)); + DatanodeDescriptor dnDescriptor = datanodeManager.getDatanode(dn); + if (dnDescriptor == null) { + LOG.warn("DataNode " + dn + " cannot be found with UUID " + + dn.getDatanodeUuid() + ", removing block invalidation work."); + invalidateBlocks.remove(dn); + return 0; + } + toInvalidate = invalidateBlocks.invalidateWork(dnDescriptor); if (toInvalidate == null) { return 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 6a523498158..80965b94df9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -593,6 +593,8 @@ public class DatanodeManager { synchronized (datanodeMap) { host2DatanodeMap.remove(datanodeMap.remove(key)); } + // Also remove all block invalidation tasks under this node + blockManager.removeFromInvalidates(new DatanodeInfo(node)); if (LOG.isDebugEnabled()) { LOG.debug(getClass().getSimpleName() + ".wipeDatanode(" + node + "): storage " + key diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java index d0edd481567..fecca4e915c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java @@ -17,66 +17,161 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import java.util.UUID; + import static org.junit.Assert.assertEquals; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.common.GenerationStamp; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import org.apache.hadoop.hdfs.server.common.StorageInfo; +import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.util.VersionInfo; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.mockito.internal.util.reflection.Whitebox; /** * Test if FSNamesystem handles heartbeat right */ public class TestComputeInvalidateWork { - /** - * Test if {@link FSNamesystem#computeInvalidateWork(int)} - * can schedule invalidate work correctly - */ - @Test - public void testCompInvalidate() throws Exception { - final Configuration conf = new HdfsConfiguration(); - final int NUM_OF_DATANODES = 3; - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_OF_DATANODES).build(); - try { - cluster.waitActive(); - final FSNamesystem namesystem = cluster.getNamesystem(); - final BlockManager bm = namesystem.getBlockManager(); - final int blockInvalidateLimit = bm.getDatanodeManager().blockInvalidateLimit; - final DatanodeDescriptor[] nodes = bm.getDatanodeManager( - ).getHeartbeatManager().getDatanodes(); - assertEquals(nodes.length, NUM_OF_DATANODES); - - namesystem.writeLock(); - try { - for (int i=0; i