From b7b9d74521e7b65e62222f9b965f01d709ba60a4 Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Tue, 28 Mar 2017 09:13:44 +0900 Subject: [PATCH] HDFS-11486. Client close() should not fail fast if the last block is being decommissioned. Contributed by Wei-Chiu Chuang and Yiqun Lin. (cherry picked from commit 64ea62c1ccc05d9b0a0030beafa60ddd31c38952) --- .../apache/hadoop/hdfs/TestDecommission.java | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java index b34f04784ae..ebfa13afcd3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java @@ -695,6 +695,56 @@ public class TestDecommission extends AdminStatesBaseTest { assertEquals(dfs.getFileStatus(file).getLen(), writtenBytes); } + + @Test(timeout=120000) + public void testCloseWhileDecommission() throws IOException, + ExecutionException, InterruptedException { + LOG.info("Starting test testCloseWhileDecommission"); + + // min replication = 2 + getConf().setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, 2); + startCluster(1, 3); + + FileSystem fileSys = getCluster().getFileSystem(0); + FSNamesystem ns = getCluster().getNamesystem(0); + + String openFile = "/testDecommissionWithOpenfile.dat"; + + writeFile(fileSys, new Path(openFile), (short)3); + // make sure the file was open for write + FSDataOutputStream fdos = fileSys.append(new Path(openFile)); + byte[] bytes = new byte[1]; + fdos.write(bytes); + fdos.hsync(); + + LocatedBlocks lbs = NameNodeAdapter.getBlockLocations( + getCluster().getNameNode(0), openFile, 0, fileSize); + + DatanodeInfo[] dnInfos4LastBlock = lbs.getLastLocatedBlock().getLocations(); + + ArrayList nodes = new ArrayList(); + ArrayList dnInfos = new ArrayList(); + + DatanodeManager dm = ns.getBlockManager().getDatanodeManager(); + //decommission 2 of the 3 nodes which have last block + nodes.add(dnInfos4LastBlock[0].getXferAddr()); + dnInfos.add(dm.getDatanode(dnInfos4LastBlock[0])); + nodes.add(dnInfos4LastBlock[1].getXferAddr()); + dnInfos.add(dm.getDatanode(dnInfos4LastBlock[1])); + + // because the cluster has only 3 nodes, and 2 of which are decomm'ed, + // the last block file will remain under replicated. + initExcludeHosts(nodes); + refreshNodes(0); + + // the close() should not fail despite the number of live replicas of + // the last block becomes one. + fdos.close(); + + // make sure the two datanodes remain in decomm in progress state + BlockManagerTestUtil.recheckDecommissionState(dm); + assertTrackedAndPending(dm.getDecomManager(), 2, 0); + } /** * Tests restart of namenode while datanode hosts are added to exclude file