From 18a5e843bcc1d86fa9e359f7a7541ed9b0f74ee1 Mon Sep 17 00:00:00 2001 From: jianghuazhu <740087514@qq.com> Date: Thu, 16 Jun 2022 19:18:58 +0800 Subject: [PATCH] HDFS-16581.Print node status when executing printTopology. (#4321) Reviewed-by: Viraj Jasani Signed-off-by: Tao Li --- .../apache/hadoop/hdfs/tools/DFSAdmin.java | 60 ++++++++++--------- .../hadoop/hdfs/tools/TestDFSAdmin.java | 48 +++++++++++++++ 2 files changed, 80 insertions(+), 28 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index 982bca19c58..bc384b8bd49 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -35,7 +35,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.TreeSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -1595,40 +1594,45 @@ public class DFSAdmin extends FsShell { * @throws IOException If an error while getting datanode report */ public int printTopology() throws IOException { - DistributedFileSystem dfs = getDFS(); - final DatanodeInfo[] report = dfs.getDataNodeStats(); + DistributedFileSystem dfs = getDFS(); + final DatanodeInfo[] report = dfs.getDataNodeStats(); - // Build a map of rack -> nodes from the datanode report - HashMap > tree = new HashMap>(); - for(DatanodeInfo dni : report) { - String location = dni.getNetworkLocation(); - String name = dni.getName(); + // Build a map of rack -> nodes from the datanode report + Map> map = new HashMap<>(); + for(DatanodeInfo dni : report) { + String location = dni.getNetworkLocation(); + String name = dni.getName(); + String dnState = dni.getAdminState().toString(); - if(!tree.containsKey(location)) { - tree.put(location, new TreeSet()); - } - - tree.get(location).add(name); + if(!map.containsKey(location)) { + map.put(location, new HashMap<>()); } - - // Sort the racks (and nodes) alphabetically, display in order - ArrayList racks = new ArrayList(tree.keySet()); - Collections.sort(racks); - - for(String r : racks) { - System.out.println("Rack: " + r); - TreeSet nodes = tree.get(r); - for(String n : nodes) { - System.out.print(" " + n); - String hostname = NetUtils.getHostNameOfIP(n); - if(hostname != null) - System.out.print(" (" + hostname + ")"); - System.out.println(); + Map node = map.get(location); + node.put(name, dnState); + } + + // Sort the racks (and nodes) alphabetically, display in order + List racks = new ArrayList<>(map.keySet()); + Collections.sort(racks); + + for(String r : racks) { + System.out.println("Rack: " + r); + Map nodes = map.get(r); + + for(Map.Entry entry : nodes.entrySet()) { + String n = entry.getKey(); + System.out.print(" " + n); + String hostname = NetUtils.getHostNameOfIP(n); + if(hostname != null) { + System.out.print(" (" + hostname + ")"); } - + System.out.print(" " + entry.getValue()); System.out.println(); } + + System.out.println(); + } return 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java index 2f4e27b6902..5b0646f3d1e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java @@ -59,6 +59,8 @@ import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; @@ -512,6 +514,52 @@ public class TestDFSAdmin { } } + @Test(timeout = 30000) + public void testPrintTopologyWithStatus() throws Exception { + redirectStream(); + final Configuration dfsConf = new HdfsConfiguration(); + final File baseDir = new File( + PathUtils.getTestDir(getClass()), + GenericTestUtils.getMethodName()); + dfsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); + + final int numDn = 4; + final String[] racks = { + "/d1/r1", "/d1/r2", + "/d2/r1", "/d2/r2"}; + + try (MiniDFSCluster miniCluster = new MiniDFSCluster.Builder(dfsConf) + .numDataNodes(numDn).racks(racks).build()) { + miniCluster.waitActive(); + assertEquals(numDn, miniCluster.getDataNodes().size()); + + DatanodeManager dm = miniCluster.getNameNode().getNamesystem(). + getBlockManager().getDatanodeManager(); + DatanodeDescriptor maintenanceNode = dm.getDatanode( + miniCluster.getDataNodes().get(1).getDatanodeId()); + maintenanceNode.setInMaintenance(); + DatanodeDescriptor demissionNode = dm.getDatanode( + miniCluster.getDataNodes().get(2).getDatanodeId()); + demissionNode.setDecommissioned(); + + final DFSAdmin dfsAdmin = new DFSAdmin(dfsConf); + + resetStream(); + final int ret = ToolRunner.run(dfsAdmin, new String[] {"-printTopology"}); + + /* collect outputs */ + final List outs = Lists.newArrayList(); + scanIntoList(out, outs); + + /* verify results */ + assertEquals(0, ret); + assertTrue(outs.get(1).contains(DatanodeInfo.AdminStates.NORMAL.toString())); + assertTrue(outs.get(4).contains(DatanodeInfo.AdminStates.IN_MAINTENANCE.toString())); + assertTrue(outs.get(7).contains(DatanodeInfo.AdminStates.DECOMMISSIONED.toString())); + assertTrue(outs.get(10).contains(DatanodeInfo.AdminStates.NORMAL.toString())); + } + } + @Test(timeout = 30000) public void testNameNodeGetReconfigurationStatus() throws IOException, InterruptedException, TimeoutException {