HDFS-11517. Expose slow disks via DataNode JMX. Contributed by Hanisha Koneru

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java

@@ -1782,6 +1782,10 @@ public DataNodeMetrics getMetrics() {
     return metrics;
   }
 
+  public DataNodeDiskMetrics getDiskMetrics() {
+    return diskMetrics;
+  }
+  
   public DataNodePeerMetrics getPeerMetrics() {
     return peerMetrics;
   }
@@ -3367,4 +3371,14 @@ public String getSendPacketDownstreamAvgInfo() {
     return peerMetrics != null ?
         peerMetrics.dumpSendPacketDownstreamAvgInfoAsJson() : null;
   }
+
+  @Override // DataNodeMXBean
+  public String getSlowDisks() {
+    if (diskMetrics == null) {
+      //Disk Stats not enabled
+      return null;
+    }
+    Set<String> slowDisks = diskMetrics.getDiskOutliersStats().keySet();
+    return JSON.toString(slowDisks);
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java

@@ -117,4 +117,11 @@ public interface DataNodeMXBean {
    * </p>
    */
   String getSendPacketDownstreamAvgInfo();
+
+  /**
+   * Gets the slow disks in the Datanode.
+   *
+   * @return list of slow disks
+   */
+  String getSlowDisks();
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java

@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.metrics;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -52,7 +54,8 @@ public class DataNodeDiskMetrics {
   private volatile boolean shouldRun;
   private OutlierDetector slowDiskDetector;
   private Daemon slowDiskDetectionDaemon;
-  private volatile Map<String, Map<DiskOutlierDetectionOp, Double>> diskOutliersStats;
+  private volatile Map<String, Map<DiskOutlierDetectionOp, Double>>
+      diskOutliersStats = Maps.newHashMap();
 
   public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs) {
     this.dn = dn;
@@ -178,4 +181,12 @@ public void shutdownAndWait() {
       LOG.error("Disk Outlier Detection daemon did not shutdown", e);
     }
   }
+
+  /**
+   * Use only for testing.
+   */
+  @VisibleForTesting
+  public void addSlowDiskForTesting(String slowDiskPath) {
+    diskOutliersStats.put(slowDiskPath, ImmutableMap.of());
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java

@@ -33,6 +33,7 @@
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.test.GenericTestUtils;
@@ -98,8 +99,12 @@ public void testDataNodeMXBean() throws Exception {
       String bpActorInfo = (String)mbs.getAttribute(mxbeanName,
           "BPServiceActorInfo");
       Assert.assertEquals(datanode.getBPServiceActorInfo(), bpActorInfo);
+      String slowDisks = (String)mbs.getAttribute(mxbeanName, "SlowDisks");
+      Assert.assertEquals(datanode.getSlowDisks(), slowDisks);
     } finally {
-      if (cluster != null) {cluster.shutdown();}
+      if (cluster != null) {
+        cluster.shutdown();
+      }
     }
   }
   
@@ -203,4 +208,30 @@ private int getTotalNumBlocks(MBeanServer mbs, ObjectName mxbeanName)
     }
     return totalBlocks;
   }
+
+  @Test
+  public void testDataNodeMXBeanSlowDisksEnabled() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setDouble(DFSConfigKeys
+        .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0);
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+
+    try {
+      List<DataNode> datanodes = cluster.getDataNodes();
+      Assert.assertEquals(datanodes.size(), 1);
+      DataNode datanode = datanodes.get(0);
+      String slowDiskPath = "test/data1/slowVolume";
+      datanode.getDiskMetrics().addSlowDiskForTesting(slowDiskPath);
+
+      MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+      ObjectName mxbeanName = new ObjectName(
+          "Hadoop:service=DataNode,name=DataNodeInfo");
+
+      String slowDisks = (String)mbs.getAttribute(mxbeanName, "SlowDisks");
+      Assert.assertEquals(datanode.getSlowDisks(), slowDisks);
+      Assert.assertTrue(slowDisks.contains(slowDiskPath));
+    } finally {
+      if (cluster != null) {cluster.shutdown();}
+    }
+  }
 }