diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index e744f767587..17e401ab402 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -214,7 +214,15 @@ Each metrics record contains tags such as HAState and Hostname as additional inf | `PendingDataNodeMessageCount` | (HA-only) Current number of pending block-related messages for later processing in the standby NameNode | | `MillisSinceLastLoadedEdits` | (HA-only) Time in milliseconds since the last time standby NameNode load edit log. In active NameNode, set to 0 | | `BlockCapacity` | Current number of block capacity | +| `NumLiveDataNodes` | Number of datanodes which are currently live | +| `NumDeadDataNodes` | Number of datanodes which are currently dead | +| `NumDecomLiveDataNodes` | Number of datanodes which have been decommissioned and are now live | +| `NumDecomDeadDataNodes` | Number of datanodes which have been decommissioned and are now dead | +| `NumDecommissioningDataNodes` | Number of datanodes in decommissioning state | +| `VolumeFailuresTotal` | Total number of volume failures across all Datanodes | +| `EstimatedCapacityLostTotal` | An estimate of the total capacity lost due to volume failures | | `StaleDataNodes` | Current number of DataNodes marked stale due to delayed heartbeat | +| `NumStaleStorages` | Number of storages marked as content stale (after NameNode restart/failover before first block report is received) | | `TotalFiles` | Deprecated: Use FilesTotal instead | | `MissingReplOneBlocks` | Current number of missing blocks with replication factor 1 | | `NumFilesUnderConstruction` | Current number of files under construction | @@ -226,6 +234,9 @@ Each metrics record contains tags such as HAState and Hostname as additional inf | `TotalSyncTimes` | Total number of milliseconds spent by various edit logs in sync operation| | `NameDirSize` | NameNode name directories size in bytes | | `NumTimedOutPendingReplications` | The number of timed out replications. Not the number of unique blocks that timed out. Note: The metric name will be changed to `NumTimedOutPendingReconstructions` in Hadoop 3 release. | +| `NumInMaintenanceLiveDataNodes` | Number of live Datanodes which are in maintenance state | +| `NumInMaintenanceDeadDataNodes` | Number of dead Datanodes which are in maintenance state | +| `NumEnteringMaintenanceDataNodes` | Number of Datanodes that are entering the maintenance state | | `FSN(Read|Write)Lock`*OperationName*`NumOps` | Total number of acquiring lock by operations | | `FSN(Read|Write)Lock`*OperationName*`AvgTime` | Average time of holding the lock by operations in milliseconds | diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 825549c9a28..59f38f4ee0f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -4649,16 +4649,20 @@ void shutdown() { } @Override // FSNamesystemMBean + @Metric({"NumLiveDataNodes", "Number of datanodes which are currently live"}) public int getNumLiveDataNodes() { return getBlockManager().getDatanodeManager().getNumLiveDataNodes(); } @Override // FSNamesystemMBean + @Metric({"NumDeadDataNodes", "Number of datanodes which are currently dead"}) public int getNumDeadDataNodes() { return getBlockManager().getDatanodeManager().getNumDeadDataNodes(); } @Override // FSNamesystemMBean + @Metric({"NumDecomLiveDataNodes", + "Number of datanodes which have been decommissioned and are now live"}) public int getNumDecomLiveDataNodes() { final List live = new ArrayList(); getBlockManager().getDatanodeManager().fetchDatanodes(live, null, false); @@ -4670,6 +4674,8 @@ public int getNumDecomLiveDataNodes() { } @Override // FSNamesystemMBean + @Metric({"NumDecomDeadDataNodes", + "Number of datanodes which have been decommissioned and are now dead"}) public int getNumDecomDeadDataNodes() { final List dead = new ArrayList(); getBlockManager().getDatanodeManager().fetchDatanodes(null, dead, false); @@ -4681,6 +4687,8 @@ public int getNumDecomDeadDataNodes() { } @Override // FSNamesystemMBean + @Metric({"VolumeFailuresTotal", + "Total number of volume failures across all Datanodes"}) public int getVolumeFailuresTotal() { List live = new ArrayList(); getBlockManager().getDatanodeManager().fetchDatanodes(live, null, false); @@ -4692,6 +4700,8 @@ public int getVolumeFailuresTotal() { } @Override // FSNamesystemMBean + @Metric({"EstimatedCapacityLostTotal", + "An estimate of the total capacity lost due to volume failures"}) public long getEstimatedCapacityLostTotal() { List live = new ArrayList(); getBlockManager().getDatanodeManager().fetchDatanodes(live, null, false); @@ -4707,6 +4717,8 @@ public long getEstimatedCapacityLostTotal() { } @Override // FSNamesystemMBean + @Metric({"NumDecommissioningDataNodes", + "Number of datanodes in decommissioning state"}) public int getNumDecommissioningDataNodes() { return getBlockManager().getDatanodeManager().getDecommissioningNodes() .size(); @@ -4724,6 +4736,8 @@ public int getNumStaleDataNodes() { * before NN receives the first Heartbeat followed by the first Blockreport. */ @Override // FSNamesystemMBean + @Metric({"NumStaleStorages", + "Number of storages marked as content stale"}) public int getNumStaleStorages() { return getBlockManager().getDatanodeManager().getNumStaleStorages(); } @@ -7063,6 +7077,8 @@ public long getBytesInFuture() { @Override // FSNamesystemMBean + @Metric({"NumInMaintenanceLiveDataNodes", + "Number of live Datanodes which are in maintenance state"}) public int getNumInMaintenanceLiveDataNodes() { final List live = new ArrayList(); getBlockManager().getDatanodeManager().fetchDatanodes(live, null, true); @@ -7074,6 +7090,8 @@ public int getNumInMaintenanceLiveDataNodes() { } @Override // FSNamesystemMBean + @Metric({"NumInMaintenanceDeadDataNodes", + "Number of dead Datanodes which are in maintenance state"}) public int getNumInMaintenanceDeadDataNodes() { final List dead = new ArrayList(); getBlockManager().getDatanodeManager().fetchDatanodes(null, dead, true); @@ -7085,6 +7103,8 @@ public int getNumInMaintenanceDeadDataNodes() { } @Override // FSNamesystemMBean + @Metric({"NumEnteringMaintenanceDataNodes", + "Number of Datanodes that are entering the maintenance state"}) public int getNumEnteringMaintenanceDataNodes() { return getBlockManager().getDatanodeManager().getEnteringMaintenanceNodes() .size(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index d7b7692e52f..f66c30d80ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -148,6 +148,8 @@ public class MiniDFSCluster implements AutoCloseable { public static final String HDFS_MINIDFS_BASEDIR = "hdfs.minidfs.basedir"; public static final String DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY = DFS_NAMENODE_SAFEMODE_EXTENSION_KEY + ".testing"; + public static final String DFS_NAMENODE_DECOMMISSION_INTERVAL_TESTING_KEY + = DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY + ".testing"; // Changing this default may break some tests that assume it is 2. private static final int DEFAULT_STORAGES_PER_DATANODE = 2; @@ -818,7 +820,9 @@ private void initMiniDFSCluster( int safemodeExtension = conf.getInt( DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY, 0); conf.setInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, safemodeExtension); - conf.setInt(DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 3); // 3 second + int decommissionInterval = conf.getInt( + DFS_NAMENODE_DECOMMISSION_INTERVAL_TESTING_KEY, 3); + conf.setInt(DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, decommissionInterval); if (!useConfiguredTopologyMappingClass) { conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, StaticMapping.class, DNSToSwitchMapping.class); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index a29bcf9e881..c257fd89c78 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode.metrics; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystemTestHelper; @@ -40,7 +41,9 @@ import java.io.File; import java.io.IOException; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.EnumSet; +import java.util.List; import java.util.Random; import com.google.common.collect.ImmutableList; @@ -64,12 +67,15 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.MockNameNodeResourceChecker; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.hdfs.tools.NNHAServiceTarget; +import org.apache.hadoop.hdfs.util.HostsFileWriter; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; @@ -106,6 +112,14 @@ public class TestNameNodeMetrics { DFS_REPLICATION_INTERVAL); CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, DFS_REPLICATION_INTERVAL); + // Set it long enough to essentially disable unless we manually call it + // Used for decommissioning DataNode metrics + CONF.setInt(MiniDFSCluster.DFS_NAMENODE_DECOMMISSION_INTERVAL_TESTING_KEY, + 9999999); + // Next two configs used for checking failed volume metrics + CONF.setTimeDuration(DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY, + 10, TimeUnit.MILLISECONDS); + CONF.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 1); CONF.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + PERCENTILES_INTERVAL); // Enable stale DataNodes checking @@ -118,6 +132,7 @@ public class TestNameNodeMetrics { private DistributedFileSystem fs; private final Random rand = new Random(); private FSNamesystem namesystem; + private HostsFileWriter hostsFileWriter; private BlockManager bm; private static Path getTestPath(String fileName) { @@ -126,7 +141,10 @@ private static Path getTestPath(String fileName) { @Before public void setUp() throws Exception { - cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(DATANODE_COUNT).build(); + hostsFileWriter = new HostsFileWriter(); + hostsFileWriter.initialize(CONF, "temp/decommission"); + cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(DATANODE_COUNT) + .build(); cluster.waitActive(); namesystem = cluster.getNamesystem(); bm = namesystem.getBlockManager(); @@ -141,6 +159,10 @@ public void tearDown() throws Exception { MetricsRecordBuilder rb = getMetrics(source); assertQuantileGauges("GetGroups1s", rb); } + if (hostsFileWriter != null) { + hostsFileWriter.cleanup(); + hostsFileWriter = null; + } if (cluster != null) { cluster.shutdown(); cluster = null; @@ -221,6 +243,96 @@ public void testStaleNodes() throws Exception { .getBlockManager()); assertGauge("StaleDataNodes", 0, getMetrics(NS_METRICS)); } + + /** + * Test metrics associated with volume failures. + */ + @Test + public void testVolumeFailures() throws Exception { + assertGauge("VolumeFailuresTotal", 0, getMetrics(NS_METRICS)); + assertGauge("EstimatedCapacityLostTotal", 0L, getMetrics(NS_METRICS)); + DataNode dn = cluster.getDataNodes().get(0); + FsDatasetSpi.FsVolumeReferences volumeReferences = + DataNodeTestUtils.getFSDataset(dn).getFsVolumeReferences(); + FsVolumeImpl fsVolume = (FsVolumeImpl) volumeReferences.get(0); + File dataDir = new File(fsVolume.getBasePath()); + long capacity = fsVolume.getCapacity(); + volumeReferences.close(); + DataNodeTestUtils.injectDataDirFailure(dataDir); + DataNodeTestUtils.waitForDiskError(dn, fsVolume); + DataNodeTestUtils.triggerHeartbeat(dn); + BlockManagerTestUtil.checkHeartbeat(bm); + assertGauge("VolumeFailuresTotal", 1, getMetrics(NS_METRICS)); + assertGauge("EstimatedCapacityLostTotal", capacity, getMetrics(NS_METRICS)); + } + + /** + * Test metrics associated with liveness and decommission status of DataNodes. + */ + @Test + public void testDataNodeLivenessAndDecom() throws Exception { + List dataNodes = cluster.getDataNodes(); + DatanodeDescriptor[] dnDescriptors = new DatanodeDescriptor[DATANODE_COUNT]; + String[] dnAddresses = new String[DATANODE_COUNT]; + for (int i = 0; i < DATANODE_COUNT; i++) { + dnDescriptors[i] = bm.getDatanodeManager() + .getDatanode(dataNodes.get(i).getDatanodeId()); + dnAddresses[i] = dnDescriptors[i].getXferAddr(); + } + // First put all DNs into include + hostsFileWriter.initIncludeHosts(dnAddresses); + bm.getDatanodeManager().refreshNodes(CONF); + assertGauge("NumDecomLiveDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumLiveDataNodes", DATANODE_COUNT, getMetrics(NS_METRICS)); + + // Now decommission one DN + hostsFileWriter.initExcludeHost(dnAddresses[0]); + bm.getDatanodeManager().refreshNodes(CONF); + assertGauge("NumDecommissioningDataNodes", 1, getMetrics(NS_METRICS)); + BlockManagerTestUtil.recheckDecommissionState(bm.getDatanodeManager()); + assertGauge("NumDecommissioningDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumDecomLiveDataNodes", 1, getMetrics(NS_METRICS)); + assertGauge("NumLiveDataNodes", DATANODE_COUNT, getMetrics(NS_METRICS)); + + // Now kill all DNs by expiring their heartbeats + for (int i = 0; i < DATANODE_COUNT; i++) { + DataNodeTestUtils.setHeartbeatsDisabledForTests(dataNodes.get(i), true); + long expireInterval = CONF.getLong( + DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, + DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT) * 2L + + CONF.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, + DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 10 * 1000L; + DFSTestUtil.resetLastUpdatesWithOffset(dnDescriptors[i], + -(expireInterval + 1)); + } + BlockManagerTestUtil.checkHeartbeat(bm); + assertGauge("NumDecomLiveDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumDecomDeadDataNodes", 1, getMetrics(NS_METRICS)); + assertGauge("NumLiveDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumDeadDataNodes", DATANODE_COUNT, getMetrics(NS_METRICS)); + + // Now remove the decommissioned DN altogether + String[] includeHosts = new String[dnAddresses.length - 1]; + for (int i = 0; i < includeHosts.length; i++) { + includeHosts[i] = dnAddresses[i + 1]; + } + hostsFileWriter.initIncludeHosts(includeHosts); + hostsFileWriter.initExcludeHosts(new ArrayList()); + bm.getDatanodeManager().refreshNodes(CONF); + assertGauge("NumDecomLiveDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumDecomDeadDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumLiveDataNodes", 0, getMetrics(NS_METRICS)); + assertGauge("NumDeadDataNodes", DATANODE_COUNT - 1, getMetrics(NS_METRICS)); + + // Finally mark the remaining DNs as live again + for (int i = 1; i < dataNodes.size(); i++) { + DataNodeTestUtils.setHeartbeatsDisabledForTests(dataNodes.get(i), false); + DFSTestUtil.resetLastUpdatesWithOffset(dnDescriptors[i], 0); + } + BlockManagerTestUtil.checkHeartbeat(bm); + assertGauge("NumLiveDataNodes", DATANODE_COUNT - 1, getMetrics(NS_METRICS)); + assertGauge("NumDeadDataNodes", 0, getMetrics(NS_METRICS)); + } /** Test metrics associated with addition of a file */ @Test