From e5b805d361dd1735c3ab615347e0bf5739759a07 Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Mon, 18 May 2015 12:37:21 -0700 Subject: [PATCH] HDFS-4185. Add a metric for number of active leases (Rakesh R via raviprak) (cherry picked from commit cdfae446ad285db979a79bf55665363fd943702c) --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hdfs/server/namenode/FSNamesystem.java | 17 ++++++ .../hdfs/server/namenode/LeaseManager.java | 9 +++ .../namenode/metrics/TestNameNodeMetrics.java | 59 +++++++++++++++++-- 4 files changed, 83 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 56eb913ebe6..36c3fe06018 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -233,6 +233,8 @@ Release 2.8.0 - UNRELEASED HDFS-8345. Storage policy APIs must be exposed via the FileSystem interface. (Arpit Agarwal) + HDFS-4185. Add a metric for number of active leases (Rakesh R via raviprak) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 13692a048ed..4974b920d8f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -5340,6 +5340,23 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, return blockManager.getTotalBlocks(); } + /** + * Get the number of files under construction in the system. + */ + @Metric({ "NumFilesUnderConstruction", + "Number of files under construction" }) + public long getNumFilesUnderConstruction() { + return leaseManager.countPath(); + } + + /** + * Get the total number of active clients holding lease in the system. + */ + @Metric({ "NumActiveClients", "Number of active clients holding lease" }) + public long getNumActiveClients() { + return leaseManager.countLease(); + } + /** * Get the total number of COMPLETE blocks in the system. * For safe mode only complete blocks are counted. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java index ade231211e3..0806f829747 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java @@ -130,6 +130,15 @@ public class LeaseManager { @VisibleForTesting public synchronized int countLease() {return sortedLeases.size();} + /** @return the number of paths contained in all leases */ + synchronized int countPath() { + int count = 0; + for (Lease lease : sortedLeases) { + count += lease.getFiles().size(); + } + return count; + } + /** * Adds (or re-adds) the lease for the specified file. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index 438c2d7746e..2d8cffcd3c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.namenode.metrics; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOGGERS_KEY; import static org.apache.hadoop.test.MetricsAsserts.assertCounter; import static org.apache.hadoop.test.MetricsAsserts.assertGauge; import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges; @@ -29,12 +28,12 @@ import java.io.DataInputStream; import java.io.IOException; import java.util.Random; import com.google.common.collect.ImmutableList; -import com.google.common.io.Files; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.Path; @@ -55,12 +54,10 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; -import org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.test.MetricsAsserts; -import org.apache.hadoop.util.Time; import org.apache.log4j.Level; import org.junit.After; import org.junit.Before; @@ -568,4 +565,58 @@ public class TestNameNodeMetrics { assertTrue(MetricsAsserts.getLongCounter("TransactionsNumOps", rbNew) > startWriteCounter); } + + /** + * Test metrics indicating the number of active clients and the files under + * construction + */ + @Test(timeout = 60000) + public void testNumActiveClientsAndFilesUnderConstructionMetrics() + throws Exception { + final Path file1 = getTestPath("testFileAdd1"); + createFile(file1, 100, (short) 3); + assertGauge("NumActiveClients", 0L, getMetrics(NS_METRICS)); + assertGauge("NumFilesUnderConstruction", 0L, getMetrics(NS_METRICS)); + + Path file2 = new Path("/testFileAdd2"); + FSDataOutputStream output2 = fs.create(file2); + output2.writeBytes("Some test data"); + assertGauge("NumActiveClients", 1L, getMetrics(NS_METRICS)); + assertGauge("NumFilesUnderConstruction", 1L, getMetrics(NS_METRICS)); + + Path file3 = new Path("/testFileAdd3"); + FSDataOutputStream output3 = fs.create(file3); + output3.writeBytes("Some test data"); + assertGauge("NumActiveClients", 1L, getMetrics(NS_METRICS)); + assertGauge("NumFilesUnderConstruction", 2L, getMetrics(NS_METRICS)); + + // create another DistributedFileSystem client + DistributedFileSystem fs1 = (DistributedFileSystem) cluster + .getNewFileSystemInstance(0); + try { + Path file4 = new Path("/testFileAdd4"); + FSDataOutputStream output4 = fs1.create(file4); + output4.writeBytes("Some test data"); + assertGauge("NumActiveClients", 2L, getMetrics(NS_METRICS)); + assertGauge("NumFilesUnderConstruction", 3L, getMetrics(NS_METRICS)); + + Path file5 = new Path("/testFileAdd35"); + FSDataOutputStream output5 = fs1.create(file5); + output5.writeBytes("Some test data"); + assertGauge("NumActiveClients", 2L, getMetrics(NS_METRICS)); + assertGauge("NumFilesUnderConstruction", 4L, getMetrics(NS_METRICS)); + + output2.close(); + output3.close(); + assertGauge("NumActiveClients", 1L, getMetrics(NS_METRICS)); + assertGauge("NumFilesUnderConstruction", 2L, getMetrics(NS_METRICS)); + + output4.close(); + output5.close(); + assertGauge("NumActiveClients", 0L, getMetrics(NS_METRICS)); + assertGauge("NumFilesUnderConstruction", 0L, getMetrics(NS_METRICS)); + } finally { + fs1.close(); + } + } }