From 8def51a708e5de8a57689b8c9b3fd034cfd2cd78 Mon Sep 17 00:00:00 2001 From: Zhe Zhang Date: Thu, 29 Oct 2015 11:14:00 -0700 Subject: [PATCH] HDFS-9229. Expose size of NameNode directory as a metric. Contributed by Surendra Singh Lilhore. Change-Id: I985627a5d1400249d72d084283ef366d5ac6e07b --- .../src/site/markdown/Metrics.md | 1 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/hdfs/server/common/Storage.java | 15 +++++ .../hadoop/hdfs/server/namenode/FSImage.java | 4 ++ .../hdfs/server/namenode/FSNamesystem.java | 5 ++ .../hdfs/server/namenode/NNStorage.java | 25 ++++++++ .../hdfs/server/namenode/NameNodeMXBean.java | 5 ++ .../server/namenode/ha/EditLogTailer.java | 4 +- .../server/namenode/TestNameNodeMXBean.java | 62 ++++++++++++++++++- 9 files changed, 122 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 3e01fe31b61..2e24aba8e2d 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -240,6 +240,7 @@ Each metrics record contains tags such as HAState and Hostname as additional inf | `LockQueueLength` | Number of threads waiting to acquire FSNameSystem lock | | `TotalSyncCount` | Total number of sync operations performed by edit log | | `TotalSyncTimes` | Total number of milliseconds spent by various edit logs in sync operation| +| `NameDirSize` | NameNode name directories size in bytes | JournalNode ----------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a87fe76b34f..36a70b86dc2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1608,6 +1608,9 @@ Release 2.8.0 - UNRELEASED HDFS-8545. Refactor FS#getUsed() to use ContentSummary and add an API to fetch the total file length from a specific path (J.Andreina via vinayakumarb) + HDFS-9229. Expose size of NameNode directory as a metric. + (Surendra Singh Lilhore via zhz) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java index c6302068ca8..7b4b571baeb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Properties; +import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -310,6 +311,20 @@ public StorageDirType getStorageDirType() { return dirType; } + /** + * Get storage directory size. + */ + public long getDirecorySize() { + try { + if (!isShared() && root != null && root.exists()) { + return FileUtils.sizeOfDirectory(root); + } + } catch (Exception e) { + LOG.warn("Failed to get directory size :" + root, e); + } + return 0; + } + public void read(File from, Storage storage) throws IOException { Properties props = readPropertiesFile(from); storage.setFieldsFromProperties(props, this); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 93dc097bb49..341dd985522 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -1064,6 +1064,8 @@ public synchronized void saveNamespace(FSNamesystem source, NameNodeFile nnf, } finally { removeFromCheckpointing(imageTxId); } + //Update NameDirSize Metric + getStorage().updateNameDirSize(); } /** @@ -1244,6 +1246,8 @@ CheckpointSignature rollEditLog(int layoutVersion) throws IOException { // we won't miss this log segment on a restart if the edits directories // go missing. storage.writeTransactionIdFileToStorage(getEditLog().getCurSegmentTxId()); + //Update NameDirSize Metric + getStorage().updateNameDirSize(); return new CheckpointSignature(this); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 969677a3994..29ee0879f3b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6402,6 +6402,11 @@ public String getSoftwareVersion() { return VersionInfo.getVersion(); } + @Override // NameNodeStatusMXBean + public String getNameDirSize() { + return getFSImage().getStorage().getNNDirectorySize(); + } + /** * Verifies that the given identifier and password are valid and match. * @param identifier Token identifier. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java index d872c039590..9b63e728b1c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.UUID; import java.util.concurrent.CopyOnWriteArrayList; @@ -52,6 +53,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.DNS; import org.apache.hadoop.util.Time; +import org.mortbay.util.ajax.JSON; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -148,6 +150,11 @@ public boolean isOfType(StorageDirType type) { */ private HashMap deprecatedProperties; + /** + * Name directories size for metric. + */ + private Map nameDirSizeMap = new HashMap<>(); + /** * Construct the NNStorage. * @param conf Namenode configuration. @@ -166,6 +173,8 @@ public NNStorage(Configuration conf, setStorageDirectories(imageDirs, Lists.newArrayList(editsDirs), FSNamesystem.getSharedEditsDirs(conf)); + //Update NameDirSize metric value after NN start + updateNameDirSize(); } @Override // Storage @@ -1075,4 +1084,20 @@ public NamespaceInfo getNamespaceInfo() { getBlockPoolID(), getCTime()); } + + public String getNNDirectorySize() { + return JSON.toString(nameDirSizeMap); + } + + public void updateNameDirSize() { + Map nnDirSizeMap = new HashMap<>(); + for (Iterator it = dirIterator(); it.hasNext();) { + StorageDirectory sd = it.next(); + if (!sd.isShared()) { + nnDirSizeMap.put(sd.getRoot().getAbsolutePath(), sd.getDirecorySize()); + } + } + nameDirSizeMap.clear(); + nameDirSizeMap.putAll(nnDirSizeMap); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java index 9dcef895d8e..d127c98308e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java @@ -272,4 +272,9 @@ public interface NameNodeMXBean { */ public Map getDistinctVersions(); + /** + * Get namenode directory size. + */ + String getNameDirSize(); + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java index 59f90a6db3e..6e60dbab7ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java @@ -372,6 +372,8 @@ private void doWork() { } finally { namesystem.cpUnlock(); } + //Update NameDirSize Metric + namesystem.getFSImage().getStorage().updateNameDirSize(); } catch (EditLogInputException elie) { LOG.warn("Error while reading edits from disk. Will try again.", elie); } catch (InterruptedException ie) { @@ -463,4 +465,4 @@ private NamenodeProtocol getActiveNodeProxy() throws IOException { return cachedActiveProxy; } } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index 559aae68c33..5c865e1d3e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -26,12 +26,16 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.hdfs.server.namenode.top.TopConf; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.util.VersionInfo; import org.codehaus.jackson.map.ObjectMapper; import org.junit.Test; @@ -40,6 +44,7 @@ import javax.management.MBeanServer; import javax.management.ObjectName; import java.io.File; +import java.io.IOException; import java.lang.management.ManagementFactory; import java.net.URI; import java.util.Collection; @@ -186,7 +191,7 @@ public void testNameNodeMXBeanInfo() throws Exception { } assertEquals(2, statusMap.get("active").size()); assertEquals(0, statusMap.get("failed").size()); - + // This will cause the first dir to fail. File failedNameDir = new File(nameDirUris.iterator().next()); assertEquals(0, FileUtil.chmod( @@ -412,4 +417,59 @@ public void testQueueLength() throws Exception { } } } + + @Test(timeout = 120000) + public void testNNDirectorySize() throws Exception{ + Configuration conf = new Configuration(); + conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); + // Have to specify IPC ports so the NNs can talk to each other. + MiniDFSNNTopology topology = new MiniDFSNNTopology() + .addNameservice(new MiniDFSNNTopology.NSConf("ns1") + .addNN(new MiniDFSNNTopology.NNConf("nn1") + .setIpcPort(ServerSocketUtil.getPort(0, 100))) + .addNN(new MiniDFSNNTopology.NNConf("nn2") + .setIpcPort(ServerSocketUtil.getPort(0, 100)))); + + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(topology).numDataNodes(0) + .build(); + FileSystem fs = null; + try { + cluster.waitActive(); + + FSNamesystem nn0 = cluster.getNamesystem(0); + FSNamesystem nn1 = cluster.getNamesystem(1); + checkNNDirSize(cluster.getNameDirs(0), nn0.getNameDirSize()); + checkNNDirSize(cluster.getNameDirs(1), nn1.getNameDirSize()); + cluster.transitionToActive(0); + fs = cluster.getFileSystem(0); + DFSTestUtil.createFile(fs, new Path("/file"), 0, (short) 1, 0L); + + //rollEditLog + HATestUtil.waitForStandbyToCatchUp(cluster.getNameNode(0), + cluster.getNameNode(1)); + checkNNDirSize(cluster.getNameDirs(0), nn0.getNameDirSize()); + checkNNDirSize(cluster.getNameDirs(1), nn1.getNameDirSize()); + + //Test metric after call saveNamespace + DFSTestUtil.createFile(fs, new Path("/file"), 0, (short) 1, 0L); + nn0.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + nn0.saveNamespace(0, 0); + checkNNDirSize(cluster.getNameDirs(0), nn0.getNameDirSize()); + } finally { + cluster.shutdown(); + } + } + + @SuppressWarnings("unchecked") + private void checkNNDirSize(Collection nameDirUris, String metric){ + Map nnDirMap = + (Map) JSON.parse(metric); + assertEquals(nameDirUris.size(), nnDirMap.size()); + for (URI dirUrl : nameDirUris) { + File dir = new File(dirUrl); + assertEquals(nnDirMap.get(dir.getAbsolutePath()).longValue(), + FileUtils.sizeOfDirectory(dir)); + } + } }