From e2113500df2479dd2d8f1d8a37fdb2bb16463181 Mon Sep 17 00:00:00 2001 From: Takanobu Asanuma Date: Sat, 29 Sep 2018 00:53:35 +0900 Subject: [PATCH] HDFS-8196. Post enabled Erasure Coding Policies on NameNode UI. Contributed by Kitti Nanasi and Kai Sasaki. --- .../namenode/ErasureCodingPolicyManager.java | 6 +++ .../hdfs/server/namenode/FSNamesystem.java | 9 ++++ .../namenode/metrics/ECBlockGroupsMBean.java | 5 ++ .../src/main/webapps/hdfs/dfshealth.html | 4 +- .../server/namenode/TestNameNodeMXBean.java | 49 +++++++++++++++++++ 5 files changed, 72 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index e7de05b00d0..d2bf3af9017 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -31,6 +31,7 @@ import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; +import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -430,4 +431,9 @@ public synchronized void loadPolicies( allPolicies = policiesByName.values().toArray(new ErasureCodingPolicyInfo[0]); } + + public String getEnabledPoliciesMetric() { + return StringUtils.join(", ", + enabledPoliciesByName.keySet()); + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 1caf7c20093..137dfb9d053 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -4973,6 +4973,15 @@ public long getTotalECBlockGroups() { return blockManager.getTotalECBlockGroups(); } + /** + * Get the enabled erasure coding policies separated with comma. + */ + @Override // ECBlockGroupsMBean + @Metric({"EnabledEcPolicies", "Enabled erasure coding policies"}) + public String getEnabledEcPolicies() { + return getErasureCodingPolicyManager().getEnabledPoliciesMetric(); + } + @Override public long getBlockDeletionStartTime() { return startTime + blockManager.getStartupDelayBlockDeletionInMs(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java index 808fc628806..e5d2a09e0b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java @@ -61,4 +61,9 @@ public interface ECBlockGroupsMBean { * Return total number of erasure coded block groups. */ long getTotalECBlockGroups(); + + /** + * @return the enabled erasure coding policies separated with comma. + */ + String getEnabledEcPolicies(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 4495b99b1fa..f993ae7fa78 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -186,10 +186,12 @@ {/eq} Block Deletion Start Time{BlockDeletionStartTime|date_tostring} {/fs} - {#fsn} Last Checkpoint Time{@if cond="{LastCheckpointTime} === 0"}Never{:else}{LastCheckpointTime|date_tostring}{/if} {/fsn} +{#ecstat} + Enabled Erasure Coding Policies{EnabledEcPolicies} +{/ecstat}

NameNode Journal Status

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index 37284202847..07a494596c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; @@ -728,6 +729,46 @@ private void checkNNDirSize(Collection nameDirUris, String metric){ } } + @Test + public void testEnabledEcPoliciesMetric() throws Exception { + MiniDFSCluster cluster = null; + DistributedFileSystem fs = null; + try { + Configuration conf = new HdfsConfiguration(); + + ErasureCodingPolicy defaultPolicy = + StripedFileTestUtil.getDefaultECPolicy(); + int dataBlocks = defaultPolicy.getNumDataUnits(); + int parityBlocks = defaultPolicy.getNumParityUnits(); + int totalSize = dataBlocks + parityBlocks; + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(totalSize).build(); + fs = cluster.getFileSystem(); + + final String defaultPolicyName = defaultPolicy.getName(); + final String rs104PolicyName = "RS-10-4-1024k"; + + assertEquals("Enabled EC policies metric should return with " + + "the default EC policy", defaultPolicyName, + getEnabledEcPoliciesMetric()); + + fs.enableErasureCodingPolicy(rs104PolicyName); + assertEquals("Enabled EC policies metric should return with " + + "both enabled policies separated by a comma", + rs104PolicyName + ", " + defaultPolicyName, + getEnabledEcPoliciesMetric()); + + fs.disableErasureCodingPolicy(defaultPolicyName); + fs.disableErasureCodingPolicy(rs104PolicyName); + assertEquals("Enabled EC policies metric should return with " + + "an empty string if there is no enabled policy", + "", getEnabledEcPoliciesMetric()); + } finally { + fs.close(); + cluster.shutdown(); + } + } + @Test public void testVerifyMissingBlockGroupsMetrics() throws Exception { MiniDFSCluster cluster = null; @@ -968,4 +1009,12 @@ void verifyTotalBlocksMetrics(long expectedTotalReplicatedBlocks, assertEquals("Unexpected total ec block groups!", expectedTotalECBlockGroups, totalECBlockGroups.longValue()); } + + private String getEnabledEcPoliciesMetric() throws Exception { + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ObjectName mxbeanName = new ObjectName( + "Hadoop:service=NameNode,name=ECBlockGroupsState"); + return (String) (mbs.getAttribute(mxbeanName, + "EnabledEcPolicies")); + } }