From 0b83c95ff43e043af58150bf7e23bf091f6d7fe7 Mon Sep 17 00:00:00 2001 From: Brahma Reddy Battula Date: Fri, 30 Nov 2018 00:18:27 +0530 Subject: [PATCH] HDFS-14095. EC: Track Erasure Coding commands in DFS statistics. Contributed by Ayush Saxena. (cherry picked from commit f534736867eed962899615ca1b7eb68bcf591d17) --- .../hadoop/hdfs/DFSOpsCountStatistics.java | 9 +++ .../hadoop/hdfs/DistributedFileSystem.java | 18 ++++++ .../hdfs/TestDistributedFileSystem.java | 63 ++++++++++++++++++- 3 files changed, 89 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java index 3dcf13b080a..b9852bad566 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java @@ -41,6 +41,7 @@ public class DFSOpsCountStatistics extends StorageStatistics { /** This is for counting distributed file system operations. */ public enum OpType { + ADD_EC_POLICY("op_add_ec_policy"), ALLOW_SNAPSHOT("op_allow_snapshot"), APPEND(CommonStatisticNames.OP_APPEND), CONCAT("op_concat"), @@ -51,10 +52,15 @@ public class DFSOpsCountStatistics extends StorageStatistics { CREATE_SYM_LINK("op_create_symlink"), DELETE(CommonStatisticNames.OP_DELETE), DELETE_SNAPSHOT("op_delete_snapshot"), + DISABLE_EC_POLICY("op_disable_ec_policy"), DISALLOW_SNAPSHOT("op_disallow_snapshot"), + ENABLE_EC_POLICY("op_enable_ec_policy"), EXISTS(CommonStatisticNames.OP_EXISTS), GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"), GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY), + GET_EC_CODECS("op_get_ec_codecs"), + GET_EC_POLICY("op_get_ec_policy"), + GET_EC_POLICIES("op_get_ec_policies"), GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"), GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM), GET_FILE_LINK_STATUS("op_get_file_link_status"), @@ -76,11 +82,13 @@ public class DFSOpsCountStatistics extends StorageStatistics { REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL), REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES), REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL), + REMOVE_EC_POLICY("op_remove_ec_policy"), REMOVE_XATTR("op_remove_xattr"), RENAME(CommonStatisticNames.OP_RENAME), RENAME_SNAPSHOT("op_rename_snapshot"), RESOLVE_LINK("op_resolve_link"), SET_ACL(CommonStatisticNames.OP_SET_ACL), + SET_EC_POLICY("op_set_ec_policy"), SET_OWNER(CommonStatisticNames.OP_SET_OWNER), SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION), SET_REPLICATION("op_set_replication"), @@ -90,6 +98,7 @@ public class DFSOpsCountStatistics extends StorageStatistics { GET_SNAPSHOT_DIFF("op_get_snapshot_diff"), GET_SNAPSHOTTABLE_DIRECTORY_LIST("op_get_snapshottable_directory_list"), TRUNCATE(CommonStatisticNames.OP_TRUNCATE), + UNSET_EC_POLICY("op_unset_ec_policy"), UNSET_STORAGE_POLICY("op_unset_storage_policy"); private static final Map SYMBOL_MAP = diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 65d211c4633..3553428252e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -2846,6 +2846,8 @@ public class DistributedFileSystem extends FileSystem */ public void setErasureCodingPolicy(final Path path, final String ecPolicyName) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.SET_EC_POLICY); Path absF = fixRelativePart(path); new FileSystemLinkResolver() { @Override @@ -2879,6 +2881,8 @@ public class DistributedFileSystem extends FileSystem */ public ErasureCodingPolicy getErasureCodingPolicy(final Path path) throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_EC_POLICY); Path absF = fixRelativePart(path); return new FileSystemLinkResolver() { @Override @@ -2910,6 +2914,8 @@ public class DistributedFileSystem extends FileSystem */ public Collection getAllErasureCodingPolicies() throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_EC_POLICIES); return Arrays.asList(dfs.getErasureCodingPolicies()); } @@ -2922,6 +2928,8 @@ public class DistributedFileSystem extends FileSystem */ public Map getAllErasureCodingCodecs() throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_EC_CODECS); return dfs.getErasureCodingCodecs(); } @@ -2938,6 +2946,8 @@ public class DistributedFileSystem extends FileSystem */ public AddErasureCodingPolicyResponse[] addErasureCodingPolicies( ErasureCodingPolicy[] policies) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.ADD_EC_POLICY); return dfs.addErasureCodingPolicies(policies); } @@ -2949,6 +2959,8 @@ public class DistributedFileSystem extends FileSystem */ public void removeErasureCodingPolicy(String ecPolicyName) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.REMOVE_EC_POLICY); dfs.removeErasureCodingPolicy(ecPolicyName); } @@ -2960,6 +2972,8 @@ public class DistributedFileSystem extends FileSystem */ public void enableErasureCodingPolicy(String ecPolicyName) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.ENABLE_EC_POLICY); dfs.enableErasureCodingPolicy(ecPolicyName); } @@ -2971,6 +2985,8 @@ public class DistributedFileSystem extends FileSystem */ public void disableErasureCodingPolicy(String ecPolicyName) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.DISABLE_EC_POLICY); dfs.disableErasureCodingPolicy(ecPolicyName); } @@ -2981,6 +2997,8 @@ public class DistributedFileSystem extends FileSystem * @throws IOException */ public void unsetErasureCodingPolicy(final Path path) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.UNSET_EC_POLICY); Path absF = fixRelativePart(path); new FileSystemLinkResolver() { @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 03e6c8a26e2..344bc215f56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -772,7 +772,68 @@ public class TestDistributedFileSystem { } finally { if (cluster != null) cluster.shutdown(); } - + } + + @Test + public void testECStatistics() throws IOException { + try (MiniDFSCluster cluster = + new MiniDFSCluster.Builder(getTestConfiguration()).build()) { + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + Path dir = new Path("/test"); + dfs.mkdirs(dir); + int readOps = 0; + int writeOps = 0; + FileSystem.clearStatistics(); + + long opCount = getOpStatistics(OpType.ENABLE_EC_POLICY); + dfs.enableErasureCodingPolicy("RS-10-4-1024k"); + checkStatistics(dfs, readOps, ++writeOps, 0); + checkOpStatistics(OpType.ENABLE_EC_POLICY, opCount + 1); + + opCount = getOpStatistics(OpType.SET_EC_POLICY); + dfs.setErasureCodingPolicy(dir, "RS-10-4-1024k"); + checkStatistics(dfs, readOps, ++writeOps, 0); + checkOpStatistics(OpType.SET_EC_POLICY, opCount + 1); + + opCount = getOpStatistics(OpType.GET_EC_POLICY); + dfs.getErasureCodingPolicy(dir); + checkStatistics(dfs, ++readOps, writeOps, 0); + checkOpStatistics(OpType.GET_EC_POLICY, opCount + 1); + + opCount = getOpStatistics(OpType.UNSET_EC_POLICY); + dfs.unsetErasureCodingPolicy(dir); + checkStatistics(dfs, readOps, ++writeOps, 0); + checkOpStatistics(OpType.UNSET_EC_POLICY, opCount + 1); + + opCount = getOpStatistics(OpType.GET_EC_POLICIES); + dfs.getAllErasureCodingPolicies(); + checkStatistics(dfs, ++readOps, writeOps, 0); + checkOpStatistics(OpType.GET_EC_POLICIES, opCount + 1); + + opCount = getOpStatistics(OpType.GET_EC_CODECS); + dfs.getAllErasureCodingCodecs(); + checkStatistics(dfs, ++readOps, writeOps, 0); + checkOpStatistics(OpType.GET_EC_CODECS, opCount + 1); + + ErasureCodingPolicy newPolicy = + new ErasureCodingPolicy(new ECSchema("rs", 5, 3), 1024 * 1024); + + opCount = getOpStatistics(OpType.ADD_EC_POLICY); + dfs.addErasureCodingPolicies(new ErasureCodingPolicy[] {newPolicy}); + checkStatistics(dfs, readOps, ++writeOps, 0); + checkOpStatistics(OpType.ADD_EC_POLICY, opCount + 1); + + opCount = getOpStatistics(OpType.REMOVE_EC_POLICY); + dfs.removeErasureCodingPolicy("RS-5-3-1024k"); + checkStatistics(dfs, readOps, ++writeOps, 0); + checkOpStatistics(OpType.REMOVE_EC_POLICY, opCount + 1); + + opCount = getOpStatistics(OpType.DISABLE_EC_POLICY); + dfs.disableErasureCodingPolicy("RS-10-4-1024k"); + checkStatistics(dfs, readOps, ++writeOps, 0); + checkOpStatistics(OpType.DISABLE_EC_POLICY, opCount + 1); + } } @SuppressWarnings("ThrowableResultOfMethodCallIgnored")