Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14095. EC: Track Erasure Coding commands in DFS statistics. Contributed by Ayush Saxena. 

(cherry picked from commit f534736867)
This commit is contained in:
Brahma Reddy Battula 2018-11-30 00:18:27 +05:30
parent 1b731de944
commit 0b83c95ff4
3 changed files with 89 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
View File

@ -41,6 +41,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
/** This is for counting distributed file system operations. */
public enum OpType {
ADD_EC_POLICY("op_add_ec_policy"),
ALLOW_SNAPSHOT("op_allow_snapshot"),
APPEND(CommonStatisticNames.OP_APPEND),
CONCAT("op_concat"),
@ -51,10 +52,15 @@ public class DFSOpsCountStatistics extends StorageStatistics {
CREATE_SYM_LINK("op_create_symlink"),
DELETE(CommonStatisticNames.OP_DELETE),
DELETE_SNAPSHOT("op_delete_snapshot"),
DISABLE_EC_POLICY("op_disable_ec_policy"),
DISALLOW_SNAPSHOT("op_disallow_snapshot"),
ENABLE_EC_POLICY("op_enable_ec_policy"),
EXISTS(CommonStatisticNames.OP_EXISTS),
GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"),
GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY),
GET_EC_CODECS("op_get_ec_codecs"),
GET_EC_POLICY("op_get_ec_policy"),
GET_EC_POLICIES("op_get_ec_policies"),
GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"),
GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM),
GET_FILE_LINK_STATUS("op_get_file_link_status"),
@ -76,11 +82,13 @@ public class DFSOpsCountStatistics extends StorageStatistics {
REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL),
REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES),
REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL),
REMOVE_EC_POLICY("op_remove_ec_policy"),
REMOVE_XATTR("op_remove_xattr"),
RENAME(CommonStatisticNames.OP_RENAME),
RENAME_SNAPSHOT("op_rename_snapshot"),
RESOLVE_LINK("op_resolve_link"),
SET_ACL(CommonStatisticNames.OP_SET_ACL),
SET_EC_POLICY("op_set_ec_policy"),
SET_OWNER(CommonStatisticNames.OP_SET_OWNER),
SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION),
SET_REPLICATION("op_set_replication"),
@ -90,6 +98,7 @@ public class DFSOpsCountStatistics extends StorageStatistics {
GET_SNAPSHOT_DIFF("op_get_snapshot_diff"),
GET_SNAPSHOTTABLE_DIRECTORY_LIST("op_get_snapshottable_directory_list"),
TRUNCATE(CommonStatisticNames.OP_TRUNCATE),
UNSET_EC_POLICY("op_unset_ec_policy"),
UNSET_STORAGE_POLICY("op_unset_storage_policy");
private static final Map<String, OpType> SYMBOL_MAP =

18
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -2846,6 +2846,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void setErasureCodingPolicy(final Path path,
final String ecPolicyName) throws IOException {
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.SET_EC_POLICY);
Path absF = fixRelativePart(path);
new FileSystemLinkResolver<Void>() {
@Override
@ -2879,6 +2881,8 @@ public class DistributedFileSystem extends FileSystem
*/
public ErasureCodingPolicy getErasureCodingPolicy(final Path path)
throws IOException {
statistics.incrementReadOps(1);
storageStatistics.incrementOpCounter(OpType.GET_EC_POLICY);
Path absF = fixRelativePart(path);
return new FileSystemLinkResolver<ErasureCodingPolicy>() {
@Override
@ -2910,6 +2914,8 @@ public class DistributedFileSystem extends FileSystem
*/
public Collection<ErasureCodingPolicyInfo> getAllErasureCodingPolicies()
throws IOException {
statistics.incrementReadOps(1);
storageStatistics.incrementOpCounter(OpType.GET_EC_POLICIES);
return Arrays.asList(dfs.getErasureCodingPolicies());
}
@ -2922,6 +2928,8 @@ public class DistributedFileSystem extends FileSystem
*/
public Map<String, String> getAllErasureCodingCodecs()
throws IOException {
statistics.incrementReadOps(1);
storageStatistics.incrementOpCounter(OpType.GET_EC_CODECS);
return dfs.getErasureCodingCodecs();
}
@ -2938,6 +2946,8 @@ public class DistributedFileSystem extends FileSystem
*/
public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
ErasureCodingPolicy[] policies) throws IOException {
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.ADD_EC_POLICY);
return dfs.addErasureCodingPolicies(policies);
}
@ -2949,6 +2959,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void removeErasureCodingPolicy(String ecPolicyName)
throws IOException {
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.REMOVE_EC_POLICY);
dfs.removeErasureCodingPolicy(ecPolicyName);
}
@ -2960,6 +2972,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void enableErasureCodingPolicy(String ecPolicyName)
throws IOException {
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.ENABLE_EC_POLICY);
dfs.enableErasureCodingPolicy(ecPolicyName);
}
@ -2971,6 +2985,8 @@ public class DistributedFileSystem extends FileSystem
*/
public void disableErasureCodingPolicy(String ecPolicyName)
throws IOException {
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.DISABLE_EC_POLICY);
dfs.disableErasureCodingPolicy(ecPolicyName);
}
@ -2981,6 +2997,8 @@ public class DistributedFileSystem extends FileSystem
* @throws IOException
*/
public void unsetErasureCodingPolicy(final Path path) throws IOException {
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.UNSET_EC_POLICY);
Path absF = fixRelativePart(path);
new FileSystemLinkResolver<Void>() {
@Override

61
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
View File

@ -772,7 +772,68 @@ public class TestDistributedFileSystem {
} finally {
if (cluster != null) cluster.shutdown();
}
}
@Test
public void testECStatistics() throws IOException {
try (MiniDFSCluster cluster =
new MiniDFSCluster.Builder(getTestConfiguration()).build()) {
cluster.waitActive();
final DistributedFileSystem dfs = cluster.getFileSystem();
Path dir = new Path("/test");
dfs.mkdirs(dir);
int readOps = 0;
int writeOps = 0;
FileSystem.clearStatistics();
long opCount = getOpStatistics(OpType.ENABLE_EC_POLICY);
dfs.enableErasureCodingPolicy("RS-10-4-1024k");
checkStatistics(dfs, readOps, ++writeOps, 0);
checkOpStatistics(OpType.ENABLE_EC_POLICY, opCount + 1);
opCount = getOpStatistics(OpType.SET_EC_POLICY);
dfs.setErasureCodingPolicy(dir, "RS-10-4-1024k");
checkStatistics(dfs, readOps, ++writeOps, 0);
checkOpStatistics(OpType.SET_EC_POLICY, opCount + 1);
opCount = getOpStatistics(OpType.GET_EC_POLICY);
dfs.getErasureCodingPolicy(dir);
checkStatistics(dfs, ++readOps, writeOps, 0);
checkOpStatistics(OpType.GET_EC_POLICY, opCount + 1);
opCount = getOpStatistics(OpType.UNSET_EC_POLICY);
dfs.unsetErasureCodingPolicy(dir);
checkStatistics(dfs, readOps, ++writeOps, 0);
checkOpStatistics(OpType.UNSET_EC_POLICY, opCount + 1);
opCount = getOpStatistics(OpType.GET_EC_POLICIES);
dfs.getAllErasureCodingPolicies();
checkStatistics(dfs, ++readOps, writeOps, 0);
checkOpStatistics(OpType.GET_EC_POLICIES, opCount + 1);
opCount = getOpStatistics(OpType.GET_EC_CODECS);
dfs.getAllErasureCodingCodecs();
checkStatistics(dfs, ++readOps, writeOps, 0);
checkOpStatistics(OpType.GET_EC_CODECS, opCount + 1);
ErasureCodingPolicy newPolicy =
new ErasureCodingPolicy(new ECSchema("rs", 5, 3), 1024 * 1024);
opCount = getOpStatistics(OpType.ADD_EC_POLICY);
dfs.addErasureCodingPolicies(new ErasureCodingPolicy[] {newPolicy});
checkStatistics(dfs, readOps, ++writeOps, 0);
checkOpStatistics(OpType.ADD_EC_POLICY, opCount + 1);
opCount = getOpStatistics(OpType.REMOVE_EC_POLICY);
dfs.removeErasureCodingPolicy("RS-5-3-1024k");
checkStatistics(dfs, readOps, ++writeOps, 0);
checkOpStatistics(OpType.REMOVE_EC_POLICY, opCount + 1);
opCount = getOpStatistics(OpType.DISABLE_EC_POLICY);
dfs.disableErasureCodingPolicy("RS-10-4-1024k");
checkStatistics(dfs, readOps, ++writeOps, 0);
checkOpStatistics(OpType.DISABLE_EC_POLICY, opCount + 1);
}
}
@SuppressWarnings("ThrowableResultOfMethodCallIgnored")