Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13305. Define common statistics names across schemes. Contributed by Mingliang Liu.

This commit is contained in:
Jitendra Pandey 2016-07-01 15:34:03 -07:00
parent d2559ca307
commit 06b89f0d14
6 changed files with 145 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java
View File

@ -118,6 +118,11 @@ public class FileSystemStorageStatistics extends StorageStatistics {
this.stats = stats;
}
@Override
public String getScheme() {
return stats.getScheme();
}
@Override
public Iterator<LongStatistic> getLongStatistics() {
return new LongStatisticIterator(stats.getData());

53
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
View File

@ -27,6 +27,51 @@ import java.util.Iterator;
*/
@InterfaceAudience.Public
public abstract class StorageStatistics {
/**
* These are common statistic names.
*
* The following names are considered general and preserved across different
* StorageStatistics classes. When implementing a new StorageStatistics, it is
* highly recommended to use the common statistic names.
*
* When adding new common statistic name constants, please make them unique.
* By convention, they are implicitly unique:
* - the name of the constants are uppercase, words separated by underscores.
* - the value of the constants are lowercase of the constant names.
*/
public interface CommonStatisticNames {
// The following names are for file system operation invocations
String OP_APPEND = "op_append";
String OP_COPY_FROM_LOCAL_FILE = "op_copy_from_local_file";
String OP_CREATE = "op_create";
String OP_CREATE_NON_RECURSIVE = "op_create_non_recursive";
String OP_DELETE = "op_delete";
String OP_EXISTS = "op_exists";
String OP_GET_CONTENT_SUMMARY = "op_get_content_summary";
String OP_GET_FILE_CHECKSUM = "op_get_file_checksum";
String OP_GET_FILE_STATUS = "op_get_file_status";
String OP_GET_STATUS = "op_get_status";
String OP_GLOB_STATUS = "op_glob_status";
String OP_IS_FILE = "op_is_file";
String OP_IS_DIRECTORY = "op_is_directory";
String OP_LIST_FILES = "op_list_files";
String OP_LIST_LOCATED_STATUS = "op_list_located_status";
String OP_LIST_STATUS = "op_list_status";
String OP_MKDIRS = "op_mkdirs";
String OP_MODIFY_ACL_ENTRIES = "op_modify_acl_entries";
String OP_OPEN = "op_open";
String OP_REMOVE_ACL = "op_remove_acl";
String OP_REMOVE_ACL_ENTRIES = "op_remove_acl_entries";
String OP_REMOVE_DEFAULT_ACL = "op_remove_default_acl";
String OP_RENAME = "op_rename";
String OP_SET_ACL = "op_set_acl";
String OP_SET_OWNER = "op_set_owner";
String OP_SET_PERMISSION = "op_set_permission";
String OP_SET_TIMES = "op_set_times";
String OP_TRUNCATE = "op_truncate";
}
/**
* A 64-bit storage statistic.
*/
@ -67,6 +112,14 @@ public abstract class StorageStatistics {
return name;
}
/**
* @return the associated file system scheme if this is scheme specific,
* else return null.
*/
public String getScheme() {
return null;
}
/**
* Get an iterator over all the currently tracked long statistics.
*

102
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import java.util.EnumMap;
import java.util.Iterator;
@ -37,55 +38,55 @@ import java.util.concurrent.atomic.AtomicLong;
*/
public class DFSOpsCountStatistics extends StorageStatistics {
/** This is for counting file system operations. */
/** This is for counting distributed file system operations. */
public enum OpType {
ALLOW_SNAPSHOT("allowSnapshot"),
APPEND("append"),
CONCAT("concat"),
COPY_FROM_LOCAL_FILE("copyFromLocalFile"),
CREATE("create"),
CREATE_NON_RECURSIVE("createNonRecursive"),
CREATE_SNAPSHOT("createSnapshot"),
CREATE_SYM_LINK("createSymlink"),
DELETE("delete"),
DELETE_SNAPSHOT("deleteSnapshot"),
DISALLOW_SNAPSHOT("disallowSnapshot"),
EXISTS("exists"),
GET_BYTES_WITH_FUTURE_GS("getBytesWithFutureGenerationStamps"),
GET_CONTENT_SUMMARY("getContentSummary"),
GET_FILE_BLOCK_LOCATIONS("getFileBlockLocations"),
GET_FILE_CHECKSUM("getFileChecksum"),
GET_FILE_LINK_STATUS("getFileLinkStatus"),
GET_FILE_STATUS("getFileStatus"),
GET_LINK_TARGET("getLinkTarget"),
GET_QUOTA_USAGE("getQuotaUsage"),
GET_STATUS("getStatus"),
GET_STORAGE_POLICIES("getStoragePolicies"),
GET_STORAGE_POLICY("getStoragePolicy"),
GET_XATTR("getXAttr"),
LIST_LOCATED_STATUS("listLocatedStatus"),
LIST_STATUS("listStatus"),
MKDIRS("mkdirs"),
MODIFY_ACL_ENTRIES("modifyAclEntries"),
OPEN("open"),
PRIMITIVE_CREATE("primitiveCreate"),
PRIMITIVE_MKDIR("primitiveMkdir"),
REMOVE_ACL("removeAcl"),
REMOVE_ACL_ENTRIES("removeAclEntries"),
REMOVE_DEFAULT_ACL("removeDefaultAcl"),
REMOVE_XATTR("removeXAttr"),
RENAME("rename"),
RENAME_SNAPSHOT("renameSnapshot"),
RESOLVE_LINK("resolveLink"),
SET_ACL("setAcl"),
SET_OWNER("setOwner"),
SET_PERMISSION("setPermission"),
SET_REPLICATION("setReplication"),
SET_STORAGE_POLICY("setStoragePolicy"),
SET_TIMES("setTimes"),
SET_XATTR("setXAttr"),
TRUNCATE("truncate"),
UNSET_STORAGE_POLICY("unsetStoragePolicy");
ALLOW_SNAPSHOT("op_allow_snapshot"),
APPEND(CommonStatisticNames.OP_APPEND),
CONCAT("op_concat"),
COPY_FROM_LOCAL_FILE(CommonStatisticNames.OP_COPY_FROM_LOCAL_FILE),
CREATE(CommonStatisticNames.OP_CREATE),
CREATE_NON_RECURSIVE(CommonStatisticNames.OP_CREATE_NON_RECURSIVE),
CREATE_SNAPSHOT("op_create_snapshot"),
CREATE_SYM_LINK("op_create_symlink"),
DELETE(CommonStatisticNames.OP_DELETE),
DELETE_SNAPSHOT("op_delete_snapshot"),
DISALLOW_SNAPSHOT("op_disallow_snapshot"),
EXISTS(CommonStatisticNames.OP_EXISTS),
GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"),
GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY),
GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"),
GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM),
GET_FILE_LINK_STATUS("op_get_file_link_status"),
GET_FILE_STATUS(CommonStatisticNames.OP_GET_FILE_STATUS),
GET_LINK_TARGET("op_get_link_target"),
GET_QUOTA_USAGE("op_get_quota_usage"),
GET_STATUS(CommonStatisticNames.OP_GET_STATUS),
GET_STORAGE_POLICIES("op_get_storage_policies"),
GET_STORAGE_POLICY("op_get_storage_policy"),
GET_XATTR("op_get_xattr"),
LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS),
LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS),
MKDIRS(CommonStatisticNames.OP_MKDIRS),
MODIFY_ACL_ENTRIES(CommonStatisticNames.OP_MODIFY_ACL_ENTRIES),
OPEN(CommonStatisticNames.OP_OPEN),
PRIMITIVE_CREATE("op_primitive_create"),
PRIMITIVE_MKDIR("op_primitive_mkdir"),
REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL),
REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES),
REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL),
REMOVE_XATTR("op_remove_xattr"),
RENAME(CommonStatisticNames.OP_RENAME),
RENAME_SNAPSHOT("op_rename_snapshot"),
RESOLVE_LINK("op_resolve_link"),
SET_ACL(CommonStatisticNames.OP_SET_ACL),
SET_OWNER(CommonStatisticNames.OP_SET_OWNER),
SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION),
SET_REPLICATION("op_set_replication"),
SET_STORAGE_POLICY("op_set_storagePolicy"),
SET_TIMES(CommonStatisticNames.OP_SET_TIMES),
SET_XATTR("op_set_xattr"),
TRUNCATE(CommonStatisticNames.OP_TRUNCATE),
UNSET_STORAGE_POLICY("op_unset_storage_policy");
private final String symbol;
@ -149,6 +150,11 @@ public class DFSOpsCountStatistics extends StorageStatistics {
}
}
@Override
public String getScheme() {
return HdfsConstants.HDFS_URI_SCHEME;
}
@Override
public Iterator<LongStatistic> getLongStatistics() {
return new LongIterator();

15
hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java
View File

@ -31,8 +31,10 @@ import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@ -66,6 +68,19 @@ public class TestDFSOpsCountStatistics {
}
}
/**
* This is to test the the {@link OpType} symbols are unique.
*/
@Test
public void testOpTypeSymbolsAreUnique() {
final Set<String> opTypeSymbols = new HashSet<>();
for (OpType opType : OpType.values()) {
assertFalse(opTypeSymbols.contains(opType.getSymbol()));
opTypeSymbols.add(opType.getSymbol());
}
assertEquals(OpType.values().length, opTypeSymbols.size());
}
@Test
public void testGetLongStatistics() {
short iterations = 0; // number of the iter.hasNext()

5
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java
View File

@ -86,6 +86,11 @@ public class S3AStorageStatistics extends StorageStatistics {
}
}
@Override
public String getScheme() {
return "s3a";
}
@Override
public Iterator<LongStatistic> getLongStatistics() {
return new LongIterator();

24
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
View File

@ -18,6 +18,8 @@
package org.apache.hadoop.fs.s3a;
import org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames;
/**
* Statistic which are collected in S3A.
* These statistics are available at a low level in {@link S3AStorageStatistics}
@ -38,27 +40,27 @@ public enum Statistic {
FILES_DELETED("files_deleted",
"Total number of files deleted from the object store."),
IGNORED_ERRORS("ignored_errors", "Errors caught and ignored"),
INVOCATION_COPY_FROM_LOCAL_FILE("invocations_copyfromlocalfile",
INVOCATION_COPY_FROM_LOCAL_FILE(CommonStatisticNames.OP_COPY_FROM_LOCAL_FILE,
"Calls of copyFromLocalFile()"),
INVOCATION_EXISTS("invocations_exists",
INVOCATION_EXISTS(CommonStatisticNames.OP_EXISTS,
"Calls of exists()"),
INVOCATION_GET_FILE_STATUS("invocations_getfilestatus",
INVOCATION_GET_FILE_STATUS(CommonStatisticNames.OP_GET_FILE_STATUS,
"Calls of getFileStatus()"),
INVOCATION_GLOB_STATUS("invocations_globstatus",
INVOCATION_GLOB_STATUS(CommonStatisticNames.OP_GLOB_STATUS,
"Calls of globStatus()"),
INVOCATION_IS_DIRECTORY("invocations_is_directory",
INVOCATION_IS_DIRECTORY(CommonStatisticNames.OP_IS_DIRECTORY,
"Calls of isDirectory()"),
INVOCATION_IS_FILE("invocations_is_file",
INVOCATION_IS_FILE(CommonStatisticNames.OP_IS_FILE,
"Calls of isFile()"),
INVOCATION_LIST_FILES("invocations_listfiles",
INVOCATION_LIST_FILES(CommonStatisticNames.OP_LIST_FILES,
"Calls of listFiles()"),
INVOCATION_LIST_LOCATED_STATUS("invocations_listlocatedstatus",
INVOCATION_LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS,
"Calls of listLocatedStatus()"),
INVOCATION_LIST_STATUS("invocations_liststatus",
INVOCATION_LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS,
"Calls of listStatus()"),
INVOCATION_MKDIRS("invocations_mdkirs",
INVOCATION_MKDIRS(CommonStatisticNames.OP_MKDIRS,
"Calls of mkdirs()"),
INVOCATION_RENAME("invocations_rename",
INVOCATION_RENAME(CommonStatisticNames.OP_RENAME,
"Calls of rename()"),
OBJECT_COPY_REQUESTS("object_copy_requests", "Object copy requests"),
OBJECT_DELETE_REQUESTS("object_delete_requests", "Object delete requests"),