HADOOP-13305. Define common statistics names across schemes. Contributed by Mingliang Liu.
This commit is contained in:
Jitendra Pandey
parent
c35a5a7a8d
commit
aa42c7a6dd
|
|
@ -118,6 +118,11 @@ public class FileSystemStorageStatistics extends StorageStatistics {
|
|||
this.stats = stats;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getScheme() {
|
||||
return stats.getScheme();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<LongStatistic> getLongStatistics() {
|
||||
return new LongStatisticIterator(stats.getData());
|
||||
|
|
|
|||
|
|
@ -27,6 +27,51 @@ import java.util.Iterator;
|
|||
*/
|
||||
@InterfaceAudience.Public
|
||||
public abstract class StorageStatistics {
|
||||
|
||||
/**
|
||||
* These are common statistic names.
|
||||
*
|
||||
* The following names are considered general and preserved across different
|
||||
* StorageStatistics classes. When implementing a new StorageStatistics, it is
|
||||
* highly recommended to use the common statistic names.
|
||||
*
|
||||
* When adding new common statistic name constants, please make them unique.
|
||||
* By convention, they are implicitly unique:
|
||||
* - the name of the constants are uppercase, words separated by underscores.
|
||||
* - the value of the constants are lowercase of the constant names.
|
||||
*/
|
||||
public interface CommonStatisticNames {
|
||||
// The following names are for file system operation invocations
|
||||
String OP_APPEND = "op_append";
|
||||
String OP_COPY_FROM_LOCAL_FILE = "op_copy_from_local_file";
|
||||
String OP_CREATE = "op_create";
|
||||
String OP_CREATE_NON_RECURSIVE = "op_create_non_recursive";
|
||||
String OP_DELETE = "op_delete";
|
||||
String OP_EXISTS = "op_exists";
|
||||
String OP_GET_CONTENT_SUMMARY = "op_get_content_summary";
|
||||
String OP_GET_FILE_CHECKSUM = "op_get_file_checksum";
|
||||
String OP_GET_FILE_STATUS = "op_get_file_status";
|
||||
String OP_GET_STATUS = "op_get_status";
|
||||
String OP_GLOB_STATUS = "op_glob_status";
|
||||
String OP_IS_FILE = "op_is_file";
|
||||
String OP_IS_DIRECTORY = "op_is_directory";
|
||||
String OP_LIST_FILES = "op_list_files";
|
||||
String OP_LIST_LOCATED_STATUS = "op_list_located_status";
|
||||
String OP_LIST_STATUS = "op_list_status";
|
||||
String OP_MKDIRS = "op_mkdirs";
|
||||
String OP_MODIFY_ACL_ENTRIES = "op_modify_acl_entries";
|
||||
String OP_OPEN = "op_open";
|
||||
String OP_REMOVE_ACL = "op_remove_acl";
|
||||
String OP_REMOVE_ACL_ENTRIES = "op_remove_acl_entries";
|
||||
String OP_REMOVE_DEFAULT_ACL = "op_remove_default_acl";
|
||||
String OP_RENAME = "op_rename";
|
||||
String OP_SET_ACL = "op_set_acl";
|
||||
String OP_SET_OWNER = "op_set_owner";
|
||||
String OP_SET_PERMISSION = "op_set_permission";
|
||||
String OP_SET_TIMES = "op_set_times";
|
||||
String OP_TRUNCATE = "op_truncate";
|
||||
}
|
||||
|
||||
/**
|
||||
* A 64-bit storage statistic.
|
||||
*/
|
||||
|
|
@ -67,6 +112,14 @@ public abstract class StorageStatistics {
|
|||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the associated file system scheme if this is scheme specific,
|
||||
* else return null.
|
||||
*/
|
||||
public String getScheme() {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an iterator over all the currently tracked long statistics.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
package org.apache.hadoop.hdfs;
|
||||
|
||||
import org.apache.hadoop.fs.StorageStatistics;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||
|
||||
import java.util.EnumMap;
|
||||
import java.util.Iterator;
|
||||
|
|
@ -37,55 +38,55 @@ import java.util.concurrent.atomic.AtomicLong;
|
|||
*/
|
||||
public class DFSOpsCountStatistics extends StorageStatistics {
|
||||
|
||||
/** This is for counting file system operations. */
|
||||
/** This is for counting distributed file system operations. */
|
||||
public enum OpType {
|
||||
ALLOW_SNAPSHOT("allowSnapshot"),
|
||||
APPEND("append"),
|
||||
CONCAT("concat"),
|
||||
COPY_FROM_LOCAL_FILE("copyFromLocalFile"),
|
||||
CREATE("create"),
|
||||
CREATE_NON_RECURSIVE("createNonRecursive"),
|
||||
CREATE_SNAPSHOT("createSnapshot"),
|
||||
CREATE_SYM_LINK("createSymlink"),
|
||||
DELETE("delete"),
|
||||
DELETE_SNAPSHOT("deleteSnapshot"),
|
||||
DISALLOW_SNAPSHOT("disallowSnapshot"),
|
||||
EXISTS("exists"),
|
||||
GET_BYTES_WITH_FUTURE_GS("getBytesWithFutureGenerationStamps"),
|
||||
GET_CONTENT_SUMMARY("getContentSummary"),
|
||||
GET_FILE_BLOCK_LOCATIONS("getFileBlockLocations"),
|
||||
GET_FILE_CHECKSUM("getFileChecksum"),
|
||||
GET_FILE_LINK_STATUS("getFileLinkStatus"),
|
||||
GET_FILE_STATUS("getFileStatus"),
|
||||
GET_LINK_TARGET("getLinkTarget"),
|
||||
GET_QUOTA_USAGE("getQuotaUsage"),
|
||||
GET_STATUS("getStatus"),
|
||||
GET_STORAGE_POLICIES("getStoragePolicies"),
|
||||
GET_STORAGE_POLICY("getStoragePolicy"),
|
||||
GET_XATTR("getXAttr"),
|
||||
LIST_LOCATED_STATUS("listLocatedStatus"),
|
||||
LIST_STATUS("listStatus"),
|
||||
MKDIRS("mkdirs"),
|
||||
MODIFY_ACL_ENTRIES("modifyAclEntries"),
|
||||
OPEN("open"),
|
||||
PRIMITIVE_CREATE("primitiveCreate"),
|
||||
PRIMITIVE_MKDIR("primitiveMkdir"),
|
||||
REMOVE_ACL("removeAcl"),
|
||||
REMOVE_ACL_ENTRIES("removeAclEntries"),
|
||||
REMOVE_DEFAULT_ACL("removeDefaultAcl"),
|
||||
REMOVE_XATTR("removeXAttr"),
|
||||
RENAME("rename"),
|
||||
RENAME_SNAPSHOT("renameSnapshot"),
|
||||
RESOLVE_LINK("resolveLink"),
|
||||
SET_ACL("setAcl"),
|
||||
SET_OWNER("setOwner"),
|
||||
SET_PERMISSION("setPermission"),
|
||||
SET_REPLICATION("setReplication"),
|
||||
SET_STORAGE_POLICY("setStoragePolicy"),
|
||||
SET_TIMES("setTimes"),
|
||||
SET_XATTR("setXAttr"),
|
||||
TRUNCATE("truncate"),
|
||||
UNSET_STORAGE_POLICY("unsetStoragePolicy");
|
||||
ALLOW_SNAPSHOT("op_allow_snapshot"),
|
||||
APPEND(CommonStatisticNames.OP_APPEND),
|
||||
CONCAT("op_concat"),
|
||||
COPY_FROM_LOCAL_FILE(CommonStatisticNames.OP_COPY_FROM_LOCAL_FILE),
|
||||
CREATE(CommonStatisticNames.OP_CREATE),
|
||||
CREATE_NON_RECURSIVE(CommonStatisticNames.OP_CREATE_NON_RECURSIVE),
|
||||
CREATE_SNAPSHOT("op_create_snapshot"),
|
||||
CREATE_SYM_LINK("op_create_symlink"),
|
||||
DELETE(CommonStatisticNames.OP_DELETE),
|
||||
DELETE_SNAPSHOT("op_delete_snapshot"),
|
||||
DISALLOW_SNAPSHOT("op_disallow_snapshot"),
|
||||
EXISTS(CommonStatisticNames.OP_EXISTS),
|
||||
GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"),
|
||||
GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY),
|
||||
GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"),
|
||||
GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM),
|
||||
GET_FILE_LINK_STATUS("op_get_file_link_status"),
|
||||
GET_FILE_STATUS(CommonStatisticNames.OP_GET_FILE_STATUS),
|
||||
GET_LINK_TARGET("op_get_link_target"),
|
||||
GET_QUOTA_USAGE("op_get_quota_usage"),
|
||||
GET_STATUS(CommonStatisticNames.OP_GET_STATUS),
|
||||
GET_STORAGE_POLICIES("op_get_storage_policies"),
|
||||
GET_STORAGE_POLICY("op_get_storage_policy"),
|
||||
GET_XATTR("op_get_xattr"),
|
||||
LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS),
|
||||
LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS),
|
||||
MKDIRS(CommonStatisticNames.OP_MKDIRS),
|
||||
MODIFY_ACL_ENTRIES(CommonStatisticNames.OP_MODIFY_ACL_ENTRIES),
|
||||
OPEN(CommonStatisticNames.OP_OPEN),
|
||||
PRIMITIVE_CREATE("op_primitive_create"),
|
||||
PRIMITIVE_MKDIR("op_primitive_mkdir"),
|
||||
REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL),
|
||||
REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES),
|
||||
REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL),
|
||||
REMOVE_XATTR("op_remove_xattr"),
|
||||
RENAME(CommonStatisticNames.OP_RENAME),
|
||||
RENAME_SNAPSHOT("op_rename_snapshot"),
|
||||
RESOLVE_LINK("op_resolve_link"),
|
||||
SET_ACL(CommonStatisticNames.OP_SET_ACL),
|
||||
SET_OWNER(CommonStatisticNames.OP_SET_OWNER),
|
||||
SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION),
|
||||
SET_REPLICATION("op_set_replication"),
|
||||
SET_STORAGE_POLICY("op_set_storagePolicy"),
|
||||
SET_TIMES(CommonStatisticNames.OP_SET_TIMES),
|
||||
SET_XATTR("op_set_xattr"),
|
||||
TRUNCATE(CommonStatisticNames.OP_TRUNCATE),
|
||||
UNSET_STORAGE_POLICY("op_unset_storage_policy");
|
||||
|
||||
private final String symbol;
|
||||
|
||||
|
|
@ -149,6 +150,11 @@ public class DFSOpsCountStatistics extends StorageStatistics {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getScheme() {
|
||||
return HdfsConstants.HDFS_URI_SCHEME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<LongStatistic> getLongStatistics() {
|
||||
return new LongIterator();
|
||||
|
|
|
|||
|
|
@ -31,8 +31,10 @@ import org.junit.rules.ExpectedException;
|
|||
import org.junit.rules.Timeout;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
|
|
@ -66,6 +68,19 @@ public class TestDFSOpsCountStatistics {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This is to test the the {@link OpType} symbols are unique.
|
||||
*/
|
||||
@Test
|
||||
public void testOpTypeSymbolsAreUnique() {
|
||||
final Set<String> opTypeSymbols = new HashSet<>();
|
||||
for (OpType opType : OpType.values()) {
|
||||
assertFalse(opTypeSymbols.contains(opType.getSymbol()));
|
||||
opTypeSymbols.add(opType.getSymbol());
|
||||
}
|
||||
assertEquals(OpType.values().length, opTypeSymbols.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetLongStatistics() {
|
||||
short iterations = 0; // number of the iter.hasNext()
|
||||
|
|
|
|||
|
|
@ -86,6 +86,11 @@ public class S3AStorageStatistics extends StorageStatistics {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getScheme() {
|
||||
return "s3a";
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<LongStatistic> getLongStatistics() {
|
||||
return new LongIterator();
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@
|
|||
|
||||
package org.apache.hadoop.fs.s3a;
|
||||
|
||||
import org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames;
|
||||
|
||||
/**
|
||||
* Statistic which are collected in S3A.
|
||||
* These statistics are available at a low level in {@link S3AStorageStatistics}
|
||||
|
|
@ -38,27 +40,27 @@ public enum Statistic {
|
|||
FILES_DELETED("files_deleted",
|
||||
"Total number of files deleted from the object store."),
|
||||
IGNORED_ERRORS("ignored_errors", "Errors caught and ignored"),
|
||||
INVOCATION_COPY_FROM_LOCAL_FILE("invocations_copyfromlocalfile",
|
||||
INVOCATION_COPY_FROM_LOCAL_FILE(CommonStatisticNames.OP_COPY_FROM_LOCAL_FILE,
|
||||
"Calls of copyFromLocalFile()"),
|
||||
INVOCATION_EXISTS("invocations_exists",
|
||||
INVOCATION_EXISTS(CommonStatisticNames.OP_EXISTS,
|
||||
"Calls of exists()"),
|
||||
INVOCATION_GET_FILE_STATUS("invocations_getfilestatus",
|
||||
INVOCATION_GET_FILE_STATUS(CommonStatisticNames.OP_GET_FILE_STATUS,
|
||||
"Calls of getFileStatus()"),
|
||||
INVOCATION_GLOB_STATUS("invocations_globstatus",
|
||||
INVOCATION_GLOB_STATUS(CommonStatisticNames.OP_GLOB_STATUS,
|
||||
"Calls of globStatus()"),
|
||||
INVOCATION_IS_DIRECTORY("invocations_is_directory",
|
||||
INVOCATION_IS_DIRECTORY(CommonStatisticNames.OP_IS_DIRECTORY,
|
||||
"Calls of isDirectory()"),
|
||||
INVOCATION_IS_FILE("invocations_is_file",
|
||||
INVOCATION_IS_FILE(CommonStatisticNames.OP_IS_FILE,
|
||||
"Calls of isFile()"),
|
||||
INVOCATION_LIST_FILES("invocations_listfiles",
|
||||
INVOCATION_LIST_FILES(CommonStatisticNames.OP_LIST_FILES,
|
||||
"Calls of listFiles()"),
|
||||
INVOCATION_LIST_LOCATED_STATUS("invocations_listlocatedstatus",
|
||||
INVOCATION_LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS,
|
||||
"Calls of listLocatedStatus()"),
|
||||
INVOCATION_LIST_STATUS("invocations_liststatus",
|
||||
INVOCATION_LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS,
|
||||
"Calls of listStatus()"),
|
||||
INVOCATION_MKDIRS("invocations_mdkirs",
|
||||
INVOCATION_MKDIRS(CommonStatisticNames.OP_MKDIRS,
|
||||
"Calls of mkdirs()"),
|
||||
INVOCATION_RENAME("invocations_rename",
|
||||
INVOCATION_RENAME(CommonStatisticNames.OP_RENAME,
|
||||
"Calls of rename()"),
|
||||
OBJECT_COPY_REQUESTS("object_copy_requests", "Object copy requests"),
|
||||
OBJECT_DELETE_REQUESTS("object_delete_requests", "Object delete requests"),
|
||||
|
|
|
|||
Loading…
Reference in New Issue