Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12218. Rename split EC / replicated block metrics in BlockManager.

This commit is contained in:
Andrew Wang 2017-09-07 16:56:35 -07:00
parent 52b894db33
commit 40c2f31f8d
9 changed files with 69 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
View File

@ -778,14 +778,14 @@ public interface ClientProtocol {
* in the filesystem.
*/
@Idempotent
BlocksStats getBlocksStats() throws IOException;
ReplicatedBlockStats getBlocksStats() throws IOException;
/**
* Get statistics pertaining to blocks of type {@link BlockType#STRIPED}
* in the filesystem.
*/
@Idempotent
ECBlockGroupsStats getECBlockGroupsStats() throws IOException;
ECBlockGroupStats getECBlockGroupsStats() throws IOException;
/**
* Get a report on the system's current datanodes.

6
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupsStats.java → hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java
View File

@ -28,14 +28,14 @@ import org.apache.hadoop.classification.InterfaceStability;
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public final class ECBlockGroupsStats {
public final class ECBlockGroupStats {
private final long lowRedundancyBlockGroupsStat;
private final long corruptBlockGroupsStat;
private final long missingBlockGroupsStat;
private final long bytesInFutureBlockGroupsStat;
private final long pendingDeletionBlockGroupsStat;
public ECBlockGroupsStats(long lowRedundancyBlockGroupsStat, long
public ECBlockGroupStats(long lowRedundancyBlockGroupsStat, long
corruptBlockGroupsStat, long missingBlockGroupsStat, long
bytesInFutureBlockGroupsStat, long pendingDeletionBlockGroupsStat) {
this.lowRedundancyBlockGroupsStat = lowRedundancyBlockGroupsStat;
@ -68,7 +68,7 @@ public final class ECBlockGroupsStats {
@Override
public String toString() {
StringBuilder statsBuilder = new StringBuilder();
statsBuilder.append("ECBlockGroupsStats=[")
statsBuilder.append("ECBlockGroupStats=[")
.append("LowRedundancyBlockGroups=").append(
getLowRedundancyBlockGroupsStat())
.append(", CorruptBlockGroups=").append(getCorruptBlockGroupsStat())

4
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlocksStats.java → hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReplicatedBlockStats.java
View File

@ -28,7 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability;
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public final class BlocksStats {
public final class ReplicatedBlockStats {
private final long lowRedundancyBlocksStat;
private final long corruptBlocksStat;
private final long missingBlocksStat;
@ -36,7 +36,7 @@ public final class BlocksStats {
private final long bytesInFutureBlocksStat;
private final long pendingDeletionBlocksStat;
public BlocksStats(long lowRedundancyBlocksStat,
public ReplicatedBlockStats(long lowRedundancyBlocksStat,
long corruptBlocksStat, long missingBlocksStat,
long missingReplicationOneBlocksStat, long bytesInFutureBlocksStat,
long pendingDeletionBlocksStat) {

8
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
View File

@ -61,7 +61,7 @@ import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupsStats;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@ -73,7 +73,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.BlocksStats;
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
@ -695,7 +695,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
public BlocksStats getBlocksStats() throws IOException {
public ReplicatedBlockStats getBlocksStats() throws IOException {
try {
return PBHelperClient.convert(rpcProxy.getFsBlocksStats(null,
VOID_GET_FS_REPLICABLOCKS_STATS_REQUEST));
@ -705,7 +705,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
public ECBlockGroupsStats getECBlockGroupsStats() throws IOException {
public ECBlockGroupStats getECBlockGroupsStats() throws IOException {
try {
return PBHelperClient.convert(rpcProxy.getFsECBlockGroupsStats(null,
VOID_GET_FS_ECBLOCKGROUPS_STATS_REQUEST));

38
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
View File

@ -76,7 +76,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupsStats;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
@ -92,7 +92,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.protocol.BlocksStats;
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeStatus;
@ -1810,17 +1810,17 @@ public class PBHelperClient {
return result;
}
public static BlocksStats convert(
public static ReplicatedBlockStats convert(
GetFsBlocksStatsResponseProto res) {
return new BlocksStats(res.getLowRedundancy(),
return new ReplicatedBlockStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
res.getPendingDeletionBlocks());
}
public static ECBlockGroupsStats convert(
public static ECBlockGroupStats convert(
GetFsECBlockGroupsStatsResponseProto res) {
return new ECBlockGroupsStats(res.getLowRedundancy(),
return new ECBlockGroupStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getBlocksInFuture(), res.getPendingDeletionBlocks());
}
@ -2237,36 +2237,36 @@ public class PBHelperClient {
}
public static GetFsBlocksStatsResponseProto convert(
BlocksStats blocksStats) {
ReplicatedBlockStats replicatedBlockStats) {
GetFsBlocksStatsResponseProto.Builder result =
GetFsBlocksStatsResponseProto.newBuilder();
result.setLowRedundancy(
blocksStats.getLowRedundancyBlocksStat());
replicatedBlockStats.getLowRedundancyBlocksStat());
result.setCorruptBlocks(
blocksStats.getCorruptBlocksStat());
replicatedBlockStats.getCorruptBlocksStat());
result.setMissingBlocks(
blocksStats.getMissingReplicaBlocksStat());
replicatedBlockStats.getMissingReplicaBlocksStat());
result.setMissingReplOneBlocks(
blocksStats.getMissingReplicationOneBlocksStat());
replicatedBlockStats.getMissingReplicationOneBlocksStat());
result.setBlocksInFuture(
blocksStats.getBytesInFutureBlocksStat());
replicatedBlockStats.getBytesInFutureBlocksStat());
result.setPendingDeletionBlocks(
blocksStats.getPendingDeletionBlocksStat());
replicatedBlockStats.getPendingDeletionBlocksStat());
return result.build();
}
public static GetFsECBlockGroupsStatsResponseProto convert(
ECBlockGroupsStats ecBlockGroupsStats) {
ECBlockGroupStats ecBlockGroupStats) {
GetFsECBlockGroupsStatsResponseProto.Builder result =
GetFsECBlockGroupsStatsResponseProto.newBuilder();
result.setLowRedundancy(
ecBlockGroupsStats.getLowRedundancyBlockGroupsStat());
result.setCorruptBlocks(ecBlockGroupsStats.getCorruptBlockGroupsStat());
result.setMissingBlocks(ecBlockGroupsStats.getMissingBlockGroupsStat());
ecBlockGroupStats.getLowRedundancyBlockGroupsStat());
result.setCorruptBlocks(ecBlockGroupStats.getCorruptBlockGroupsStat());
result.setMissingBlocks(ecBlockGroupStats.getMissingBlockGroupsStat());
result.setBlocksInFuture(
ecBlockGroupsStats.getBytesInFutureBlockGroupsStat());
ecBlockGroupStats.getBytesInFutureBlockGroupsStat());
result.setPendingDeletionBlocks(
ecBlockGroupsStats.getPendingDeletionBlockGroupsStat());
ecBlockGroupStats.getPendingDeletionBlockGroupsStat());
return result.build();
}

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -89,8 +89,9 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*;
import org.apache.hadoop.hdfs.protocol.BlocksStats;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupsStats;
import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.server.namenode.metrics.ReplicatedBlocksMBean;
@ -4082,8 +4083,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* <p>
* @see ClientProtocol#getBlocksStats()
*/
BlocksStats getBlocksStats() {
return new BlocksStats(getLowRedundancyReplicatedBlocks(),
ReplicatedBlockStats getBlocksStats() {
return new ReplicatedBlockStats(getLowRedundancyReplicatedBlocks(),
getCorruptReplicatedBlocks(), getMissingReplicatedBlocks(),
getMissingReplicationOneBlocks(), getBytesInFutureReplicatedBlocks(),
getPendingDeletionReplicatedBlocks());
@ -4095,8 +4096,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* <p>
* @see ClientProtocol#getECBlockGroupsStats()
*/
ECBlockGroupsStats getECBlockGroupsStats() {
return new ECBlockGroupsStats(getLowRedundancyECBlockGroups(),
ECBlockGroupStats getECBlockGroupsStats() {
return new ECBlockGroupStats(getLowRedundancyECBlockGroups(),
getCorruptECBlockGroups(), getMissingECBlockGroups(),
getBytesInFutureECBlockGroups(), getPendingDeletionECBlockGroups());
}

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
View File

@ -98,7 +98,7 @@ import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupsStats;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@ -116,7 +116,7 @@ import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException;
import org.apache.hadoop.hdfs.protocol.BlocksStats;
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
@ -1163,14 +1163,14 @@ public class NameNodeRpcServer implements NamenodeProtocols {
}
@Override // ClientProtocol
public BlocksStats getBlocksStats() throws IOException {
public ReplicatedBlockStats getBlocksStats() throws IOException {
checkNNStartup();
namesystem.checkOperation(OperationCategory.READ);
return namesystem.getBlocksStats();
}
@Override // ClientProtocol
public ECBlockGroupsStats getECBlockGroupsStats() throws IOException {
public ECBlockGroupStats getECBlockGroupsStats() throws IOException {
checkNNStartup();
namesystem.checkOperation(OperationCategory.READ);
return namesystem.getECBlockGroupsStats();

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
View File

@ -66,13 +66,13 @@ import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
import org.apache.hadoop.hdfs.protocol.BlocksStats;
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeVolumeInfo;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupsStats;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
@ -534,30 +534,30 @@ public class DFSAdmin extends FsShell {
* minutes. Use "-metaSave" to list of all such blocks and accurate
* counts.
*/
BlocksStats blocksStats = dfs.getClient().getNamenode().getBlocksStats();
ReplicatedBlockStats replicatedBlockStats = dfs.getClient().getNamenode().getBlocksStats();
System.out.println("Replicated Blocks:");
System.out.println("\tUnder replicated blocks: " +
blocksStats.getLowRedundancyBlocksStat());
replicatedBlockStats.getLowRedundancyBlocksStat());
System.out.println("\tBlocks with corrupt replicas: " +
blocksStats.getCorruptBlocksStat());
replicatedBlockStats.getCorruptBlocksStat());
System.out.println("\tMissing blocks: " +
blocksStats.getMissingReplicaBlocksStat());
replicatedBlockStats.getMissingReplicaBlocksStat());
System.out.println("\tMissing blocks (with replication factor 1): " +
blocksStats.getMissingReplicationOneBlocksStat());
replicatedBlockStats.getMissingReplicationOneBlocksStat());
System.out.println("\tPending deletion blocks: " +
blocksStats.getPendingDeletionBlocksStat());
replicatedBlockStats.getPendingDeletionBlocksStat());
ECBlockGroupsStats ecBlockGroupsStats =
ECBlockGroupStats ecBlockGroupStats =
dfs.getClient().getNamenode().getECBlockGroupsStats();
System.out.println("Erasure Coded Block Groups: ");
System.out.println("\tLow redundancy block groups: " +
ecBlockGroupsStats.getLowRedundancyBlockGroupsStat());
ecBlockGroupStats.getLowRedundancyBlockGroupsStat());
System.out.println("\tBlock groups with corrupt internal blocks: " +
ecBlockGroupsStats.getCorruptBlockGroupsStat());
ecBlockGroupStats.getCorruptBlockGroupsStat());
System.out.println("\tMissing block groups: " +
ecBlockGroupsStats.getMissingBlockGroupsStat());
ecBlockGroupStats.getMissingBlockGroupsStat());
System.out.println("\tPending deletion block groups: " +
ecBlockGroupsStats.getPendingDeletionBlockGroupsStat());
ecBlockGroupStats.getPendingDeletionBlockGroupsStat());
System.out.println();

30
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
View File

@ -117,8 +117,8 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupsStats;
import org.apache.hadoop.hdfs.protocol.BlocksStats;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupStats;
import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@ -1667,36 +1667,36 @@ public class DFSTestUtil {
cluster.getFileSystem(0).getUri(),
ClientProtocol.class).getProxy();
long[] aggregatedStats = cluster.getNameNode().getRpcServer().getStats();
BlocksStats blocksStats =
ReplicatedBlockStats replicatedBlockStats =
client.getBlocksStats();
ECBlockGroupsStats ecBlockGroupsStats = client.getECBlockGroupsStats();
ECBlockGroupStats ecBlockGroupStats = client.getECBlockGroupsStats();
assertEquals("Under replicated stats not matching!",
aggregatedStats[ClientProtocol.GET_STATS_LOW_REDUNDANCY_IDX],
aggregatedStats[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX]);
assertEquals("Low redundancy stats not matching!",
aggregatedStats[ClientProtocol.GET_STATS_LOW_REDUNDANCY_IDX],
blocksStats.getLowRedundancyBlocksStat() +
ecBlockGroupsStats.getLowRedundancyBlockGroupsStat());
replicatedBlockStats.getLowRedundancyBlocksStat() +
ecBlockGroupStats.getLowRedundancyBlockGroupsStat());
assertEquals("Corrupt blocks stats not matching!",
aggregatedStats[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX],
blocksStats.getCorruptBlocksStat() +
ecBlockGroupsStats.getCorruptBlockGroupsStat());
replicatedBlockStats.getCorruptBlocksStat() +
ecBlockGroupStats.getCorruptBlockGroupsStat());
assertEquals("Missing blocks stats not matching!",
aggregatedStats[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX],
blocksStats.getMissingReplicaBlocksStat() +
ecBlockGroupsStats.getMissingBlockGroupsStat());
replicatedBlockStats.getMissingReplicaBlocksStat() +
ecBlockGroupStats.getMissingBlockGroupsStat());
assertEquals("Missing blocks with replication factor one not matching!",
aggregatedStats[ClientProtocol.GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX],
blocksStats.getMissingReplicationOneBlocksStat());
replicatedBlockStats.getMissingReplicationOneBlocksStat());
assertEquals("Bytes in future blocks stats not matching!",
aggregatedStats[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX],
blocksStats.getBytesInFutureBlocksStat() +
ecBlockGroupsStats.getBytesInFutureBlockGroupsStat());
replicatedBlockStats.getBytesInFutureBlocksStat() +
ecBlockGroupStats.getBytesInFutureBlockGroupsStat());
assertEquals("Pending deletion blocks stats not matching!",
aggregatedStats[ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX],
blocksStats.getPendingDeletionBlocksStat() +
ecBlockGroupsStats.getPendingDeletionBlockGroupsStat());
replicatedBlockStats.getPendingDeletionBlocksStat() +
ecBlockGroupStats.getPendingDeletionBlockGroupsStat());
}
/**