HDFS-13658. Expose HighestPriorityLowRedundancy blocks statistics. Contributed by Kitti Nanasi.
This commit is contained in:
parent
ff06bd1be8
commit
9499df7b81
|
@ -244,6 +244,8 @@ Each metrics record contains tags such as HAState and Hostname as additional inf
|
|||
| `StaleDataNodes` | Current number of DataNodes marked stale due to delayed heartbeat |
|
||||
| `NumStaleStorages` | Number of storages marked as content stale (after NameNode restart/failover before first block report is received) |
|
||||
| `MissingReplOneBlocks` | Current number of missing blocks with replication factor 1 |
|
||||
| `HighestPriorityLowRedundancyReplicatedBlocks` | Current number of non-corrupt, low redundancy replicated blocks with the highest risk of loss (have 0 or 1 replica). Will be recovered with the highest priority. |
|
||||
| `HighestPriorityLowRedundancyECBlocks` | Current number of non-corrupt, low redundancy EC blocks with the highest risk of loss. Will be recovered with the highest priority. |
|
||||
| `NumFilesUnderConstruction` | Current number of files under construction |
|
||||
| `NumActiveClients` | Current number of active clients holding lease |
|
||||
| `HAState` | (HA-only) Current state of the NameNode: initializing or active or standby or stopping state |
|
||||
|
|
|
@ -34,15 +34,26 @@ public final class ECBlockGroupStats {
|
|||
private final long missingBlockGroups;
|
||||
private final long bytesInFutureBlockGroups;
|
||||
private final long pendingDeletionBlocks;
|
||||
private final Long highestPriorityLowRedundancyBlocks;
|
||||
|
||||
public ECBlockGroupStats(long lowRedundancyBlockGroups,
|
||||
long corruptBlockGroups, long missingBlockGroups,
|
||||
long bytesInFutureBlockGroups, long pendingDeletionBlocks) {
|
||||
this(lowRedundancyBlockGroups, corruptBlockGroups, missingBlockGroups,
|
||||
bytesInFutureBlockGroups, pendingDeletionBlocks, null);
|
||||
}
|
||||
|
||||
public ECBlockGroupStats(long lowRedundancyBlockGroups,
|
||||
long corruptBlockGroups, long missingBlockGroups,
|
||||
long bytesInFutureBlockGroups, long pendingDeletionBlocks,
|
||||
Long highestPriorityLowRedundancyBlocks) {
|
||||
this.lowRedundancyBlockGroups = lowRedundancyBlockGroups;
|
||||
this.corruptBlockGroups = corruptBlockGroups;
|
||||
this.missingBlockGroups = missingBlockGroups;
|
||||
this.bytesInFutureBlockGroups = bytesInFutureBlockGroups;
|
||||
this.pendingDeletionBlocks = pendingDeletionBlocks;
|
||||
this.highestPriorityLowRedundancyBlocks
|
||||
= highestPriorityLowRedundancyBlocks;
|
||||
}
|
||||
|
||||
public long getBytesInFutureBlockGroups() {
|
||||
|
@ -65,6 +76,14 @@ public final class ECBlockGroupStats {
|
|||
return pendingDeletionBlocks;
|
||||
}
|
||||
|
||||
public boolean hasHighestPriorityLowRedundancyBlocks() {
|
||||
return getHighestPriorityLowRedundancyBlocks() != null;
|
||||
}
|
||||
|
||||
public Long getHighestPriorityLowRedundancyBlocks() {
|
||||
return highestPriorityLowRedundancyBlocks;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder statsBuilder = new StringBuilder();
|
||||
|
@ -76,8 +95,12 @@ public final class ECBlockGroupStats {
|
|||
.append(", BytesInFutureBlockGroups=").append(
|
||||
getBytesInFutureBlockGroups())
|
||||
.append(", PendingDeletionBlocks=").append(
|
||||
getPendingDeletionBlocks())
|
||||
.append("]");
|
||||
getPendingDeletionBlocks());
|
||||
if (hasHighestPriorityLowRedundancyBlocks()) {
|
||||
statsBuilder.append(", HighestPriorityLowRedundancyBlocks=")
|
||||
.append(getHighestPriorityLowRedundancyBlocks());
|
||||
}
|
||||
statsBuilder.append("]");
|
||||
return statsBuilder.toString();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -35,17 +35,29 @@ public final class ReplicatedBlockStats {
|
|||
private final long missingReplicationOneBlocks;
|
||||
private final long bytesInFutureBlocks;
|
||||
private final long pendingDeletionBlocks;
|
||||
private final Long highestPriorityLowRedundancyBlocks;
|
||||
|
||||
public ReplicatedBlockStats(long lowRedundancyBlocks,
|
||||
long corruptBlocks, long missingBlocks,
|
||||
long missingReplicationOneBlocks, long bytesInFutureBlocks,
|
||||
long pendingDeletionBlocks) {
|
||||
this(lowRedundancyBlocks, corruptBlocks, missingBlocks,
|
||||
missingReplicationOneBlocks, bytesInFutureBlocks, pendingDeletionBlocks,
|
||||
null);
|
||||
}
|
||||
|
||||
public ReplicatedBlockStats(long lowRedundancyBlocks,
|
||||
long corruptBlocks, long missingBlocks,
|
||||
long missingReplicationOneBlocks, long bytesInFutureBlocks,
|
||||
long pendingDeletionBlocks, Long highestPriorityLowRedundancyBlocks) {
|
||||
this.lowRedundancyBlocks = lowRedundancyBlocks;
|
||||
this.corruptBlocks = corruptBlocks;
|
||||
this.missingBlocks = missingBlocks;
|
||||
this.missingReplicationOneBlocks = missingReplicationOneBlocks;
|
||||
this.bytesInFutureBlocks = bytesInFutureBlocks;
|
||||
this.pendingDeletionBlocks = pendingDeletionBlocks;
|
||||
this.highestPriorityLowRedundancyBlocks
|
||||
= highestPriorityLowRedundancyBlocks;
|
||||
}
|
||||
|
||||
public long getLowRedundancyBlocks() {
|
||||
|
@ -72,6 +84,14 @@ public final class ReplicatedBlockStats {
|
|||
return pendingDeletionBlocks;
|
||||
}
|
||||
|
||||
public boolean hasHighestPriorityLowRedundancyBlocks() {
|
||||
return getHighestPriorityLowRedundancyBlocks() != null;
|
||||
}
|
||||
|
||||
public Long getHighestPriorityLowRedundancyBlocks(){
|
||||
return highestPriorityLowRedundancyBlocks;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder statsBuilder = new StringBuilder();
|
||||
|
@ -83,8 +103,12 @@ public final class ReplicatedBlockStats {
|
|||
getMissingReplicationOneBlocks())
|
||||
.append(", BytesInFutureBlocks=").append(getBytesInFutureBlocks())
|
||||
.append(", PendingDeletionBlocks=").append(
|
||||
getPendingDeletionBlocks())
|
||||
.append("]");
|
||||
getPendingDeletionBlocks());
|
||||
if (hasHighestPriorityLowRedundancyBlocks()) {
|
||||
statsBuilder.append(", HighestPriorityLowRedundancyBlocks=").append(
|
||||
getHighestPriorityLowRedundancyBlocks());
|
||||
}
|
||||
statsBuilder.append("]");
|
||||
return statsBuilder.toString();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1990,6 +1990,13 @@ public class PBHelperClient {
|
|||
|
||||
public static ReplicatedBlockStats convert(
|
||||
GetFsReplicatedBlockStatsResponseProto res) {
|
||||
if (res.hasHighestPrioLowRedundancyBlocks()) {
|
||||
return new ReplicatedBlockStats(res.getLowRedundancy(),
|
||||
res.getCorruptBlocks(), res.getMissingBlocks(),
|
||||
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
|
||||
res.getPendingDeletionBlocks(),
|
||||
res.getHighestPrioLowRedundancyBlocks());
|
||||
}
|
||||
return new ReplicatedBlockStats(res.getLowRedundancy(),
|
||||
res.getCorruptBlocks(), res.getMissingBlocks(),
|
||||
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
|
||||
|
@ -1998,6 +2005,12 @@ public class PBHelperClient {
|
|||
|
||||
public static ECBlockGroupStats convert(
|
||||
GetFsECBlockGroupStatsResponseProto res) {
|
||||
if (res.hasHighestPrioLowRedundancyBlocks()) {
|
||||
return new ECBlockGroupStats(res.getLowRedundancy(),
|
||||
res.getCorruptBlocks(), res.getMissingBlocks(),
|
||||
res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
|
||||
res.getHighestPrioLowRedundancyBlocks());
|
||||
}
|
||||
return new ECBlockGroupStats(res.getLowRedundancy(),
|
||||
res.getCorruptBlocks(), res.getMissingBlocks(),
|
||||
res.getBlocksInFuture(), res.getPendingDeletionBlocks());
|
||||
|
@ -2432,6 +2445,10 @@ public class PBHelperClient {
|
|||
replicatedBlockStats.getBytesInFutureBlocks());
|
||||
result.setPendingDeletionBlocks(
|
||||
replicatedBlockStats.getPendingDeletionBlocks());
|
||||
if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
|
||||
result.setHighestPrioLowRedundancyBlocks(
|
||||
replicatedBlockStats.getHighestPriorityLowRedundancyBlocks());
|
||||
}
|
||||
return result.build();
|
||||
}
|
||||
|
||||
|
@ -2447,6 +2464,10 @@ public class PBHelperClient {
|
|||
ecBlockGroupStats.getBytesInFutureBlockGroups());
|
||||
result.setPendingDeletionBlocks(
|
||||
ecBlockGroupStats.getPendingDeletionBlocks());
|
||||
if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
|
||||
result.setHighestPrioLowRedundancyBlocks(
|
||||
ecBlockGroupStats.getHighestPriorityLowRedundancyBlocks());
|
||||
}
|
||||
return result.build();
|
||||
}
|
||||
|
||||
|
|
|
@ -348,6 +348,8 @@ message GetFsReplicatedBlockStatsResponseProto {
|
|||
required uint64 missing_repl_one_blocks = 4;
|
||||
required uint64 blocks_in_future = 5;
|
||||
required uint64 pending_deletion_blocks = 6;
|
||||
optional uint64 highest_prio_low_redundancy_blocks = 7;
|
||||
|
||||
}
|
||||
|
||||
message GetFsECBlockGroupStatsRequestProto { // no input paramters
|
||||
|
@ -359,6 +361,7 @@ message GetFsECBlockGroupStatsResponseProto {
|
|||
required uint64 missing_blocks = 3;
|
||||
required uint64 blocks_in_future = 4;
|
||||
required uint64 pending_deletion_blocks = 5;
|
||||
optional uint64 highest_prio_low_redundancy_blocks = 6;
|
||||
}
|
||||
|
||||
enum DatanodeReportTypeProto { // type of the datanode report
|
||||
|
|
|
@ -320,6 +320,16 @@ public class NamenodeBeanMetrics
|
|||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getHighestPriorityLowRedundancyReplicatedBlocks() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getHighestPriorityLowRedundancyECBlocks() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCorruptFiles() {
|
||||
return "N/A";
|
||||
|
|
|
@ -185,12 +185,25 @@ public class ErasureCoding {
|
|||
long missingBlockGroups = 0;
|
||||
long bytesInFutureBlockGroups = 0;
|
||||
long pendingDeletionBlocks = 0;
|
||||
long highestPriorityLowRedundancyBlocks = 0;
|
||||
boolean hasHighestPriorityLowRedundancyBlocks = false;
|
||||
|
||||
for (ECBlockGroupStats stats : allStats.values()) {
|
||||
lowRedundancyBlockGroups += stats.getLowRedundancyBlockGroups();
|
||||
corruptBlockGroups += stats.getCorruptBlockGroups();
|
||||
missingBlockGroups += stats.getMissingBlockGroups();
|
||||
bytesInFutureBlockGroups += stats.getBytesInFutureBlockGroups();
|
||||
pendingDeletionBlocks += stats.getPendingDeletionBlocks();
|
||||
if (stats.hasHighestPriorityLowRedundancyBlocks()) {
|
||||
hasHighestPriorityLowRedundancyBlocks = true;
|
||||
highestPriorityLowRedundancyBlocks +=
|
||||
stats.getHighestPriorityLowRedundancyBlocks();
|
||||
}
|
||||
}
|
||||
if (hasHighestPriorityLowRedundancyBlocks) {
|
||||
return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
|
||||
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
|
||||
highestPriorityLowRedundancyBlocks);
|
||||
}
|
||||
return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
|
||||
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks);
|
||||
|
|
|
@ -4428,6 +4428,14 @@ public class BlockManager implements BlockStatsMXBean {
|
|||
return this.neededReconstruction.getCorruptReplicationOneBlockSize();
|
||||
}
|
||||
|
||||
public long getHighestPriorityReplicatedBlockCount(){
|
||||
return this.neededReconstruction.getHighestPriorityReplicatedBlockCount();
|
||||
}
|
||||
|
||||
public long getHighestPriorityECBlockCount(){
|
||||
return this.neededReconstruction.getHighestPriorityECBlockCount();
|
||||
}
|
||||
|
||||
public BlockInfo addBlockCollection(BlockInfo block,
|
||||
BlockCollection bc) {
|
||||
return blocksMap.addBlockCollection(block, bc);
|
||||
|
|
|
@ -92,6 +92,10 @@ class LowRedundancyBlocks implements Iterable<BlockInfo> {
|
|||
private final LongAdder corruptReplicationOneBlocks = new LongAdder();
|
||||
private final LongAdder lowRedundancyECBlockGroups = new LongAdder();
|
||||
private final LongAdder corruptECBlockGroups = new LongAdder();
|
||||
private final LongAdder highestPriorityLowRedundancyReplicatedBlocks
|
||||
= new LongAdder();
|
||||
private final LongAdder highestPriorityLowRedundancyECBlocks
|
||||
= new LongAdder();
|
||||
|
||||
/** Create an object. */
|
||||
LowRedundancyBlocks() {
|
||||
|
@ -162,6 +166,18 @@ class LowRedundancyBlocks implements Iterable<BlockInfo> {
|
|||
return corruptReplicationOneBlocks.longValue();
|
||||
}
|
||||
|
||||
/** Return the number of under replicated blocks
|
||||
* with the highest priority to recover */
|
||||
long getHighestPriorityReplicatedBlockCount() {
|
||||
return highestPriorityLowRedundancyReplicatedBlocks.longValue();
|
||||
}
|
||||
|
||||
/** Return the number of under replicated EC blocks
|
||||
* with the highest priority to recover */
|
||||
long getHighestPriorityECBlockCount() {
|
||||
return highestPriorityLowRedundancyECBlocks.longValue();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return low redundancy striped blocks excluding corrupt blocks.
|
||||
*/
|
||||
|
@ -300,6 +316,9 @@ class LowRedundancyBlocks implements Iterable<BlockInfo> {
|
|||
if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS) {
|
||||
corruptECBlockGroups.increment();
|
||||
}
|
||||
if (priLevel == QUEUE_HIGHEST_PRIORITY) {
|
||||
highestPriorityLowRedundancyECBlocks.increment();
|
||||
}
|
||||
} else {
|
||||
lowRedundancyBlocks.increment();
|
||||
if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS) {
|
||||
|
@ -308,6 +327,9 @@ class LowRedundancyBlocks implements Iterable<BlockInfo> {
|
|||
corruptReplicationOneBlocks.increment();
|
||||
}
|
||||
}
|
||||
if (priLevel == QUEUE_HIGHEST_PRIORITY) {
|
||||
highestPriorityLowRedundancyReplicatedBlocks.increment();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -380,6 +402,9 @@ class LowRedundancyBlocks implements Iterable<BlockInfo> {
|
|||
if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS) {
|
||||
corruptECBlockGroups.decrement();
|
||||
}
|
||||
if (priLevel == QUEUE_HIGHEST_PRIORITY) {
|
||||
highestPriorityLowRedundancyECBlocks.decrement();
|
||||
}
|
||||
} else {
|
||||
lowRedundancyBlocks.decrement();
|
||||
if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS) {
|
||||
|
@ -391,6 +416,9 @@ class LowRedundancyBlocks implements Iterable<BlockInfo> {
|
|||
"should be non-negative";
|
||||
}
|
||||
}
|
||||
if (priLevel == QUEUE_HIGHEST_PRIORITY) {
|
||||
highestPriorityLowRedundancyReplicatedBlocks.decrement();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -4218,7 +4218,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
|
|||
return new ReplicatedBlockStats(getLowRedundancyReplicatedBlocks(),
|
||||
getCorruptReplicatedBlocks(), getMissingReplicatedBlocks(),
|
||||
getMissingReplicationOneBlocks(), getBytesInFutureReplicatedBlocks(),
|
||||
getPendingDeletionReplicatedBlocks());
|
||||
getPendingDeletionReplicatedBlocks(),
|
||||
getHighestPriorityLowRedundancyReplicatedBlocks());
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -4230,7 +4231,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
|
|||
ECBlockGroupStats getECBlockGroupStats() {
|
||||
return new ECBlockGroupStats(getLowRedundancyECBlockGroups(),
|
||||
getCorruptECBlockGroups(), getMissingECBlockGroups(),
|
||||
getBytesInFutureECBlockGroups(), getPendingDeletionECBlocks());
|
||||
getBytesInFutureECBlockGroups(), getPendingDeletionECBlocks(),
|
||||
getHighestPriorityLowRedundancyECBlocks());
|
||||
}
|
||||
|
||||
@Override // FSNamesystemMBean
|
||||
|
@ -4837,6 +4839,20 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
|
|||
return blockManager.getMissingReplicationOneBlocks();
|
||||
}
|
||||
|
||||
@Override // ReplicatedBlocksMBean
|
||||
@Metric({"HighestPriorityLowRedundancyReplicatedBlocks", "Number of " +
|
||||
"replicated blocks which have the highest risk of loss."})
|
||||
public long getHighestPriorityLowRedundancyReplicatedBlocks() {
|
||||
return blockManager.getHighestPriorityReplicatedBlockCount();
|
||||
}
|
||||
|
||||
@Override // ReplicatedBlocksMBean
|
||||
@Metric({"HighestPriorityLowRedundancyECBlocks", "Number of erasure coded " +
|
||||
"blocks which have the highest risk of loss."})
|
||||
public long getHighestPriorityLowRedundancyECBlocks() {
|
||||
return blockManager.getHighestPriorityECBlockCount();
|
||||
}
|
||||
|
||||
@Override // ReplicatedBlocksMBean
|
||||
@Metric({"BytesInFutureReplicatedBlocks", "Total bytes in replicated " +
|
||||
"blocks with future generation stamp"})
|
||||
|
|
|
@ -162,6 +162,24 @@ public interface NameNodeMXBean {
|
|||
*/
|
||||
public long getNumberOfMissingBlocksWithReplicationFactorOne();
|
||||
|
||||
/**
|
||||
* Gets the total number of replicated low redundancy blocks on the cluster
|
||||
* with the highest risk of loss.
|
||||
*
|
||||
* @return the total number of low redundancy blocks on the cluster
|
||||
* with the highest risk of loss.
|
||||
*/
|
||||
public long getHighestPriorityLowRedundancyReplicatedBlocks();
|
||||
|
||||
/**
|
||||
* Gets the total number of erasure coded low redundancy blocks on the cluster
|
||||
* with the highest risk of loss
|
||||
*
|
||||
* @return the total number of low redundancy blocks on the cluster
|
||||
* with the highest risk of loss
|
||||
*/
|
||||
public long getHighestPriorityLowRedundancyECBlocks();
|
||||
|
||||
/**
|
||||
* Gets the total number of snapshottable dirs in the system.
|
||||
*
|
||||
|
|
|
@ -549,6 +549,11 @@ public class DFSAdmin extends FsShell {
|
|||
replicatedBlockStats.getMissingReplicaBlocks());
|
||||
System.out.println("\tMissing blocks (with replication factor 1): " +
|
||||
replicatedBlockStats.getMissingReplicationOneBlocks());
|
||||
if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
|
||||
System.out.println("\tLow redundancy blocks with highest priority " +
|
||||
"to recover: " +
|
||||
replicatedBlockStats.getHighestPriorityLowRedundancyBlocks());
|
||||
}
|
||||
System.out.println("\tPending deletion blocks: " +
|
||||
replicatedBlockStats.getPendingDeletionBlocks());
|
||||
|
||||
|
@ -561,6 +566,11 @@ public class DFSAdmin extends FsShell {
|
|||
ecBlockGroupStats.getCorruptBlockGroups());
|
||||
System.out.println("\tMissing block groups: " +
|
||||
ecBlockGroupStats.getMissingBlockGroups());
|
||||
if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
|
||||
System.out.println("\tLow redundancy blocks with highest priority " +
|
||||
"to recover: " +
|
||||
ecBlockGroupStats.getHighestPriorityLowRedundancyBlocks());
|
||||
}
|
||||
System.out.println("\tPending deletion blocks: " +
|
||||
ecBlockGroupStats.getPendingDeletionBlocks());
|
||||
|
||||
|
|
|
@ -63,7 +63,8 @@ public class TestLowRedundancyBlockQueues {
|
|||
private void verifyBlockStats(LowRedundancyBlocks queues,
|
||||
int lowRedundancyReplicaCount, int corruptReplicaCount,
|
||||
int corruptReplicationOneCount, int lowRedundancyStripedCount,
|
||||
int corruptStripedCount) {
|
||||
int corruptStripedCount, int highestPriorityReplicatedBlockCount,
|
||||
int highestPriorityECBlockCount) {
|
||||
assertEquals("Low redundancy replica count incorrect!",
|
||||
lowRedundancyReplicaCount, queues.getLowRedundancyBlocks());
|
||||
assertEquals("Corrupt replica count incorrect!",
|
||||
|
@ -81,6 +82,14 @@ public class TestLowRedundancyBlockQueues {
|
|||
assertEquals("LowRedundancyBlocks queue size incorrect!",
|
||||
(lowRedundancyReplicaCount + corruptReplicaCount +
|
||||
lowRedundancyStripedCount + corruptStripedCount), queues.size());
|
||||
assertEquals("Highest priority replicated low redundancy " +
|
||||
"blocks count is incorrect!",
|
||||
highestPriorityReplicatedBlockCount,
|
||||
queues.getHighestPriorityReplicatedBlockCount());
|
||||
assertEquals("Highest priority erasure coded low redundancy " +
|
||||
"blocks count is incorrect!",
|
||||
highestPriorityECBlockCount,
|
||||
queues.getHighestPriorityECBlockCount());
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -100,42 +109,46 @@ public class TestLowRedundancyBlockQueues {
|
|||
// Add a block with a single entry
|
||||
assertAdded(queues, block1, 1, 0, 3);
|
||||
assertInLevel(queues, block1, LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY);
|
||||
verifyBlockStats(queues, 1, 0, 0, 0, 0);
|
||||
verifyBlockStats(queues, 1, 0, 0, 0, 0, 1, 0);
|
||||
|
||||
// Repeated additions fail
|
||||
assertFalse(queues.add(block1, 1, 0, 0, 3));
|
||||
verifyBlockStats(queues, 1, 0, 0, 0, 0);
|
||||
verifyBlockStats(queues, 1, 0, 0, 0, 0, 1, 0);
|
||||
|
||||
// Add a second block with two replicas
|
||||
assertAdded(queues, block2, 2, 0, 3);
|
||||
assertInLevel(queues, block2, LowRedundancyBlocks.QUEUE_LOW_REDUNDANCY);
|
||||
verifyBlockStats(queues, 2, 0, 0, 0, 0);
|
||||
verifyBlockStats(queues, 2, 0, 0, 0, 0, 1, 0);
|
||||
|
||||
// Now try to add a block that is corrupt
|
||||
assertAdded(queues, block_corrupt, 0, 0, 3);
|
||||
assertInLevel(queues, block_corrupt,
|
||||
LowRedundancyBlocks.QUEUE_WITH_CORRUPT_BLOCKS);
|
||||
verifyBlockStats(queues, 2, 1, 0, 0, 0);
|
||||
verifyBlockStats(queues, 2, 1, 0, 0, 0, 1, 0);
|
||||
|
||||
// Insert a very insufficiently redundancy block
|
||||
assertAdded(queues, block_very_low_redundancy, 4, 0, 25);
|
||||
assertInLevel(queues, block_very_low_redundancy,
|
||||
LowRedundancyBlocks.QUEUE_VERY_LOW_REDUNDANCY);
|
||||
verifyBlockStats(queues, 3, 1, 0, 0, 0);
|
||||
verifyBlockStats(queues, 3, 1, 0, 0, 0, 1, 0);
|
||||
|
||||
// Insert a corrupt block with replication factor 1
|
||||
assertAdded(queues, block_corrupt_repl_one, 0, 0, 1);
|
||||
verifyBlockStats(queues, 3, 2, 1, 0, 0);
|
||||
verifyBlockStats(queues, 3, 2, 1, 0, 0, 1, 0);
|
||||
|
||||
// Bump up the expected count for corrupt replica one block from 1 to 3
|
||||
queues.update(block_corrupt_repl_one, 0, 0, 0, 3, 0, 2);
|
||||
verifyBlockStats(queues, 3, 2, 0, 0, 0);
|
||||
verifyBlockStats(queues, 3, 2, 0, 0, 0, 1, 0);
|
||||
|
||||
// Reduce the expected replicas to 1
|
||||
queues.update(block_corrupt, 0, 0, 0, 1, 0, -2);
|
||||
verifyBlockStats(queues, 3, 2, 1, 0, 0);
|
||||
verifyBlockStats(queues, 3, 2, 1, 0, 0, 1, 0);
|
||||
queues.update(block_very_low_redundancy, 0, 0, 0, 1, -4, -24);
|
||||
verifyBlockStats(queues, 2, 3, 2, 0, 0);
|
||||
verifyBlockStats(queues, 2, 3, 2, 0, 0, 1, 0);
|
||||
|
||||
// Reduce the expected replicas to 1 for block1
|
||||
queues.update(block1, 1, 0, 0, 1, 0, 0);
|
||||
verifyBlockStats(queues, 2, 3, 2, 0, 0, 0, 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
@ -145,12 +158,12 @@ public class TestLowRedundancyBlockQueues {
|
|||
assertAdded(queues, corruptBlock, 0, 0, 3);
|
||||
assertInLevel(queues, corruptBlock,
|
||||
LowRedundancyBlocks.QUEUE_WITH_CORRUPT_BLOCKS);
|
||||
verifyBlockStats(queues, 0, 1, 0, 0, 0);
|
||||
verifyBlockStats(queues, 0, 1, 0, 0, 0, 0, 0);
|
||||
|
||||
// Remove with wrong priority
|
||||
queues.remove(corruptBlock, LowRedundancyBlocks.QUEUE_LOW_REDUNDANCY);
|
||||
// Verify the number of corrupt block is decremented
|
||||
verifyBlockStats(queues, 0, 0, 0, 0, 0);
|
||||
verifyBlockStats(queues, 0, 0, 0, 0, 0, 0, 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
@ -186,17 +199,17 @@ public class TestLowRedundancyBlockQueues {
|
|||
assertInLevel(queues, block,
|
||||
LowRedundancyBlocks.QUEUE_LOW_REDUNDANCY);
|
||||
}
|
||||
verifyBlockStats(queues, 0, 0, 0, numUR, 0);
|
||||
verifyBlockStats(queues, 0, 0, 0, numUR, 0, 0, 1);
|
||||
}
|
||||
|
||||
// add a corrupted block
|
||||
BlockInfo block_corrupt = genStripedBlockInfo(-10, numBytes);
|
||||
assertEquals(numCorrupt, queues.getCorruptBlockSize());
|
||||
verifyBlockStats(queues, 0, 0, 0, numUR, numCorrupt);
|
||||
verifyBlockStats(queues, 0, 0, 0, numUR, numCorrupt, 0, 1);
|
||||
|
||||
assertAdded(queues, block_corrupt, dataBlkNum - 1, 0, groupSize);
|
||||
numCorrupt++;
|
||||
verifyBlockStats(queues, 0, 0, 0, numUR, numCorrupt);
|
||||
verifyBlockStats(queues, 0, 0, 0, numUR, numCorrupt, 0, 1);
|
||||
|
||||
assertInLevel(queues, block_corrupt,
|
||||
LowRedundancyBlocks.QUEUE_WITH_CORRUPT_BLOCKS);
|
||||
|
|
|
@ -412,10 +412,12 @@ public class TestNameNodeMetrics {
|
|||
// Verify replica metrics
|
||||
assertGauge("LowRedundancyReplicatedBlocks", 0L, rb);
|
||||
assertGauge("CorruptReplicatedBlocks", 0L, rb);
|
||||
assertGauge("HighestPriorityLowRedundancyReplicatedBlocks", 0L, rb);
|
||||
|
||||
// Verify striped block groups metrics
|
||||
assertGauge("LowRedundancyECBlockGroups", 0L, rb);
|
||||
assertGauge("CorruptECBlockGroups", 0L, rb);
|
||||
assertGauge("HighestPriorityLowRedundancyECBlocks", 0L, rb);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -492,9 +494,11 @@ public class TestNameNodeMetrics {
|
|||
// Verify replicated blocks metrics
|
||||
assertGauge("LowRedundancyReplicatedBlocks", 1L, rb);
|
||||
assertGauge("CorruptReplicatedBlocks", 1L, rb);
|
||||
assertGauge("HighestPriorityLowRedundancyReplicatedBlocks", 1L, rb);
|
||||
// Verify striped blocks metrics
|
||||
assertGauge("LowRedundancyECBlockGroups", 0L, rb);
|
||||
assertGauge("CorruptECBlockGroups", 0L, rb);
|
||||
assertGauge("HighestPriorityLowRedundancyECBlocks", 0L, rb);
|
||||
|
||||
verifyAggregatedMetricsTally();
|
||||
|
||||
|
@ -517,9 +521,11 @@ public class TestNameNodeMetrics {
|
|||
// Verify replicated blocks metrics
|
||||
assertGauge("LowRedundancyReplicatedBlocks", 0L, rb);
|
||||
assertGauge("CorruptReplicatedBlocks", 0L, rb);
|
||||
assertGauge("HighestPriorityLowRedundancyReplicatedBlocks", 0L, rb);
|
||||
// Verify striped blocks metrics
|
||||
assertGauge("LowRedundancyECBlockGroups", 0L, rb);
|
||||
assertGauge("CorruptECBlockGroups", 0L, rb);
|
||||
assertGauge("HighestPriorityLowRedundancyECBlocks", 0L, rb);
|
||||
|
||||
verifyAggregatedMetricsTally();
|
||||
|
||||
|
@ -580,9 +586,11 @@ public class TestNameNodeMetrics {
|
|||
// Verify replica metrics
|
||||
assertGauge("LowRedundancyReplicatedBlocks", 0L, rb);
|
||||
assertGauge("CorruptReplicatedBlocks", 0L, rb);
|
||||
assertGauge("HighestPriorityLowRedundancyReplicatedBlocks", 0L, rb);
|
||||
// Verify striped block groups metrics
|
||||
assertGauge("LowRedundancyECBlockGroups", 1L, rb);
|
||||
assertGauge("CorruptECBlockGroups", 1L, rb);
|
||||
assertGauge("HighestPriorityLowRedundancyECBlocks", 1L, rb);
|
||||
|
||||
verifyAggregatedMetricsTally();
|
||||
|
||||
|
@ -602,9 +610,11 @@ public class TestNameNodeMetrics {
|
|||
// Verify replicated blocks metrics
|
||||
assertGauge("LowRedundancyReplicatedBlocks", 0L, rb);
|
||||
assertGauge("CorruptReplicatedBlocks", 0L, rb);
|
||||
assertGauge("HighestPriorityLowRedundancyReplicatedBlocks", 0L, rb);
|
||||
// Verify striped blocks metrics
|
||||
assertGauge("LowRedundancyECBlockGroups", 0L, rb);
|
||||
assertGauge("CorruptECBlockGroups", 0L, rb);
|
||||
assertGauge("HighestPriorityLowRedundancyECBlocks", 0L, rb);
|
||||
|
||||
verifyAggregatedMetricsTally();
|
||||
|
||||
|
@ -666,6 +676,8 @@ public class TestNameNodeMetrics {
|
|||
assertGauge("UnderReplicatedBlocks", 1L, rb);
|
||||
assertGauge("MissingBlocks", 1L, rb);
|
||||
assertGauge("MissingReplOneBlocks", 1L, rb);
|
||||
assertGauge("HighestPriorityLowRedundancyReplicatedBlocks", 0L, rb);
|
||||
assertGauge("HighestPriorityLowRedundancyECBlocks", 0L, rb);
|
||||
fs.delete(file, true);
|
||||
waitForDnMetricValue(NS_METRICS, "UnderReplicatedBlocks", 0L);
|
||||
}
|
||||
|
|
|
@ -579,7 +579,7 @@ public class TestDFSAdmin {
|
|||
// Verify report command for all counts to be zero
|
||||
resetStream();
|
||||
assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"}));
|
||||
verifyNodesAndCorruptBlocks(numDn, numDn, 0, 0, client);
|
||||
verifyNodesAndCorruptBlocks(numDn, numDn, 0, 0, client, 0L, 0L);
|
||||
|
||||
final short replFactor = 1;
|
||||
final long fileLength = 512L;
|
||||
|
@ -614,7 +614,7 @@ public class TestDFSAdmin {
|
|||
// Verify report command for all counts to be zero
|
||||
resetStream();
|
||||
assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"}));
|
||||
verifyNodesAndCorruptBlocks(numDn, numDn, 0, 0, client);
|
||||
verifyNodesAndCorruptBlocks(numDn, numDn, 0, 0, client, 0L, 0L);
|
||||
|
||||
// Choose a DataNode to shutdown
|
||||
final List<DataNode> datanodes = miniCluster.getDataNodes();
|
||||
|
@ -636,7 +636,7 @@ public class TestDFSAdmin {
|
|||
|
||||
// Verify report command to show dead DataNode
|
||||
assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"}));
|
||||
verifyNodesAndCorruptBlocks(numDn, numDn - 1, 0, 0, client);
|
||||
verifyNodesAndCorruptBlocks(numDn, numDn - 1, 0, 0, client, 0L, 1L);
|
||||
|
||||
// Corrupt the replicated block
|
||||
final int blockFilesCorrupted = miniCluster
|
||||
|
@ -664,7 +664,7 @@ public class TestDFSAdmin {
|
|||
// verify report command for corrupt replicated block
|
||||
resetStream();
|
||||
assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"}));
|
||||
verifyNodesAndCorruptBlocks(numDn, numDn - 1, 1, 0, client);
|
||||
verifyNodesAndCorruptBlocks(numDn, numDn - 1, 1, 0, client, 0L, 1L);
|
||||
|
||||
lbs = miniCluster.getFileSystem().getClient().
|
||||
getNamenode().getBlockLocations(
|
||||
|
@ -689,7 +689,7 @@ public class TestDFSAdmin {
|
|||
// and EC block group
|
||||
resetStream();
|
||||
assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"}));
|
||||
verifyNodesAndCorruptBlocks(numDn, numDn - 1, 1, 1, client);
|
||||
verifyNodesAndCorruptBlocks(numDn, numDn - 1, 1, 1, client, 0L, 0L);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -834,7 +834,10 @@ public class TestDFSAdmin {
|
|||
final int numLiveDn,
|
||||
final int numCorruptBlocks,
|
||||
final int numCorruptECBlockGroups,
|
||||
final DFSClient client) throws IOException {
|
||||
final DFSClient client,
|
||||
final Long highestPriorityLowRedundancyReplicatedBlocks,
|
||||
final Long highestPriorityLowRedundancyECBlocks)
|
||||
throws IOException {
|
||||
|
||||
/* init vars */
|
||||
final String outStr = scanIntoString(out);
|
||||
|
@ -847,12 +850,23 @@ public class TestDFSAdmin {
|
|||
final String expectedCorruptedECBlockGroupsStr = String.format(
|
||||
"Block groups with corrupt internal blocks: %d",
|
||||
numCorruptECBlockGroups);
|
||||
final String highestPriorityLowRedundancyReplicatedBlocksStr
|
||||
= String.format(
|
||||
"\tLow redundancy blocks with highest priority " +
|
||||
"to recover: %d",
|
||||
highestPriorityLowRedundancyReplicatedBlocks);
|
||||
final String highestPriorityLowRedundancyECBlocksStr = String.format(
|
||||
"\tLow redundancy blocks with highest priority " +
|
||||
"to recover: %d",
|
||||
highestPriorityLowRedundancyReplicatedBlocks);
|
||||
|
||||
// verify nodes and corrupt blocks
|
||||
assertThat(outStr, is(allOf(
|
||||
containsString(expectedLiveNodesStr),
|
||||
containsString(expectedCorruptedBlocksStr),
|
||||
containsString(expectedCorruptedECBlockGroupsStr))));
|
||||
containsString(expectedCorruptedECBlockGroupsStr),
|
||||
containsString(highestPriorityLowRedundancyReplicatedBlocksStr),
|
||||
containsString(highestPriorityLowRedundancyECBlocksStr))));
|
||||
|
||||
assertEquals(
|
||||
numDn,
|
||||
|
@ -867,8 +881,12 @@ public class TestDFSAdmin {
|
|||
client.getCorruptBlocksCount());
|
||||
assertEquals(numCorruptBlocks, client.getNamenode()
|
||||
.getReplicatedBlockStats().getCorruptBlocks());
|
||||
assertEquals(highestPriorityLowRedundancyReplicatedBlocks, client.getNamenode()
|
||||
.getReplicatedBlockStats().getHighestPriorityLowRedundancyBlocks());
|
||||
assertEquals(numCorruptECBlockGroups, client.getNamenode()
|
||||
.getECBlockGroupStats().getCorruptBlockGroups());
|
||||
assertEquals(highestPriorityLowRedundancyECBlocks, client.getNamenode()
|
||||
.getECBlockGroupStats().getHighestPriorityLowRedundancyBlocks());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
Loading…
Reference in New Issue