Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-2237. Change UnderReplicatedBlocks from public to package private. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1156424 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2011-08-11 01:05:45 +00:00
parent ef223e8e8e
commit cde987996a
4 changed files with 47 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hdfs/CHANGES.txt
View File

@ -660,6 +660,9 @@ Trunk (unreleased changes)
HDFS-2241. Remove implementing FSConstants interface to just get the
constants from the interface. (suresh)
HDFS-2237. Change UnderReplicatedBlocks from public to package private.
(szetszwo)
OPTIMIZATIONS
HDFS-1458. Improve checkpoint performance by avoiding unnecessary image

12
hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -51,7 +51,6 @@ import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks.BlockIterator;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.common.Util;
@ -2588,9 +2587,14 @@ public class BlockManager {
/**
* Return an iterator over the set of blocks for which there are no replicas.
*/
public BlockIterator getCorruptReplicaBlockIterator() {
return neededReplications
.iterator(UnderReplicatedBlocks.QUEUE_WITH_CORRUPT_BLOCKS);
public Iterator<Block> getCorruptReplicaBlockIterator() {
return neededReplications.iterator(
UnderReplicatedBlocks.QUEUE_WITH_CORRUPT_BLOCKS);
}
/** @return the size of UnderReplicatedBlocks */
public int numOfUnderReplicatedBlocks() {
return neededReplications.size();
}
/**

57
hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
View File

@ -17,21 +17,26 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.*;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NavigableSet;
import java.util.TreeSet;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
/* Class for keeping track of under replication blocks
/** Keep track of under replication blocks.
* Blocks have replication priority, with priority 0 indicating the highest
* Blocks have only one replicas has the highest
*/
public class UnderReplicatedBlocks implements Iterable<Block> {
class UnderReplicatedBlocks implements Iterable<Block> {
static final int LEVEL = 5;
static public final int QUEUE_WITH_CORRUPT_BLOCKS = 4;
private List<TreeSet<Block>> priorityQueues = new ArrayList<TreeSet<Block>>();
static final int QUEUE_WITH_CORRUPT_BLOCKS = 4;
private final List<NavigableSet<Block>> priorityQueues
= new ArrayList<NavigableSet<Block>>();
/* constructor */
/** Create an object. */
UnderReplicatedBlocks() {
for(int i=0; i<LEVEL; i++) {
priorityQueues.add(new TreeSet<Block>());
@ -47,8 +52,8 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
}
}
/* Return the total number of under replication blocks */
public synchronized int size() {
/** Return the total number of under replication blocks */
synchronized int size() {
int size = 0;
for (int i=0; i<LEVEL; i++) {
size += priorityQueues.get(i).size();
@ -56,7 +61,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return size;
}
/* Return the number of under replication blocks excluding corrupt blocks */
/** Return the number of under replication blocks excluding corrupt blocks */
synchronized int getUnderReplicatedBlockCount() {
int size = 0;
for (int i=0; i<QUEUE_WITH_CORRUPT_BLOCKS; i++) {
@ -70,15 +75,15 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return priorityQueues.get(QUEUE_WITH_CORRUPT_BLOCKS).size();
}
/* Check if a block is in the neededReplication queue */
public synchronized boolean contains(Block block) {
for(TreeSet<Block> set:priorityQueues) {
/** Check if a block is in the neededReplication queue */
synchronized boolean contains(Block block) {
for(NavigableSet<Block> set : priorityQueues) {
if(set.contains(block)) { return true; }
}
return false;
}
/* Return the priority of a block
/** Return the priority of a block
* @param block a under replication block
* @param curReplicas current number of replicas of the block
* @param expectedReplicas expected number of replicas of the block
@ -106,7 +111,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
}
}
/* add a block to a under replication queue according to its priority
/** add a block to a under replication queue according to its priority
* @param block a under replication block
* @param curReplicas current number of replicas of the block
* @param expectedReplicas expected number of replicas of the block
@ -134,7 +139,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return false;
}
/* remove a block from a under replication queue */
/** remove a block from a under replication queue */
synchronized boolean remove(Block block,
int oldReplicas,
int decommissionedReplicas,
@ -145,7 +150,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return remove(block, priLevel);
}
/* remove a block from a under replication queue given a priority*/
/** remove a block from a under replication queue given a priority*/
boolean remove(Block block, int priLevel) {
if(priLevel >= 0 && priLevel < LEVEL
&& priorityQueues.get(priLevel).remove(block)) {
@ -174,7 +179,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return false;
}
/* update the priority level of a block */
/** update the priority level of a block */
synchronized void update(Block block, int curReplicas,
int decommissionedReplicas,
int curExpectedReplicas,
@ -209,30 +214,29 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
}
}
/* returns an iterator of all blocks in a given priority queue */
/** returns an iterator of all blocks in a given priority queue */
synchronized BlockIterator iterator(int level) {
return new BlockIterator(level);
}
/* return an iterator of all the under replication blocks */
/** return an iterator of all the under replication blocks */
public synchronized BlockIterator iterator() {
return new BlockIterator();
}
public class BlockIterator implements Iterator<Block> {
class BlockIterator implements Iterator<Block> {
private int level;
private boolean isIteratorForLevel = false;
private List<Iterator<Block>> iterators = new ArrayList<Iterator<Block>>();
BlockIterator()
{
private BlockIterator() {
level=0;
for(int i=0; i<LEVEL; i++) {
iterators.add(priorityQueues.get(i).iterator());
}
}
BlockIterator(int l) {
private BlockIterator(int l) {
level = l;
isIteratorForLevel = true;
iterators.add(priorityQueues.get(level).iterator());
@ -246,6 +250,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
}
}
@Override
public Block next() {
if (isIteratorForLevel)
return iterators.get(0).next();
@ -253,6 +258,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return iterators.get(level).next();
}
@Override
public boolean hasNext() {
if (isIteratorForLevel)
return iterators.get(0).hasNext();
@ -260,6 +266,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return iterators.get(level).hasNext();
}
@Override
public void remove() {
if (isIteratorForLevel)
iterators.get(0).remove();
@ -267,8 +274,8 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
iterators.get(level).remove();
}
public int getPriority() {
int getPriority() {
return level;
};
}
}
}

7
hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -39,6 +39,7 @@ import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -97,7 +98,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStatistics;
import org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole;
@ -2910,7 +2910,7 @@ public class FSNamesystem implements RwLock, FSClusterStats,
+ nt.getNumOfRacks() + " racks and "
+ nt.getNumOfLeaves() + " datanodes");
NameNode.stateChangeLog.info("STATE* UnderReplicatedBlocks has "
+blockManager.neededReplications.size()+" blocks");
+ blockManager.numOfUnderReplicatedBlocks() + " blocks");
}
/**
@ -3954,7 +3954,8 @@ public class FSNamesystem implements RwLock, FSClusterStats,
if (startBlockAfter != null) {
startBlockId = Block.filename2id(startBlockAfter);
}
UnderReplicatedBlocks.BlockIterator blkIterator = blockManager.getCorruptReplicaBlockIterator();
final Iterator<Block> blkIterator = blockManager.getCorruptReplicaBlockIterator();
while (blkIterator.hasNext()) {
Block blk = blkIterator.next();
INode inode = blockManager.getINode(blk);