Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-2237. Change UnderReplicatedBlocks from public to package private. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1156424 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2011-08-11 01:05:45 +00:00
parent ef223e8e8e
commit cde987996a
4 changed files with 47 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hdfs/CHANGES.txt
View File

@ -660,6 +660,9 @@ Trunk (unreleased changes)
HDFS-2241. Remove implementing FSConstants interface to just get the HDFS-2241. Remove implementing FSConstants interface to just get the
constants from the interface. (suresh) constants from the interface. (suresh)
HDFS-2237. Change UnderReplicatedBlocks from public to package private.
(szetszwo)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-1458. Improve checkpoint performance by avoiding unnecessary image HDFS-1458. Improve checkpoint performance by avoiding unnecessary image

12
hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -51,7 +51,6 @@ import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks.BlockIterator;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState; import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState; import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.common.Util;
@ -2588,9 +2587,14 @@ public class BlockManager {
/** /**
* Return an iterator over the set of blocks for which there are no replicas. * Return an iterator over the set of blocks for which there are no replicas.
*/ */
public BlockIterator getCorruptReplicaBlockIterator() { public Iterator<Block> getCorruptReplicaBlockIterator() {
return neededReplications return neededReplications.iterator(
.iterator(UnderReplicatedBlocks.QUEUE_WITH_CORRUPT_BLOCKS); UnderReplicatedBlocks.QUEUE_WITH_CORRUPT_BLOCKS);
}
/** @return the size of UnderReplicatedBlocks */
public int numOfUnderReplicatedBlocks() {
return neededReplications.size();
} }
/** /**

57
hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
View File

@ -17,21 +17,26 @@
*/ */
package org.apache.hadoop.hdfs.server.blockmanagement; package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.*; import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NavigableSet;
import java.util.TreeSet;
import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNode;
/* Class for keeping track of under replication blocks /** Keep track of under replication blocks.
* Blocks have replication priority, with priority 0 indicating the highest * Blocks have replication priority, with priority 0 indicating the highest
* Blocks have only one replicas has the highest * Blocks have only one replicas has the highest
*/ */
public class UnderReplicatedBlocks implements Iterable<Block> { class UnderReplicatedBlocks implements Iterable<Block> {
static final int LEVEL = 5; static final int LEVEL = 5;
static public final int QUEUE_WITH_CORRUPT_BLOCKS = 4; static final int QUEUE_WITH_CORRUPT_BLOCKS = 4;
private List<TreeSet<Block>> priorityQueues = new ArrayList<TreeSet<Block>>(); private final List<NavigableSet<Block>> priorityQueues
= new ArrayList<NavigableSet<Block>>();
/* constructor */ /** Create an object. */
UnderReplicatedBlocks() { UnderReplicatedBlocks() {
for(int i=0; i<LEVEL; i++) { for(int i=0; i<LEVEL; i++) {
priorityQueues.add(new TreeSet<Block>()); priorityQueues.add(new TreeSet<Block>());
@ -47,8 +52,8 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
} }
} }
/* Return the total number of under replication blocks */ /** Return the total number of under replication blocks */
public synchronized int size() { synchronized int size() {
int size = 0; int size = 0;
for (int i=0; i<LEVEL; i++) { for (int i=0; i<LEVEL; i++) {
size += priorityQueues.get(i).size(); size += priorityQueues.get(i).size();
@ -56,7 +61,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return size; return size;
} }
/* Return the number of under replication blocks excluding corrupt blocks */ /** Return the number of under replication blocks excluding corrupt blocks */
synchronized int getUnderReplicatedBlockCount() { synchronized int getUnderReplicatedBlockCount() {
int size = 0; int size = 0;
for (int i=0; i<QUEUE_WITH_CORRUPT_BLOCKS; i++) { for (int i=0; i<QUEUE_WITH_CORRUPT_BLOCKS; i++) {
@ -70,15 +75,15 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return priorityQueues.get(QUEUE_WITH_CORRUPT_BLOCKS).size(); return priorityQueues.get(QUEUE_WITH_CORRUPT_BLOCKS).size();
} }
/* Check if a block is in the neededReplication queue */ /** Check if a block is in the neededReplication queue */
public synchronized boolean contains(Block block) { synchronized boolean contains(Block block) {
for(TreeSet<Block> set:priorityQueues) { for(NavigableSet<Block> set : priorityQueues) {
if(set.contains(block)) { return true; } if(set.contains(block)) { return true; }
} }
return false; return false;
} }
/* Return the priority of a block /** Return the priority of a block
* @param block a under replication block * @param block a under replication block
* @param curReplicas current number of replicas of the block * @param curReplicas current number of replicas of the block
* @param expectedReplicas expected number of replicas of the block * @param expectedReplicas expected number of replicas of the block
@ -106,7 +111,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
} }
} }
/* add a block to a under replication queue according to its priority /** add a block to a under replication queue according to its priority
* @param block a under replication block * @param block a under replication block
* @param curReplicas current number of replicas of the block * @param curReplicas current number of replicas of the block
* @param expectedReplicas expected number of replicas of the block * @param expectedReplicas expected number of replicas of the block
@ -134,7 +139,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return false; return false;
} }
/* remove a block from a under replication queue */ /** remove a block from a under replication queue */
synchronized boolean remove(Block block, synchronized boolean remove(Block block,
int oldReplicas, int oldReplicas,
int decommissionedReplicas, int decommissionedReplicas,
@ -145,7 +150,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return remove(block, priLevel); return remove(block, priLevel);
} }
/* remove a block from a under replication queue given a priority*/ /** remove a block from a under replication queue given a priority*/
boolean remove(Block block, int priLevel) { boolean remove(Block block, int priLevel) {
if(priLevel >= 0 && priLevel < LEVEL if(priLevel >= 0 && priLevel < LEVEL
&& priorityQueues.get(priLevel).remove(block)) { && priorityQueues.get(priLevel).remove(block)) {
@ -174,7 +179,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return false; return false;
} }
/* update the priority level of a block */ /** update the priority level of a block */
synchronized void update(Block block, int curReplicas, synchronized void update(Block block, int curReplicas,
int decommissionedReplicas, int decommissionedReplicas,
int curExpectedReplicas, int curExpectedReplicas,
@ -209,30 +214,29 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
} }
} }
/* returns an iterator of all blocks in a given priority queue */ /** returns an iterator of all blocks in a given priority queue */
synchronized BlockIterator iterator(int level) { synchronized BlockIterator iterator(int level) {
return new BlockIterator(level); return new BlockIterator(level);
} }
/* return an iterator of all the under replication blocks */ /** return an iterator of all the under replication blocks */
public synchronized BlockIterator iterator() { public synchronized BlockIterator iterator() {
return new BlockIterator(); return new BlockIterator();
} }
public class BlockIterator implements Iterator<Block> { class BlockIterator implements Iterator<Block> {
private int level; private int level;
private boolean isIteratorForLevel = false; private boolean isIteratorForLevel = false;
private List<Iterator<Block>> iterators = new ArrayList<Iterator<Block>>(); private List<Iterator<Block>> iterators = new ArrayList<Iterator<Block>>();
BlockIterator() private BlockIterator() {
{
level=0; level=0;
for(int i=0; i<LEVEL; i++) { for(int i=0; i<LEVEL; i++) {
iterators.add(priorityQueues.get(i).iterator()); iterators.add(priorityQueues.get(i).iterator());
} }
} }
BlockIterator(int l) { private BlockIterator(int l) {
level = l; level = l;
isIteratorForLevel = true; isIteratorForLevel = true;
iterators.add(priorityQueues.get(level).iterator()); iterators.add(priorityQueues.get(level).iterator());
@ -246,6 +250,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
} }
} }
@Override
public Block next() { public Block next() {
if (isIteratorForLevel) if (isIteratorForLevel)
return iterators.get(0).next(); return iterators.get(0).next();
@ -253,6 +258,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return iterators.get(level).next(); return iterators.get(level).next();
} }
@Override
public boolean hasNext() { public boolean hasNext() {
if (isIteratorForLevel) if (isIteratorForLevel)
return iterators.get(0).hasNext(); return iterators.get(0).hasNext();
@ -260,6 +266,7 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
return iterators.get(level).hasNext(); return iterators.get(level).hasNext();
} }
@Override
public void remove() { public void remove() {
if (isIteratorForLevel) if (isIteratorForLevel)
iterators.get(0).remove(); iterators.get(0).remove();
@ -267,8 +274,8 @@ public class UnderReplicatedBlocks implements Iterable<Block> {
iterators.get(level).remove(); iterators.get(level).remove();
} }
public int getPriority() { int getPriority() {
return level; return level;
}; }
} }
} }

7
hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -39,6 +39,7 @@ import java.util.Date;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -97,7 +98,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStatistics; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStatistics;
import org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks;
import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState; import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole;
@ -2910,7 +2910,7 @@ public class FSNamesystem implements RwLock, FSClusterStats,
+ nt.getNumOfRacks() + " racks and " + nt.getNumOfRacks() + " racks and "
+ nt.getNumOfLeaves() + " datanodes"); + nt.getNumOfLeaves() + " datanodes");
NameNode.stateChangeLog.info("STATE* UnderReplicatedBlocks has " NameNode.stateChangeLog.info("STATE* UnderReplicatedBlocks has "
+blockManager.neededReplications.size()+" blocks"); + blockManager.numOfUnderReplicatedBlocks() + " blocks");
} }
/** /**
@ -3954,7 +3954,8 @@ public class FSNamesystem implements RwLock, FSClusterStats,
if (startBlockAfter != null) { if (startBlockAfter != null) {
startBlockId = Block.filename2id(startBlockAfter); startBlockId = Block.filename2id(startBlockAfter);
} }
UnderReplicatedBlocks.BlockIterator blkIterator = blockManager.getCorruptReplicaBlockIterator();
final Iterator<Block> blkIterator = blockManager.getCorruptReplicaBlockIterator();
while (blkIterator.hasNext()) { while (blkIterator.hasNext()) {
Block blk = blkIterator.next(); Block blk = blkIterator.next();
INode inode = blockManager.getINode(blk); INode inode = blockManager.getINode(blk);