From cde987996ae727154b5081bf0a76e10c7c236118 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 11 Aug 2011 01:05:45 +0000 Subject: [PATCH] HDFS-2237. Change UnderReplicatedBlocks from public to package private. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1156424 13f79535-47bb-0310-9956-ffa450edef68 --- hdfs/CHANGES.txt | 3 + .../server/blockmanagement/BlockManager.java | 12 ++-- .../UnderReplicatedBlocks.java | 57 +++++++++++-------- .../hdfs/server/namenode/FSNamesystem.java | 7 ++- 4 files changed, 47 insertions(+), 32 deletions(-) diff --git a/hdfs/CHANGES.txt b/hdfs/CHANGES.txt index 436182cb324..7dba58c0635 100644 --- a/hdfs/CHANGES.txt +++ b/hdfs/CHANGES.txt @@ -660,6 +660,9 @@ Trunk (unreleased changes) HDFS-2241. Remove implementing FSConstants interface to just get the constants from the interface. (suresh) + HDFS-2237. Change UnderReplicatedBlocks from public to package private. + (szetszwo) + OPTIMIZATIONS HDFS-1458. Improve checkpoint performance by avoiding unnecessary image diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 45dbe72525a..e552ccdeab1 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -51,7 +51,6 @@ import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; -import org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks.BlockIterator; import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState; import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState; import org.apache.hadoop.hdfs.server.common.Util; @@ -2588,9 +2587,14 @@ public class BlockManager { /** * Return an iterator over the set of blocks for which there are no replicas. */ - public BlockIterator getCorruptReplicaBlockIterator() { - return neededReplications - .iterator(UnderReplicatedBlocks.QUEUE_WITH_CORRUPT_BLOCKS); + public Iterator getCorruptReplicaBlockIterator() { + return neededReplications.iterator( + UnderReplicatedBlocks.QUEUE_WITH_CORRUPT_BLOCKS); + } + + /** @return the size of UnderReplicatedBlocks */ + public int numOfUnderReplicatedBlocks() { + return neededReplications.size(); } /** diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java index 9c2a213985c..7b39860d06f 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java @@ -17,21 +17,26 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import java.util.*; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.namenode.NameNode; -/* Class for keeping track of under replication blocks +/** Keep track of under replication blocks. * Blocks have replication priority, with priority 0 indicating the highest * Blocks have only one replicas has the highest */ -public class UnderReplicatedBlocks implements Iterable { +class UnderReplicatedBlocks implements Iterable { static final int LEVEL = 5; - static public final int QUEUE_WITH_CORRUPT_BLOCKS = 4; - private List> priorityQueues = new ArrayList>(); + static final int QUEUE_WITH_CORRUPT_BLOCKS = 4; + private final List> priorityQueues + = new ArrayList>(); - /* constructor */ + /** Create an object. */ UnderReplicatedBlocks() { for(int i=0; i()); @@ -47,8 +52,8 @@ public class UnderReplicatedBlocks implements Iterable { } } - /* Return the total number of under replication blocks */ - public synchronized int size() { + /** Return the total number of under replication blocks */ + synchronized int size() { int size = 0; for (int i=0; i { return size; } - /* Return the number of under replication blocks excluding corrupt blocks */ + /** Return the number of under replication blocks excluding corrupt blocks */ synchronized int getUnderReplicatedBlockCount() { int size = 0; for (int i=0; i { return priorityQueues.get(QUEUE_WITH_CORRUPT_BLOCKS).size(); } - /* Check if a block is in the neededReplication queue */ - public synchronized boolean contains(Block block) { - for(TreeSet set:priorityQueues) { + /** Check if a block is in the neededReplication queue */ + synchronized boolean contains(Block block) { + for(NavigableSet set : priorityQueues) { if(set.contains(block)) { return true; } } return false; } - /* Return the priority of a block + /** Return the priority of a block * @param block a under replication block * @param curReplicas current number of replicas of the block * @param expectedReplicas expected number of replicas of the block @@ -106,7 +111,7 @@ public class UnderReplicatedBlocks implements Iterable { } } - /* add a block to a under replication queue according to its priority + /** add a block to a under replication queue according to its priority * @param block a under replication block * @param curReplicas current number of replicas of the block * @param expectedReplicas expected number of replicas of the block @@ -134,7 +139,7 @@ public class UnderReplicatedBlocks implements Iterable { return false; } - /* remove a block from a under replication queue */ + /** remove a block from a under replication queue */ synchronized boolean remove(Block block, int oldReplicas, int decommissionedReplicas, @@ -145,7 +150,7 @@ public class UnderReplicatedBlocks implements Iterable { return remove(block, priLevel); } - /* remove a block from a under replication queue given a priority*/ + /** remove a block from a under replication queue given a priority*/ boolean remove(Block block, int priLevel) { if(priLevel >= 0 && priLevel < LEVEL && priorityQueues.get(priLevel).remove(block)) { @@ -174,7 +179,7 @@ public class UnderReplicatedBlocks implements Iterable { return false; } - /* update the priority level of a block */ + /** update the priority level of a block */ synchronized void update(Block block, int curReplicas, int decommissionedReplicas, int curExpectedReplicas, @@ -209,30 +214,29 @@ public class UnderReplicatedBlocks implements Iterable { } } - /* returns an iterator of all blocks in a given priority queue */ + /** returns an iterator of all blocks in a given priority queue */ synchronized BlockIterator iterator(int level) { return new BlockIterator(level); } - /* return an iterator of all the under replication blocks */ + /** return an iterator of all the under replication blocks */ public synchronized BlockIterator iterator() { return new BlockIterator(); } - public class BlockIterator implements Iterator { + class BlockIterator implements Iterator { private int level; private boolean isIteratorForLevel = false; private List> iterators = new ArrayList>(); - BlockIterator() - { + private BlockIterator() { level=0; for(int i=0; i { } } + @Override public Block next() { if (isIteratorForLevel) return iterators.get(0).next(); @@ -253,6 +258,7 @@ public class UnderReplicatedBlocks implements Iterable { return iterators.get(level).next(); } + @Override public boolean hasNext() { if (isIteratorForLevel) return iterators.get(0).hasNext(); @@ -260,6 +266,7 @@ public class UnderReplicatedBlocks implements Iterable { return iterators.get(level).hasNext(); } + @Override public void remove() { if (isIteratorForLevel) iterators.get(0).remove(); @@ -267,8 +274,8 @@ public class UnderReplicatedBlocks implements Iterable { iterators.get(level).remove(); } - public int getPriority() { + int getPriority() { return level; - }; + } } } diff --git a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 556e7291304..5ddd92b4058 100644 --- a/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -39,6 +39,7 @@ import java.util.Date; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -97,7 +98,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStatistics; -import org.apache.hadoop.hdfs.server.blockmanagement.UnderReplicatedBlocks; import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.common.HdfsConstants.BlockUCState; import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole; @@ -2910,7 +2910,7 @@ public class FSNamesystem implements RwLock, FSClusterStats, + nt.getNumOfRacks() + " racks and " + nt.getNumOfLeaves() + " datanodes"); NameNode.stateChangeLog.info("STATE* UnderReplicatedBlocks has " - +blockManager.neededReplications.size()+" blocks"); + + blockManager.numOfUnderReplicatedBlocks() + " blocks"); } /** @@ -3954,7 +3954,8 @@ public class FSNamesystem implements RwLock, FSClusterStats, if (startBlockAfter != null) { startBlockId = Block.filename2id(startBlockAfter); } - UnderReplicatedBlocks.BlockIterator blkIterator = blockManager.getCorruptReplicaBlockIterator(); + + final Iterator blkIterator = blockManager.getCorruptReplicaBlockIterator(); while (blkIterator.hasNext()) { Block blk = blkIterator.next(); INode inode = blockManager.getINode(blk);