Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-8278. When computing max-size-to-move in Balancer, count only the storage with remaining >= default block size.

This commit is contained in:
Tsz-Wo Nicholas Sze 2015-08-17 17:55:25 -07:00
parent 076c688780
commit 9ab89a9266
2 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -458,6 +458,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8880. NameNode metrics logging. (Arpit Agarwal) HDFS-8880. NameNode metrics logging. (Arpit Agarwal)
HDFS-8278. When computing max-size-to-move in Balancer, count only the
storage with remaining >= default block size. (szetszwo)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
View File

@ -192,6 +192,7 @@ public class Balancer {
private final boolean runDuringUpgrade; private final boolean runDuringUpgrade;
private final double threshold; private final double threshold;
private final long maxSizeToMove; private final long maxSizeToMove;
private final long defaultBlockSize;
// all data node lists // all data node lists
private final Collection<Source> overUtilized = new LinkedList<Source>(); private final Collection<Source> overUtilized = new LinkedList<Source>();
@ -270,6 +271,9 @@ public class Balancer {
this.maxSizeToMove = getLong(conf, this.maxSizeToMove = getLong(conf,
DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY, DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY,
DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_DEFAULT); DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_DEFAULT);
this.defaultBlockSize = getLong(conf,
DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
} }
private static long getCapacity(DatanodeStorageReport report, StorageType t) { private static long getCapacity(DatanodeStorageReport report, StorageType t) {
@ -282,11 +286,13 @@ public class Balancer {
return capacity; return capacity;
} }
private static long getRemaining(DatanodeStorageReport report, StorageType t) { private long getRemaining(DatanodeStorageReport report, StorageType t) {
long remaining = 0L; long remaining = 0L;
for(StorageReport r : report.getStorageReports()) { for(StorageReport r : report.getStorageReports()) {
if (r.getStorage().getStorageType() == t) { if (r.getStorage().getStorageType() == t) {
remaining += r.getRemaining(); if (r.getRemaining() >= defaultBlockSize) {
remaining += r.getRemaining();
}
} }
} }
return remaining; return remaining;
@ -323,7 +329,7 @@ public class Balancer {
final double utilizationDiff = utilization - policy.getAvgUtilization(t); final double utilizationDiff = utilization - policy.getAvgUtilization(t);
final double thresholdDiff = Math.abs(utilizationDiff) - threshold; final double thresholdDiff = Math.abs(utilizationDiff) - threshold;
final long maxSize2Move = computeMaxSize2Move(capacity, final long maxSize2Move = computeMaxSize2Move(capacity,
getRemaining(r, t), utilizationDiff, threshold, maxSizeToMove); getRemaining(r, t), utilizationDiff, maxSizeToMove);
final StorageGroup g; final StorageGroup g;
if (utilizationDiff > 0) { if (utilizationDiff > 0) {
@ -360,8 +366,8 @@ public class Balancer {
} }
private static long computeMaxSize2Move(final long capacity, final long remaining, private static long computeMaxSize2Move(final long capacity, final long remaining,
final double utilizationDiff, final double threshold, final long max) { final double utilizationDiff, final long max) {
final double diff = Math.min(threshold, Math.abs(utilizationDiff)); final double diff = Math.abs(utilizationDiff);
long maxSizeToMove = percentage2bytes(diff, capacity); long maxSizeToMove = percentage2bytes(diff, capacity);
if (utilizationDiff < 0) { if (utilizationDiff < 0) {
maxSizeToMove = Math.min(remaining, maxSizeToMove); maxSizeToMove = Math.min(remaining, maxSizeToMove);