diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 31b197a9187..24ce9429e93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -458,6 +458,9 @@ Release 2.8.0 - UNRELEASED HDFS-8880. NameNode metrics logging. (Arpit Agarwal) + HDFS-8278. When computing max-size-to-move in Balancer, count only the + storage with remaining >= default block size. (szetszwo) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 4f3f18e28be..16945df1a80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -192,6 +192,7 @@ public class Balancer { private final boolean runDuringUpgrade; private final double threshold; private final long maxSizeToMove; + private final long defaultBlockSize; // all data node lists private final Collection overUtilized = new LinkedList(); @@ -270,6 +271,9 @@ static int getInt(Configuration conf, String key, int defaultValue) { this.maxSizeToMove = getLong(conf, DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY, DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_DEFAULT); + this.defaultBlockSize = getLong(conf, + DFSConfigKeys.DFS_BLOCK_SIZE_KEY, + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); } private static long getCapacity(DatanodeStorageReport report, StorageType t) { @@ -282,11 +286,13 @@ private static long getCapacity(DatanodeStorageReport report, StorageType t) { return capacity; } - private static long getRemaining(DatanodeStorageReport report, StorageType t) { + private long getRemaining(DatanodeStorageReport report, StorageType t) { long remaining = 0L; for(StorageReport r : report.getStorageReports()) { if (r.getStorage().getStorageType() == t) { - remaining += r.getRemaining(); + if (r.getRemaining() >= defaultBlockSize) { + remaining += r.getRemaining(); + } } } return remaining; @@ -323,7 +329,7 @@ private long init(List reports) { final double utilizationDiff = utilization - policy.getAvgUtilization(t); final double thresholdDiff = Math.abs(utilizationDiff) - threshold; final long maxSize2Move = computeMaxSize2Move(capacity, - getRemaining(r, t), utilizationDiff, threshold, maxSizeToMove); + getRemaining(r, t), utilizationDiff, maxSizeToMove); final StorageGroup g; if (utilizationDiff > 0) { @@ -360,8 +366,8 @@ private long init(List reports) { } private static long computeMaxSize2Move(final long capacity, final long remaining, - final double utilizationDiff, final double threshold, final long max) { - final double diff = Math.min(threshold, Math.abs(utilizationDiff)); + final double utilizationDiff, final long max) { + final double diff = Math.abs(utilizationDiff); long maxSizeToMove = percentage2bytes(diff, capacity); if (utilizationDiff < 0) { maxSizeToMove = Math.min(remaining, maxSizeToMove);