Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-15911 : Provide blocks moved count in Balancer iteration result (#2794) 

Contributed by Viraj Jasani.

Signed-off-by: Mingliang Liu <liuml07@apache.org>
Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
This commit is contained in:
Viraj Jasani 2021-03-24 11:17:45 +05:30 committed by GitHub
parent 03cfc85279
commit 4b4ccce02f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 30 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
View File

@ -38,6 +38,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.slf4j.Logger;
@ -638,13 +639,15 @@ public class Balancer {
private final long bytesLeftToMove;
private final long bytesBeingMoved;
private final long bytesAlreadyMoved;
private final long blocksMoved;
Result(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved,
long bytesAlreadyMoved) {
long bytesAlreadyMoved, long blocksMoved) {
this.exitStatus = exitStatus;
this.bytesLeftToMove = bytesLeftToMove;
this.bytesBeingMoved = bytesBeingMoved;
this.bytesAlreadyMoved = bytesAlreadyMoved;
this.blocksMoved = blocksMoved;
}
public ExitStatus getExitStatus() {
@ -663,23 +666,40 @@ public class Balancer {
return bytesAlreadyMoved;
}
public long getBlocksMoved() {
return blocksMoved;
}
void print(int iteration, NameNodeConnector nnc, PrintStream out) {
out.printf("%-24s %10d %19s %18s %17s %s%n",
out.printf("%-24s %10d %19s %18s %17s %17s %s%n",
DateFormat.getDateTimeInstance().format(new Date()), iteration,
StringUtils.byteDesc(bytesAlreadyMoved),
StringUtils.byteDesc(bytesLeftToMove),
StringUtils.byteDesc(bytesBeingMoved),
blocksMoved,
nnc.getNameNodeUri());
}
@Override
public String toString() {
return new ToStringBuilder(this)
.append("exitStatus", exitStatus)
.append("bytesLeftToMove", bytesLeftToMove)
.append("bytesBeingMoved", bytesBeingMoved)
.append("bytesAlreadyMoved", bytesAlreadyMoved)
.append("blocksMoved", blocksMoved)
.toString();
}
}
Result newResult(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved) {
return new Result(exitStatus, bytesLeftToMove, bytesBeingMoved,
dispatcher.getBytesMoved());
dispatcher.getBytesMoved(), dispatcher.getBblocksMoved());
}
Result newResult(ExitStatus exitStatus) {
return new Result(exitStatus, -1, -1, dispatcher.getBytesMoved());
return new Result(exitStatus, -1, -1, dispatcher.getBytesMoved(),
dispatcher.getBblocksMoved());
}
/** Run an iteration for all datanodes. */

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
View File

@ -1658,6 +1658,7 @@ public class TestBalancer {
// a block is moved unexpectedly, IN_PROGRESS will be reported.
assertEquals("We expect ExitStatus.NO_MOVE_PROGRESS to be reported.",
ExitStatus.NO_MOVE_PROGRESS, r.getExitStatus());
assertEquals(0, r.getBlocksMoved());
}
} finally {
for (NameNodeConnector nnc : connectors) {
@ -2309,8 +2310,11 @@ public class TestBalancer {
// Hence, overall total blocks moved by HDFS balancer would be either of these 2 options:
// a) 2 blocks of total size (100B + 100B)
// b) 3 blocks of total size (50B + 100B + 100B)
assertTrue(balancerResult.getBytesAlreadyMoved() == 200
|| balancerResult.getBytesAlreadyMoved() == 250);
assertTrue("BalancerResult is not as expected. " + balancerResult,
(balancerResult.getBytesAlreadyMoved() == 200
&& balancerResult.getBlocksMoved() == 2)
|| (balancerResult.getBytesAlreadyMoved() == 250
&& balancerResult.getBlocksMoved() == 3));
// 100% and 95% used nodes will be balanced, so top used will be 900
assertEquals(900, maxUsage);
}