Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-10559. DiskBalancer: Use SHA1 for Plan ID. Contributed by Xiaobing Zhou.

This commit is contained in:
Anu Engineer 2016-08-15 20:10:21 -07:00
parent 02abd131b8
commit 5628b36c08
9 changed files with 19 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
View File

@ -175,7 +175,7 @@ public interface ClientDatanodeProtocol {
/** /**
* Cancel an executing plan. * Cancel an executing plan.
* *
* @param planID - A SHA512 hash of the plan string. * @param planID - A SHA-1 hash of the plan string.
*/ */
void cancelDiskBalancePlan(String planID) throws IOException; void cancelDiskBalancePlan(String planID) throws IOException;

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
View File

@ -369,7 +369,7 @@ public class ClientDatanodeProtocolTranslatorPB implements
/** /**
* Cancels an executing disk balancer plan. * Cancels an executing disk balancer plan.
* *
* @param planID - A SHA512 hash of the plan string. * @param planID - A SHA-1 hash of the plan string.
* @throws IOException on error * @throws IOException on error
*/ */
@Override @Override

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto
View File

@ -154,7 +154,7 @@ message GetBalancerBandwidthResponseProto {
* balancer plan to a data node. * balancer plan to a data node.
*/ */
message SubmitDiskBalancerPlanRequestProto { message SubmitDiskBalancerPlanRequestProto {
required string planID = 1; // A hash of the plan like SHA512 required string planID = 1; // A hash of the plan like SHA-1
required string plan = 2; // Plan file data in Json format required string plan = 2; // Plan file data in Json format
optional uint64 planVersion = 3; // Plan version number optional uint64 planVersion = 3; // Plan version number
optional bool ignoreDateCheck = 4; // Ignore date checks on this plan. optional bool ignoreDateCheck = 4; // Ignore date checks on this plan.

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
View File

@ -157,7 +157,7 @@ public class DiskBalancer {
* Takes a client submitted plan and converts into a set of work items that * Takes a client submitted plan and converts into a set of work items that
* can be executed by the blockMover. * can be executed by the blockMover.
* *
* @param planId - A SHA512 of the plan string * @param planId - A SHA-1 of the plan string
* @param planVersion - version of the plan string - for future use. * @param planVersion - version of the plan string - for future use.
* @param planFileName - Plan file name * @param planFileName - Plan file name
* @param planData - Plan data in json format * @param planData - Plan data in json format
@ -308,7 +308,7 @@ public class DiskBalancer {
/** /**
* Verifies that user provided plan is valid. * Verifies that user provided plan is valid.
* *
* @param planID - SHA 512 of the plan. * @param planID - SHA-1 of the plan.
* @param planVersion - Version of the plan, for future use. * @param planVersion - Version of the plan, for future use.
* @param plan - Plan String in Json. * @param plan - Plan String in Json.
* @param force - Skip verifying when the plan was generated. * @param force - Skip verifying when the plan was generated.
@ -345,15 +345,15 @@ public class DiskBalancer {
} }
/** /**
* Verifies that plan matches the SHA512 provided by the client. * Verifies that plan matches the SHA-1 provided by the client.
* *
* @param planID - Sha512 Hex Bytes * @param planID - SHA-1 Hex Bytes
* @param plan - Plan String * @param plan - Plan String
* @throws DiskBalancerException * @throws DiskBalancerException
*/ */
private NodePlan verifyPlanHash(String planID, String plan) private NodePlan verifyPlanHash(String planID, String plan)
throws DiskBalancerException { throws DiskBalancerException {
final long sha512Length = 128; final long sha1Length = 40;
if (plan == null || plan.length() == 0) { if (plan == null || plan.length() == 0) {
LOG.error("Disk Balancer - Invalid plan."); LOG.error("Disk Balancer - Invalid plan.");
throw new DiskBalancerException("Invalid plan.", throw new DiskBalancerException("Invalid plan.",
@ -361,8 +361,8 @@ public class DiskBalancer {
} }
if ((planID == null) || if ((planID == null) ||
(planID.length() != sha512Length) || (planID.length() != sha1Length) ||
!DigestUtils.sha512Hex(plan.getBytes(Charset.forName("UTF-8"))) !DigestUtils.shaHex(plan.getBytes(Charset.forName("UTF-8")))
.equalsIgnoreCase(planID)) { .equalsIgnoreCase(planID)) {
LOG.error("Disk Balancer - Invalid plan hash."); LOG.error("Disk Balancer - Invalid plan hash.");
throw new DiskBalancerException("Invalid or mis-matched hash.", throw new DiskBalancerException("Invalid or mis-matched hash.",

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
View File

@ -93,7 +93,7 @@ public class CancelCommand extends Command {
String dataNodeAddress = plan.getNodeName() + ":" + plan.getPort(); String dataNodeAddress = plan.getNodeName() + ":" + plan.getPort();
Preconditions.checkNotNull(dataNodeAddress); Preconditions.checkNotNull(dataNodeAddress);
ClientDatanodeProtocol dataNode = getDataNodeProxy(dataNodeAddress); ClientDatanodeProtocol dataNode = getDataNodeProxy(dataNodeAddress);
String planHash = DigestUtils.sha512Hex(planData); String planHash = DigestUtils.shaHex(planData);
try { try {
dataNode.cancelDiskBalancePlan(planHash); dataNode.cancelDiskBalancePlan(planHash);
} catch (DiskBalancerException ex) { } catch (DiskBalancerException ex) {

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
View File

@ -85,7 +85,7 @@ public class ExecuteCommand extends Command {
String dataNodeAddress = plan.getNodeName() + ":" + plan.getPort(); String dataNodeAddress = plan.getNodeName() + ":" + plan.getPort();
Preconditions.checkNotNull(dataNodeAddress); Preconditions.checkNotNull(dataNodeAddress);
ClientDatanodeProtocol dataNode = getDataNodeProxy(dataNodeAddress); ClientDatanodeProtocol dataNode = getDataNodeProxy(dataNodeAddress);
String planHash = DigestUtils.sha512Hex(planData); String planHash = DigestUtils.shaHex(planData);
try { try {
// TODO : Support skipping date check. // TODO : Support skipping date check.
dataNode.submitDiskBalancerPlan(planHash, DiskBalancer.PLAN_VERSION, dataNode.submitDiskBalancerPlan(planHash, DiskBalancer.PLAN_VERSION,

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java
View File

@ -191,7 +191,7 @@ public class TestDiskBalancer {
plan.setNodeUUID(dnNode.getDatanodeUuid()); plan.setNodeUUID(dnNode.getDatanodeUuid());
plan.setTimeStamp(Time.now()); plan.setTimeStamp(Time.now());
String planJson = plan.toJson(); String planJson = plan.toJson();
String planID = DigestUtils.sha512Hex(planJson); String planID = DigestUtils.shaHex(planJson);
assertNotNull(plan.getVolumeSetPlans()); assertNotNull(plan.getVolumeSetPlans());
assertTrue(plan.getVolumeSetPlans().size() > 0); assertTrue(plan.getVolumeSetPlans().size() > 0);
plan.getVolumeSetPlans().get(0).setTolerancePercent(10); plan.getVolumeSetPlans().get(0).setTolerancePercent(10);
@ -307,7 +307,7 @@ public class TestDiskBalancer {
plan.setNodeUUID(dnNode.getDatanodeUuid()); plan.setNodeUUID(dnNode.getDatanodeUuid());
plan.setTimeStamp(Time.now()); plan.setTimeStamp(Time.now());
String planJson = plan.toJson(); String planJson = plan.toJson();
String planID = DigestUtils.sha512Hex(planJson); String planID = DigestUtils.shaHex(planJson);
dataNode.submitDiskBalancerPlan(planID, 1, PLAN_FILE, planJson, false); dataNode.submitDiskBalancerPlan(planID, 1, PLAN_FILE, planJson, false);

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
View File

@ -317,7 +317,7 @@ public class TestDiskBalancerRPC {
plan = new NodePlan(node.getDataNodeName(), node.getDataNodePort()); plan = new NodePlan(node.getDataNodeName(), node.getDataNodePort());
planner.balanceVolumeSet(node, node.getVolumeSets().get("DISK"), plan); planner.balanceVolumeSet(node, node.getVolumeSets().get("DISK"), plan);
planVersion = 1; planVersion = 1;
planHash = DigestUtils.sha512Hex(plan.toJson()); planHash = DigestUtils.shaHex(plan.toJson());
return this; return this;
} }
} }

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java
View File

@ -125,7 +125,7 @@ public class TestDiskBalancerWithMockMover {
private void executeSubmitPlan(NodePlan plan, DiskBalancer balancer, private void executeSubmitPlan(NodePlan plan, DiskBalancer balancer,
int version) throws IOException { int version) throws IOException {
String planJson = plan.toJson(); String planJson = plan.toJson();
String planID = DigestUtils.sha512Hex(planJson); String planID = DigestUtils.shaHex(planJson);
balancer.submitPlan(planID, version, PLAN_FILE, planJson, false); balancer.submitPlan(planID, version, PLAN_FILE, planJson, false);
} }
@ -214,7 +214,7 @@ public class TestDiskBalancerWithMockMover {
NodePlan plan = mockMoverHelper.getPlan(); NodePlan plan = mockMoverHelper.getPlan();
DiskBalancer balancer = mockMoverHelper.getBalancer(); DiskBalancer balancer = mockMoverHelper.getBalancer();
String planJson = plan.toJson(); String planJson = plan.toJson();
String planID = DigestUtils.sha512Hex(planJson); String planID = DigestUtils.shaHex(planJson);
thrown.expect(DiskBalancerException.class); thrown.expect(DiskBalancerException.class);
thrown.expect(new DiskBalancerResultVerifier(DiskBalancerException thrown.expect(new DiskBalancerResultVerifier(DiskBalancerException
@ -231,7 +231,7 @@ public class TestDiskBalancerWithMockMover {
String planJson = plan.toJson(); String planJson = plan.toJson();
String planID = DigestUtils.sha512Hex(planJson); String planID = DigestUtils.shaHex(planJson);
char repChar = planID.charAt(0); char repChar = planID.charAt(0);
repChar++; repChar++;
@ -261,7 +261,7 @@ public class TestDiskBalancerWithMockMover {
String planJson = plan.toJson(); String planJson = plan.toJson();
String planID = DigestUtils.sha512Hex(planJson); String planID = DigestUtils.shaHex(planJson);
balancer.cancelPlan(planID); balancer.cancelPlan(planID);
DiskBalancerWorkStatus status = balancer.queryWorkStatus(); DiskBalancerWorkStatus status = balancer.queryWorkStatus();