HDFS-10518. DiskBalancer: Pretty-print json in Query command. Contributed by Anu Engineer.
This commit is contained in:
parent
af11ab34d0
commit
7e2be5c4a0
|
@ -34,6 +34,8 @@ import java.io.IOException;
|
|||
@InterfaceStability.Unstable
|
||||
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||
public class DiskBalancerWorkItem {
|
||||
private long startTime;
|
||||
private long secondsElapsed;
|
||||
private long bytesToCopy;
|
||||
private long bytesCopied;
|
||||
private long errorCount;
|
||||
|
@ -242,4 +244,44 @@ public class DiskBalancerWorkItem {
|
|||
public void setBandwidth(long bandwidth) {
|
||||
this.bandwidth = bandwidth;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Records the Start time of execution.
|
||||
* @return startTime
|
||||
*/
|
||||
public long getStartTime() {
|
||||
return startTime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the Start time.
|
||||
* @param startTime - Time stamp for start of execution.
|
||||
*/
|
||||
public void setStartTime(long startTime) {
|
||||
this.startTime = startTime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the number of seconds elapsed from the start time.
|
||||
*
|
||||
* The reason why we have this is of time skews. The client's current time
|
||||
* may not match with the server time stamp, hence the elapsed second
|
||||
* cannot be computed from only startTime.
|
||||
*
|
||||
* @return seconds elapsed from start time.
|
||||
*/
|
||||
public long getSecondsElapsed() {
|
||||
return secondsElapsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets number of seconds elapsed.
|
||||
*
|
||||
* This is updated whenever we update the other counters.
|
||||
* @param secondsElapsed - seconds elapsed.
|
||||
*/
|
||||
public void setSecondsElapsed(long secondsElapsed) {
|
||||
this.secondsElapsed = secondsElapsed;
|
||||
}
|
||||
}
|
|
@ -24,6 +24,7 @@ import com.google.common.base.Preconditions;
|
|||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
import org.codehaus.jackson.map.SerializationConfig;
|
||||
|
||||
import static org.codehaus.jackson.map.type.TypeFactory.defaultInstance;
|
||||
|
||||
|
@ -128,6 +129,7 @@ public class DiskBalancerWorkStatus {
|
|||
**/
|
||||
public String currentStateString() throws IOException {
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
mapper.enable(SerializationConfig.Feature.INDENT_OUTPUT);
|
||||
return mapper.writeValueAsString(currentState);
|
||||
}
|
||||
|
||||
|
|
|
@ -552,6 +552,21 @@ public class DiskBalancer {
|
|||
* @return FsDatasetSpi
|
||||
*/
|
||||
FsDatasetSpi getDataset();
|
||||
|
||||
/**
|
||||
* Returns time when this plan started executing.
|
||||
*
|
||||
* @return Start time in milliseconds.
|
||||
*/
|
||||
long getStartTime();
|
||||
|
||||
/**
|
||||
* Number of seconds elapsed.
|
||||
*
|
||||
* @return time in seconds
|
||||
*/
|
||||
long getElapsedSeconds();
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -622,6 +637,8 @@ public class DiskBalancer {
|
|||
private long maxDiskErrors;
|
||||
private int poolIndex;
|
||||
private AtomicBoolean shouldRun;
|
||||
private long startTime;
|
||||
private long secondsElapsed;
|
||||
|
||||
/**
|
||||
* Constructs diskBalancerMover.
|
||||
|
@ -897,6 +914,9 @@ public class DiskBalancer {
|
|||
FsVolumeSpi source = pair.getSource();
|
||||
FsVolumeSpi dest = pair.getDest();
|
||||
List<FsVolumeSpi.BlockIterator> poolIters = new LinkedList<>();
|
||||
startTime = Time.now();
|
||||
item.setStartTime(startTime);
|
||||
secondsElapsed = 0;
|
||||
|
||||
if (source.isTransientStorage() || dest.isTransientStorage()) {
|
||||
return;
|
||||
|
@ -937,7 +957,7 @@ public class DiskBalancer {
|
|||
if (block == null) {
|
||||
this.setExitFlag();
|
||||
LOG.error("No source blocks, exiting the copy. Source: {}, " +
|
||||
"dest:{}", source.getBasePath(), dest.getBasePath());
|
||||
"dest:{}", source.getBasePath(), dest.getBasePath());
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -973,9 +993,6 @@ public class DiskBalancer {
|
|||
block.getNumBytes(), source.getBasePath(),
|
||||
dest.getBasePath());
|
||||
|
||||
item.incCopiedSoFar(block.getNumBytes());
|
||||
item.incBlocksCopied();
|
||||
|
||||
// Check for the max throughput constraint.
|
||||
// We sleep here to keep the promise that we will not
|
||||
// copy more than Max MB/sec. we sleep enough time
|
||||
|
@ -984,6 +1001,14 @@ public class DiskBalancer {
|
|||
// we exit via Thread Interrupted exception.
|
||||
Thread.sleep(computeDelay(block.getNumBytes(), timeUsed, item));
|
||||
|
||||
// We delay updating the info to avoid confusing the user.
|
||||
// This way we report the copy only if it is under the
|
||||
// throughput threshold.
|
||||
item.incCopiedSoFar(block.getNumBytes());
|
||||
item.incBlocksCopied();
|
||||
secondsElapsed = TimeUnit.MILLISECONDS.toSeconds(Time.now() -
|
||||
startTime);
|
||||
item.setSecondsElapsed(secondsElapsed);
|
||||
} catch (IOException ex) {
|
||||
LOG.error("Exception while trying to copy blocks. error: {}", ex);
|
||||
item.incErrorCount();
|
||||
|
@ -1009,5 +1034,25 @@ public class DiskBalancer {
|
|||
public FsDatasetSpi getDataset() {
|
||||
return dataset;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns time when this plan started executing.
|
||||
*
|
||||
* @return Start time in milliseconds.
|
||||
*/
|
||||
@Override
|
||||
public long getStartTime() {
|
||||
return startTime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Number of seconds elapsed.
|
||||
*
|
||||
* @return time in seconds
|
||||
*/
|
||||
@Override
|
||||
public long getElapsedSeconds() {
|
||||
return secondsElapsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -46,6 +46,7 @@ public class QueryCommand extends Command {
|
|||
" plan running on a given datanode.");
|
||||
addValidCommandParameters(DiskBalancer.VERBOSE, "Prints verbose results.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the Client Calls.
|
||||
*
|
||||
|
@ -62,7 +63,7 @@ public class QueryCommand extends Command {
|
|||
String nodeAddress = nodeName;
|
||||
|
||||
// if the string is not name:port format use the default port.
|
||||
if(!nodeName.matches("^.*:\\d$")) {
|
||||
if (!nodeName.matches("^.*:\\d$")) {
|
||||
int defaultIPC = NetUtils.createSocketAddr(
|
||||
getConf().getTrimmed(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY,
|
||||
DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_DEFAULT)).getPort();
|
||||
|
@ -76,7 +77,7 @@ public class QueryCommand extends Command {
|
|||
System.out.printf("Plan ID: %s %nResult: %s%n", workStatus.getPlanID(),
|
||||
workStatus.getResult().toString());
|
||||
|
||||
if(cmd.hasOption(DiskBalancer.VERBOSE)) {
|
||||
if (cmd.hasOption(DiskBalancer.VERBOSE)) {
|
||||
System.out.printf("%s", workStatus.currentStateString());
|
||||
}
|
||||
} catch (DiskBalancerException ex) {
|
||||
|
|
|
@ -436,6 +436,26 @@ public class TestDiskBalancerWithMockMover {
|
|||
return this.dataset;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns time when this plan started executing.
|
||||
*
|
||||
* @return Start time in milliseconds.
|
||||
*/
|
||||
@Override
|
||||
public long getStartTime() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Number of seconds elapsed.
|
||||
*
|
||||
* @return time in seconds
|
||||
*/
|
||||
@Override
|
||||
public long getElapsedSeconds() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
public int getRunCount() {
|
||||
synchronized (runCount) {
|
||||
LOG.info("Run count : " + runCount.intValue());
|
||||
|
|
Loading…
Reference in New Issue