Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14346. Add better time precision to Configuration#getTimeDuration, allowing return unit and default unit to be specified independently. Contributed by Chao Sun.

Browse Source 
(cherry picked from commit 66357574ae1da09ced735da36bf7d80a40c3fa1b)
(cherry picked from commit fec7c5f3ebbaea7b290e0904570c3a485b541a22)
(cherry picked from commit cb76fc5618c582887e4e71b98f3065c5a916334a)
This commit is contained in:
Erik Krogen 2019-03-13 09:38:10 -07:00
parent 7f70a78ce3
commit 9a5daa5ec6
9 changed files with 77 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -1740,6 +1740,7 @@ public void setTimeDuration(String name, long value, TimeUnit unit) {
* Return time duration in the given time unit. Valid units are encoded in
* properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
* (ms), seconds (s), minutes (m), hours (h), and days (d).
*
* @param name Property name
* @param defaultValue Value returned if no mapping exists.
* @param unit Unit to convert the stored property, if it exists.
@ -1747,20 +1748,44 @@ public void setTimeDuration(String name, long value, TimeUnit unit) {
* a number
*/
public long getTimeDuration(String name, long defaultValue, TimeUnit unit) {
return getTimeDuration(name, defaultValue, unit, unit);
}
public long getTimeDuration(String name, String defaultValue, TimeUnit unit) {
return getTimeDuration(name, defaultValue, unit, unit);
}
/**
* Return time duration in the given time unit. Valid units are encoded in
* properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
* (ms), seconds (s), minutes (m), hours (h), and days (d). If no unit is
* provided, the default unit is applied.
*
* @param name Property name
* @param defaultValue Value returned if no mapping exists.
* @param defaultUnit Default time unit if no valid suffix is provided.
* @param returnUnit The unit used for the returned value.
* @throws NumberFormatException If the property stripped of its unit is not
* a number
* @return time duration in given time unit
*/
public long getTimeDuration(String name, long defaultValue,
TimeUnit defaultUnit, TimeUnit returnUnit) {
String vStr = get(name);
if (null == vStr) {
return defaultValue;
} else {
return getTimeDurationHelper(name, vStr, unit);
return getTimeDurationHelper(name, vStr, defaultUnit, returnUnit);
}
}
public long getTimeDuration(String name, String defaultValue, TimeUnit unit) {
public long getTimeDuration(String name, String defaultValue,
TimeUnit defaultUnit, TimeUnit returnUnit) {
String vStr = get(name);
if (null == vStr) {
return getTimeDurationHelper(name, defaultValue, unit);
return getTimeDurationHelper(name, defaultValue, defaultUnit, returnUnit);
} else {
return getTimeDurationHelper(name, vStr, unit);
return getTimeDurationHelper(name, vStr, defaultUnit, returnUnit);
}
}
@ -1768,26 +1793,43 @@ public long getTimeDuration(String name, String defaultValue, TimeUnit unit) {
* Return time duration in the given time unit. Valid units are encoded in
* properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
* (ms), seconds (s), minutes (m), hours (h), and days (d).
*
* @param name Property name
* @param vStr The string value with time unit suffix to be converted.
* @param unit Unit to convert the stored property, if it exists.
*/
public long getTimeDurationHelper(String name, String vStr, TimeUnit unit) {
return getTimeDurationHelper(name, vStr, unit, unit);
}
/**
* Return time duration in the given time unit. Valid units are encoded in
* properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
* (ms), seconds (s), minutes (m), hours (h), and days (d).
*
* @param name Property name
* @param vStr The string value with time unit suffix to be converted.
* @param defaultUnit Unit to convert the stored property, if it exists.
* @param returnUnit Unit for the returned value.
*/
private long getTimeDurationHelper(String name, String vStr,
TimeUnit defaultUnit, TimeUnit returnUnit) {
vStr = vStr.trim();
vStr = StringUtils.toLowerCase(vStr);
ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr);
if (null == vUnit) {
logDeprecation("No unit for " + name + "(" + vStr + ") assuming " + unit);
vUnit = ParsedTimeDuration.unitFor(unit);
logDeprecation("No unit for " + name + "(" + vStr + ") assuming " +
defaultUnit);
vUnit = ParsedTimeDuration.unitFor(defaultUnit);
} else {
vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix()));
}
long raw = Long.parseLong(vStr);
long converted = unit.convert(raw, vUnit.unit());
if (vUnit.unit().convert(converted, unit) < raw) {
long converted = returnUnit.convert(raw, vUnit.unit());
if (vUnit.unit().convert(converted, returnUnit) < raw) {
logDeprecation("Possible loss of precision converting " + vStr
+ vUnit.suffix() + " to " + unit + " for " + name);
+ vUnit.suffix() + " to " + returnUnit + " for " + name);
}
return converted;
}

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
View File

@ -1370,7 +1370,10 @@ public void testTimeDuration() {
conf.setTimeDuration("test.time.a", 7L, SECONDS);
assertEquals("7s", conf.get("test.time.a"));
assertEquals(0L, conf.getTimeDuration("test.time.a", 30, MINUTES));
assertEquals(0L, conf.getTimeDuration("test.time.a", 30, SECONDS, MINUTES));
assertEquals(7L, conf.getTimeDuration("test.time.a", 30, SECONDS));
assertEquals(7L,
conf.getTimeDuration("test.time.a", 30, MILLISECONDS, SECONDS));
assertEquals(7000L, conf.getTimeDuration("test.time.a", 30, MILLISECONDS));
assertEquals(7000000L,
conf.getTimeDuration("test.time.a", 30, MICROSECONDS));
@ -1387,6 +1390,8 @@ public void testTimeDuration() {
assertEquals(30L, conf.getTimeDuration("test.time.X", 30, SECONDS));
conf.set("test.time.X", "30");
assertEquals(30L, conf.getTimeDuration("test.time.X", 40, SECONDS));
assertEquals(30000L,
conf.getTimeDuration("test.time.X", 40, SECONDS, MILLISECONDS));
assertEquals(10L, conf.getTimeDuration("test.time.c", "10", SECONDS));
assertEquals(30L, conf.getTimeDuration("test.time.c", "30s", SECONDS));
assertEquals(120L, conf.getTimeDuration("test.time.c", "2m", SECONDS));

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
View File

@ -239,7 +239,7 @@ public DfsClientConf(Configuration conf) {
datanodeRestartTimeout = conf.getTimeDuration(
DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY,
DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT,
TimeUnit.SECONDS) * 1000;
TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
slowIoWarningThresholdMs = conf.getLong(
DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY,
DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT);

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
View File

@ -676,11 +676,11 @@ static int run(Collection<URI> namenodes, final BalancerParameters p,
final long sleeptime =
conf.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
TimeUnit.SECONDS) * 2000 +
TimeUnit.SECONDS, TimeUnit.MILLISECONDS) * 2 +
conf.getTimeDuration(
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT,
TimeUnit.SECONDS) * 1000;
TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
LOG.info("namenodes = " + namenodes);
LOG.info("parameters = " + p);
LOG.info("included nodes = " + p.getIncludedNodes());

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -508,7 +508,7 @@ public BlockManager(final Namesystem namesystem, boolean haEnabled,
this.redundancyRecheckIntervalMs = conf.getTimeDuration(
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT,
TimeUnit.SECONDS) * 1000;
TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
this.storageInfoDefragmentInterval =
conf.getLong(

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
View File

@ -116,7 +116,8 @@ public void initialize(Configuration conf, FSClusterStats stats,
this.host2datanodeMap = host2datanodeMap;
this.heartbeatInterval = conf.getTimeDuration(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 1000;
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
this.tolerateHeartbeatMultiplier = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_KEY,
DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_DEFAULT);

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
View File

@ -199,7 +199,8 @@ public DNConf(final Configurable dn) {
long initBRDelay = getConf().getTimeDuration(
DFS_BLOCKREPORT_INITIAL_DELAY_KEY,
DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT, TimeUnit.SECONDS) * 1000L;
DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT,
TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
if (initBRDelay >= blockReportInterval) {
initBRDelay = 0;
DataNode.LOG.info("dfs.blockreport.initialDelay is "
@ -209,11 +210,13 @@ public DNConf(final Configurable dn) {
initialBlockReportDelayMs = initBRDelay;
heartBeatInterval = getConf().getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY,
DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 1000L;
DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS,
TimeUnit.MILLISECONDS);
long confLifelineIntervalMs =
getConf().getLong(DFS_DATANODE_LIFELINE_INTERVAL_SECONDS_KEY,
3 * getConf().getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY,
DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS)) * 1000L;
DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS,
TimeUnit.MILLISECONDS));
if (confLifelineIntervalMs <= heartBeatInterval) {
confLifelineIntervalMs = 3 * heartBeatInterval;
DataNode.LOG.warn(

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
View File

@ -631,11 +631,11 @@ static int run(Map<URI, List<Path>> namenodes, Configuration conf)
final long sleeptime =
conf.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
TimeUnit.SECONDS) * 2000 +
TimeUnit.SECONDS, TimeUnit.MILLISECONDS) * 2 +
conf.getTimeDuration(
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT,
TimeUnit.SECONDS) * 1000;
TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
AtomicInteger retryCount = new AtomicInteger(0);
// TODO: Need to limit the size of the pinned blocks to limit memory usage
Map<Long, Set<DatanodeInfo>> excludedPinnedBlocks = new HashMap<>();

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
View File

@ -172,7 +172,8 @@ public EditLogTailer(FSNamesystem namesystem, Configuration conf) {
logRollPeriodMs = conf.getTimeDuration(
DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY,
DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT, TimeUnit.SECONDS) * 1000;
DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT,
TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
List<RemoteNameNodeInfo> nns = Collections.emptyList();
if (logRollPeriodMs >= 0) {
try {
@ -199,11 +200,13 @@ public EditLogTailer(FSNamesystem namesystem, Configuration conf) {
sleepTimeMs = conf.getTimeDuration(
DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,
DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT, TimeUnit.SECONDS) * 1000;
DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT,
TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
rollEditsTimeoutMs = conf.getInt(
rollEditsTimeoutMs = conf.getTimeDuration(
DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_KEY,
DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_DEFAULT) * 1000;
DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_DEFAULT,
TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
rollEditsRpcExecutor = Executors.newSingleThreadExecutor(
new ThreadFactoryBuilder().setDaemon(true).build());