Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14991. Backport better time precision of configuration#getTimeDuration to branch-2 to support SBN read. Contributed by Chen Liang.

Browse Source
This commit is contained in:
Chen Liang 2019-11-18 20:26:27 -08:00
parent a74411ebaa
commit 238781ab98
3 changed files with 61 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -1680,6 +1680,7 @@ public void setTimeDuration(String name, long value, TimeUnit unit) {
* Return time duration in the given time unit. Valid units are encoded in
* properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
* (ms), seconds (s), minutes (m), hours (h), and days (d).
*
* @param name Property name
* @param defaultValue Value returned if no mapping exists.
* @param unit Unit to convert the stored property, if it exists.
@ -1687,23 +1688,58 @@ public void setTimeDuration(String name, long value, TimeUnit unit) {
* a number
*/
public long getTimeDuration(String name, long defaultValue, TimeUnit unit) {
return getTimeDuration(name, defaultValue, unit, unit);
}
/**
* Return time duration in the given time unit. Valid units are encoded in
* properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
* (ms), seconds (s), minutes (m), hours (h), and days (d). If no unit is
* provided, the default unit is applied.
*
* @param name Property name
* @param defaultValue Value returned if no mapping exists.
* @param defaultUnit Default time unit if no valid suffix is provided.
* @param returnUnit The unit used for the returned value.
* @throws NumberFormatException If the property stripped of its unit is not
* a number
* @return time duration in given time unit
*/
public long getTimeDuration(String name, long defaultValue,
TimeUnit defaultUnit, TimeUnit returnUnit) {
String vStr = get(name);
if (null == vStr) {
return defaultValue;
return returnUnit.convert(defaultValue, defaultUnit);
}
vStr = vStr.trim();
return getTimeDurationHelper(name, vStr, unit);
return getTimeDurationHelper(name, vStr, defaultUnit, returnUnit);
}
private long getTimeDurationHelper(String name, String vStr, TimeUnit unit) {
return getTimeDurationHelper(name, vStr, unit, unit);
}
/**
* Return time duration in the given time unit. Valid units are encoded in
* properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
* (ms), seconds (s), minutes (m), hours (h), and days (d).
*
* @param name Property name
* @param vStr The string value with time unit suffix to be converted.
* @param defaultUnit Unit to convert the stored property, if it exists.
* @param returnUnit Unit for the returned value.
*/
private long getTimeDurationHelper(String name, String vStr,
TimeUnit defaultUnit, TimeUnit returnUnit) {
ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr);
if (null == vUnit) {
LOG.warn("No unit for " + name + "(" + vStr + ") assuming " + unit);
vUnit = ParsedTimeDuration.unitFor(unit);
LOG.warn("No unit for " + name + "(" + vStr + ") assuming "
+ defaultUnit);
vUnit = ParsedTimeDuration.unitFor(defaultUnit);
} else {
vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix()));
}
return unit.convert(Long.parseLong(vStr), vUnit.unit());
return returnUnit.convert(Long.parseLong(vStr), vUnit.unit());
}
public long[] getTimeDurations(String name, TimeUnit unit) {

9
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
View File

@ -1225,10 +1225,17 @@ public void testEnumFromXml() throws IOException {
public void testTimeDuration() {
Configuration conf = new Configuration(false);
assertEquals(7000L,
conf.getTimeDuration("test.time.a", 7L, SECONDS, MILLISECONDS));
conf.setTimeDuration("test.time.a", 7L, SECONDS);
assertEquals("7s", conf.get("test.time.a"));
assertEquals(0L, conf.getTimeDuration("test.time.a", 30, MINUTES));
assertEquals(0L, conf.getTimeDuration("test.time.a", 30, SECONDS, MINUTES));
assertEquals(7L, conf.getTimeDuration("test.time.a", 30, SECONDS));
assertEquals(7L,
conf.getTimeDuration("test.time.a", 30, MILLISECONDS, SECONDS));
assertEquals(7000L, conf.getTimeDuration("test.time.a", 30, MILLISECONDS));
assertEquals(7000000L,
conf.getTimeDuration("test.time.a", 30, MICROSECONDS));
@ -1245,6 +1252,8 @@ public void testTimeDuration() {
assertEquals(30L, conf.getTimeDuration("test.time.X", 30, SECONDS));
conf.set("test.time.X", "30");
assertEquals(30L, conf.getTimeDuration("test.time.X", 40, SECONDS));
assertEquals(30000L,
conf.getTimeDuration("test.time.X", 40, SECONDS, MILLISECONDS));
for (Configuration.ParsedTimeDuration ptd :
Configuration.ParsedTimeDuration.values()) {

17
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
View File

@ -162,8 +162,10 @@ public EditLogTailer(FSNamesystem namesystem, Configuration conf) {
lastLoadTimeMs = monotonicNow();
lastRollTimeMs = monotonicNow();
logRollPeriodMs = conf.getInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY,
DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT) * 1000;
logRollPeriodMs = conf.getTimeDuration(
DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY,
DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT,
TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
List<RemoteNameNodeInfo> nns = Collections.emptyList();
if (logRollPeriodMs >= 0) {
try {
@ -188,8 +190,10 @@ public EditLogTailer(FSNamesystem namesystem, Configuration conf) {
DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY + " is negative.");
}
sleepTimeMs = conf.getInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,
DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT) * 1000;
sleepTimeMs = conf.getTimeDuration(
DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,
DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT,
TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
long maxSleepTimeMsTemp = conf.getTimeDuration(
DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_BACKOFF_MAX_KEY,
DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_BACKOFF_MAX_DEFAULT,
@ -223,9 +227,10 @@ public EditLogTailer(FSNamesystem namesystem, Configuration conf) {
// setup the iterator to endlessly loop the nns
this.nnLookup = Iterators.cycle(nns);
rollEditsTimeoutMs = conf.getInt(
rollEditsTimeoutMs = conf.getTimeDuration(
DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_KEY,
DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_DEFAULT) * 1000;
DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_DEFAULT,
TimeUnit.SECONDS, TimeUnit.MILLISECONDS);
rollEditsRpcExecutor = Executors.newSingleThreadExecutor(
new ThreadFactoryBuilder().setDaemon(true).build());