From 66357574ae1da09ced735da36bf7d80a40c3fa1b Mon Sep 17 00:00:00 2001 From: Erik Krogen Date: Wed, 13 Mar 2019 09:38:10 -0700 Subject: [PATCH] HDFS-14346. Add better time precision to Configuration#getTimeDuration, allowing return unit and default unit to be specified independently. Contributed by Chao Sun. --- .../org/apache/hadoop/conf/Configuration.java | 60 ++++++++++++++++--- .../apache/hadoop/conf/TestConfiguration.java | 5 ++ .../hdfs/client/impl/DfsClientConf.java | 2 +- .../hadoop/hdfs/server/balancer/Balancer.java | 4 +- .../server/blockmanagement/BlockManager.java | 2 +- .../BlockPlacementPolicyDefault.java | 3 +- .../hadoop/hdfs/server/datanode/DNConf.java | 9 ++- .../hadoop/hdfs/server/mover/Mover.java | 4 +- .../server/namenode/ha/EditLogTailer.java | 11 ++-- 9 files changed, 77 insertions(+), 23 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 6d8f58764fb..272482b12d8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -1805,6 +1805,7 @@ public void setTimeDuration(String name, long value, TimeUnit unit) { * Return time duration in the given time unit. Valid units are encoded in * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds * (ms), seconds (s), minutes (m), hours (h), and days (d). + * * @param name Property name * @param defaultValue Value returned if no mapping exists. * @param unit Unit to convert the stored property, if it exists. @@ -1813,20 +1814,44 @@ public void setTimeDuration(String name, long value, TimeUnit unit) { * @return time duration in given time unit */ public long getTimeDuration(String name, long defaultValue, TimeUnit unit) { + return getTimeDuration(name, defaultValue, unit, unit); + } + + public long getTimeDuration(String name, String defaultValue, TimeUnit unit) { + return getTimeDuration(name, defaultValue, unit, unit); + } + + /** + * Return time duration in the given time unit. Valid units are encoded in + * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds + * (ms), seconds (s), minutes (m), hours (h), and days (d). If no unit is + * provided, the default unit is applied. + * + * @param name Property name + * @param defaultValue Value returned if no mapping exists. + * @param defaultUnit Default time unit if no valid suffix is provided. + * @param returnUnit The unit used for the returned value. + * @throws NumberFormatException If the property stripped of its unit is not + * a number + * @return time duration in given time unit + */ + public long getTimeDuration(String name, long defaultValue, + TimeUnit defaultUnit, TimeUnit returnUnit) { String vStr = get(name); if (null == vStr) { return defaultValue; } else { - return getTimeDurationHelper(name, vStr, unit); + return getTimeDurationHelper(name, vStr, defaultUnit, returnUnit); } } - public long getTimeDuration(String name, String defaultValue, TimeUnit unit) { + public long getTimeDuration(String name, String defaultValue, + TimeUnit defaultUnit, TimeUnit returnUnit) { String vStr = get(name); if (null == vStr) { - return getTimeDurationHelper(name, defaultValue, unit); + return getTimeDurationHelper(name, defaultValue, defaultUnit, returnUnit); } else { - return getTimeDurationHelper(name, vStr, unit); + return getTimeDurationHelper(name, vStr, defaultUnit, returnUnit); } } @@ -1834,26 +1859,43 @@ public long getTimeDuration(String name, String defaultValue, TimeUnit unit) { * Return time duration in the given time unit. Valid units are encoded in * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds * (ms), seconds (s), minutes (m), hours (h), and days (d). + * * @param name Property name * @param vStr The string value with time unit suffix to be converted. * @param unit Unit to convert the stored property, if it exists. */ public long getTimeDurationHelper(String name, String vStr, TimeUnit unit) { + return getTimeDurationHelper(name, vStr, unit, unit); + } + + /** + * Return time duration in the given time unit. Valid units are encoded in + * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds + * (ms), seconds (s), minutes (m), hours (h), and days (d). + * + * @param name Property name + * @param vStr The string value with time unit suffix to be converted. + * @param defaultUnit Unit to convert the stored property, if it exists. + * @param returnUnit Unit for the returned value. + */ + private long getTimeDurationHelper(String name, String vStr, + TimeUnit defaultUnit, TimeUnit returnUnit) { vStr = vStr.trim(); vStr = StringUtils.toLowerCase(vStr); ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr); if (null == vUnit) { - logDeprecation("No unit for " + name + "(" + vStr + ") assuming " + unit); - vUnit = ParsedTimeDuration.unitFor(unit); + logDeprecation("No unit for " + name + "(" + vStr + ") assuming " + + defaultUnit); + vUnit = ParsedTimeDuration.unitFor(defaultUnit); } else { vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix())); } long raw = Long.parseLong(vStr); - long converted = unit.convert(raw, vUnit.unit()); - if (vUnit.unit().convert(converted, unit) < raw) { + long converted = returnUnit.convert(raw, vUnit.unit()); + if (vUnit.unit().convert(converted, returnUnit) < raw) { logDeprecation("Possible loss of precision converting " + vStr - + vUnit.suffix() + " to " + unit + " for " + name); + + vUnit.suffix() + " to " + returnUnit + " for " + name); } return converted; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 4c80d105651..1cf7a20c610 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -1404,7 +1404,10 @@ public void testTimeDuration() { conf.setTimeDuration("test.time.a", 7L, SECONDS); assertEquals("7s", conf.get("test.time.a")); assertEquals(0L, conf.getTimeDuration("test.time.a", 30, MINUTES)); + assertEquals(0L, conf.getTimeDuration("test.time.a", 30, SECONDS, MINUTES)); assertEquals(7L, conf.getTimeDuration("test.time.a", 30, SECONDS)); + assertEquals(7L, + conf.getTimeDuration("test.time.a", 30, MILLISECONDS, SECONDS)); assertEquals(7000L, conf.getTimeDuration("test.time.a", 30, MILLISECONDS)); assertEquals(7000000L, conf.getTimeDuration("test.time.a", 30, MICROSECONDS)); @@ -1421,6 +1424,8 @@ public void testTimeDuration() { assertEquals(30L, conf.getTimeDuration("test.time.X", 30, SECONDS)); conf.set("test.time.X", "30"); assertEquals(30L, conf.getTimeDuration("test.time.X", 40, SECONDS)); + assertEquals(30000L, + conf.getTimeDuration("test.time.X", 40, SECONDS, MILLISECONDS)); assertEquals(10L, conf.getTimeDuration("test.time.c", "10", SECONDS)); assertEquals(30L, conf.getTimeDuration("test.time.c", "30s", SECONDS)); assertEquals(120L, conf.getTimeDuration("test.time.c", "2m", SECONDS)); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java index 4110f9413dc..04bdfe47960 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java @@ -248,7 +248,7 @@ public DfsClientConf(Configuration conf) { datanodeRestartTimeout = conf.getTimeDuration( DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY, DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT, - TimeUnit.SECONDS) * 1000; + TimeUnit.SECONDS, TimeUnit.MILLISECONDS); slowIoWarningThresholdMs = conf.getLong( DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY, DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index d21d13cd9b1..fe187cbc686 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -677,11 +677,11 @@ static int run(Collection namenodes, final BalancerParameters p, final long sleeptime = conf.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, - TimeUnit.SECONDS) * 2000 + + TimeUnit.SECONDS, TimeUnit.MILLISECONDS) * 2 + conf.getTimeDuration( DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT, - TimeUnit.SECONDS) * 1000; + TimeUnit.SECONDS, TimeUnit.MILLISECONDS); LOG.info("namenodes = " + namenodes); LOG.info("parameters = " + p); LOG.info("included nodes = " + p.getIncludedNodes()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 6d142f9e64a..f66ac42a9e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -520,7 +520,7 @@ public BlockManager(final Namesystem namesystem, boolean haEnabled, this.redundancyRecheckIntervalMs = conf.getTimeDuration( DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT, - TimeUnit.SECONDS) * 1000; + TimeUnit.SECONDS, TimeUnit.MILLISECONDS); this.storageInfoDefragmentInterval = conf.getLong( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 5b8a8b419fc..d13782686e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -124,7 +124,8 @@ public void initialize(Configuration conf, FSClusterStats stats, this.host2datanodeMap = host2datanodeMap; this.heartbeatInterval = conf.getTimeDuration( DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, - DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 1000; + DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, + TimeUnit.SECONDS, TimeUnit.MILLISECONDS); this.tolerateHeartbeatMultiplier = conf.getInt( DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_KEY, DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java index d396600a6ac..de26c90d37e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java @@ -199,7 +199,8 @@ public DNConf(final Configurable dn) { long initBRDelay = getConf().getTimeDuration( DFS_BLOCKREPORT_INITIAL_DELAY_KEY, - DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT, TimeUnit.SECONDS) * 1000L; + DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT, + TimeUnit.SECONDS, TimeUnit.MILLISECONDS); if (initBRDelay >= blockReportInterval) { initBRDelay = 0; DataNode.LOG.info("dfs.blockreport.initialDelay is " @@ -209,11 +210,13 @@ public DNConf(final Configurable dn) { initialBlockReportDelayMs = initBRDelay; heartBeatInterval = getConf().getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY, - DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 1000L; + DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS, + TimeUnit.MILLISECONDS); long confLifelineIntervalMs = getConf().getLong(DFS_DATANODE_LIFELINE_INTERVAL_SECONDS_KEY, 3 * getConf().getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY, - DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS)) * 1000L; + DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS, + TimeUnit.MILLISECONDS)); if (confLifelineIntervalMs <= heartBeatInterval) { confLifelineIntervalMs = 3 * heartBeatInterval; DataNode.LOG.warn( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java index 80df71195d1..06c7cc5a5ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java @@ -631,11 +631,11 @@ static int run(Map> namenodes, Configuration conf) final long sleeptime = conf.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, - TimeUnit.SECONDS) * 2000 + + TimeUnit.SECONDS, TimeUnit.MILLISECONDS) * 2 + conf.getTimeDuration( DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT, - TimeUnit.SECONDS) * 1000; + TimeUnit.SECONDS, TimeUnit.MILLISECONDS); AtomicInteger retryCount = new AtomicInteger(0); // TODO: Need to limit the size of the pinned blocks to limit memory usage Map> excludedPinnedBlocks = new HashMap<>(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java index 0ab8ef19902..9d0602e0252 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java @@ -176,7 +176,8 @@ public EditLogTailer(FSNamesystem namesystem, Configuration conf) { logRollPeriodMs = conf.getTimeDuration( DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, - DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT, TimeUnit.SECONDS) * 1000; + DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT, + TimeUnit.SECONDS, TimeUnit.MILLISECONDS); List nns = Collections.emptyList(); if (logRollPeriodMs >= 0) { try { @@ -203,11 +204,13 @@ public EditLogTailer(FSNamesystem namesystem, Configuration conf) { sleepTimeMs = conf.getTimeDuration( DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, - DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT, TimeUnit.SECONDS) * 1000; + DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT, + TimeUnit.SECONDS, TimeUnit.MILLISECONDS); - rollEditsTimeoutMs = conf.getInt( + rollEditsTimeoutMs = conf.getTimeDuration( DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_KEY, - DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_DEFAULT) * 1000; + DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_DEFAULT, + TimeUnit.SECONDS, TimeUnit.MILLISECONDS); rollEditsRpcExecutor = Executors.newSingleThreadExecutor( new ThreadFactoryBuilder().setDaemon(true).build());