diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java index 9ffcea1a95c..5c5b1328800 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/NamenodeHeartbeatService.java @@ -21,6 +21,8 @@ import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEALTH_MONITOR_TIMEOUT_DEFAULT; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS; import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_HEARTBEAT_INTERVAL_MS_DEFAULT; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_NAMENODE_HEARTBEAT_JMX_INTERVAL_MS; +import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_NAMENODE_HEARTBEAT_JMX_INTERVAL_MS_DEFAULT; import java.io.IOException; import java.net.InetAddress; @@ -47,6 +49,7 @@ import org.apache.hadoop.hdfs.tools.NNHAServiceTarget; import org.apache.hadoop.hdfs.web.URLConnectionFactory; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.util.Time; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -107,6 +110,15 @@ public class NamenodeHeartbeatService extends PeriodicService { /** URL scheme to use for JMX calls. */ private String scheme; + /** Frequency of updates to JMX report. */ + private long updateJmxIntervalMs; + /** Timestamp of last attempt to update JMX report. */ + private long lastJmxUpdateAttempt; + /** Result of the last successful FsNamesystemMetrics report. */ + private JSONArray fsNamesystemMetrics; + /** Result of the last successful NamenodeInfoMetrics report. */ + private JSONArray namenodeInfoMetrics; + private String resolvedHost; private String originalNnId; @@ -233,6 +245,9 @@ public class NamenodeHeartbeatService extends PeriodicService { this.healthMonitorTimeoutMs = (int) timeoutMs; } + this.updateJmxIntervalMs = conf.getTimeDuration(DFS_ROUTER_NAMENODE_HEARTBEAT_JMX_INTERVAL_MS, + DFS_ROUTER_NAMENODE_HEARTBEAT_JMX_INTERVAL_MS_DEFAULT, TimeUnit.MILLISECONDS); + super.serviceInit(configuration); } @@ -447,8 +462,13 @@ public class NamenodeHeartbeatService extends PeriodicService { String address, NamenodeStatusReport report) { try { // TODO part of this should be moved to its own utility - getFsNamesystemMetrics(address, report); - getNamenodeInfoMetrics(address, report); + if (shouldUpdateJmx()) { + this.lastJmxUpdateAttempt = Time.monotonicNow(); + getFsNamesystemMetrics(address); + getNamenodeInfoMetrics(address); + } + populateFsNamesystemMetrics(this.fsNamesystemMetrics, report); + populateNamenodeInfoMetrics(this.namenodeInfoMetrics, report); } catch (Exception e) { LOG.error("Cannot get stat from {} using JMX", getNamenodeDesc(), e); } @@ -482,17 +502,38 @@ public class NamenodeHeartbeatService extends PeriodicService { } } + /** + * Evaluates whether the JMX report should be refreshed by + * calling the Namenode, based on the following conditions: + * 1. JMX Updates must be enabled. + * 2. The last attempt to update JMX occurred before the + * configured interval (if any). + */ + private boolean shouldUpdateJmx() { + if (this.updateJmxIntervalMs < 0) { + return false; + } + + return Time.monotonicNow() - this.lastJmxUpdateAttempt > this.updateJmxIntervalMs; + } + /** * Fetches NamenodeInfo metrics from namenode. * @param address Web interface of the Namenode to monitor. - * @param report Namenode status report to update with JMX data. - * @throws JSONException */ - private void getNamenodeInfoMetrics(String address, - NamenodeStatusReport report) throws JSONException { + private void getNamenodeInfoMetrics(String address) { String query = "Hadoop:service=NameNode,name=NameNodeInfo"; - JSONArray aux = - FederationUtil.getJmx(query, address, connectionFactory, scheme); + this.namenodeInfoMetrics = FederationUtil.getJmx(query, address, connectionFactory, scheme); + } + + /** + * Populates NamenodeInfo metrics into report. + * @param aux NamenodeInfo metrics from namenode. + * @param report Namenode status report to update with JMX data. + * @throws JSONException When an invalid JSONObject is found + */ + private void populateNamenodeInfoMetrics(JSONArray aux, NamenodeStatusReport report) + throws JSONException { if (aux != null && aux.length() > 0) { JSONObject jsonObject = aux.getJSONObject(0); String name = jsonObject.getString("name"); @@ -510,14 +551,20 @@ public class NamenodeHeartbeatService extends PeriodicService { /** * Fetches FSNamesystem* metrics from namenode. * @param address Web interface of the Namenode to monitor. - * @param report Namenode status report to update with JMX data. - * @throws JSONException */ - private void getFsNamesystemMetrics(String address, - NamenodeStatusReport report) throws JSONException { + private void getFsNamesystemMetrics(String address) { String query = "Hadoop:service=NameNode,name=FSNamesystem*"; - JSONArray aux = FederationUtil.getJmx( - query, address, connectionFactory, scheme); + this.fsNamesystemMetrics = FederationUtil.getJmx(query, address, connectionFactory, scheme); + } + + /** + * Populates FSNamesystem* metrics into report. + * @param aux FSNamesystem* metrics from namenode. + * @param report Namenode status report to update with JMX data. + * @throws JSONException When invalid JSONObject is found. + */ + private void populateFsNamesystemMetrics(JSONArray aux, NamenodeStatusReport report) + throws JSONException { if (aux != null) { for (int i = 0; i < aux.length(); i++) { JSONObject jsonObject = aux.getJSONObject(i); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java index f47d6ceb269..3230af8e07d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java @@ -115,6 +115,9 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic { FEDERATION_ROUTER_PREFIX + "heartbeat-state.interval"; public static final long DFS_ROUTER_HEARTBEAT_STATE_INTERVAL_MS_DEFAULT = TimeUnit.SECONDS.toMillis(5); + public static final String DFS_ROUTER_NAMENODE_HEARTBEAT_JMX_INTERVAL_MS = + FEDERATION_ROUTER_PREFIX + "namenode.heartbeat.jmx.interval"; + public static final long DFS_ROUTER_NAMENODE_HEARTBEAT_JMX_INTERVAL_MS_DEFAULT = 0; // HDFS Router NN client public static final String diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml index c7b403ce634..8322a72abaa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml @@ -471,6 +471,16 @@ + + dfs.federation.router.namenode.heartbeat.jmx.interval + 0 + + How often the Router should request JMX reports from the Namenode in miliseconds. + If this value is 0, it will request JMX reports every time a Namenode report is requested. + If this value is negative, it will disable JMX reports from the Namenode. + + + dfs.federation.router.store.router.expiration 5m diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java index bae2dea3cea..9ee9692aad1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeMonitoring.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -293,7 +294,32 @@ public class TestRouterNamenodeMonitoring { verifyUrlSchemes(HttpConfig.Policy.HTTPS_ONLY.name()); } + @Test + public void testJmxRequestFrequency() { + // Disable JMX requests + Configuration conf = getNamenodesConfig(); + conf.setLong(RBFConfigKeys.DFS_ROUTER_NAMENODE_HEARTBEAT_JMX_INTERVAL_MS, -1); + verifyUrlSchemes(HttpConfig.Policy.HTTPS_ONLY.name(), conf, 0, 0, 1); + + // Set JMX requests to lower frequency + conf = getNamenodesConfig(); + conf.setLong(RBFConfigKeys.DFS_ROUTER_NAMENODE_HEARTBEAT_JMX_INTERVAL_MS, + TimeUnit.MINUTES.toMillis(5)); + verifyUrlSchemes(HttpConfig.Policy.HTTPS_ONLY.name(), conf, 0, 1, 2); + + // Set JMX requests to default frequency + conf = getNamenodesConfig(); + verifyUrlSchemes(HttpConfig.Policy.HTTPS_ONLY.name(), conf, 0, 2, 2); + } + private void verifyUrlSchemes(String scheme) { + int httpRequests = HttpConfig.Policy.HTTP_ONLY.name().equals(scheme) ? 1 : 0; + int httpsRequests = HttpConfig.Policy.HTTPS_ONLY.name().equals(scheme) ? 1 : 0; + verifyUrlSchemes(scheme, getNamenodesConfig(), httpRequests, httpsRequests, 1); + } + + private void verifyUrlSchemes(String scheme, Configuration conf, int httpRequests, + int httpsRequests, int requestsPerService) { // Attach our own log appender so we can verify output final LogVerificationAppender appender = @@ -304,7 +330,6 @@ public class TestRouterNamenodeMonitoring { GenericTestUtils.setRootLogLevel(Level.DEBUG); // Setup and start the Router - Configuration conf = getNamenodesConfig(); conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, scheme); Configuration routerConf = new RouterConfigBuilder(conf) .heartbeat(true) @@ -318,15 +343,12 @@ public class TestRouterNamenodeMonitoring { Collection heartbeatServices = router.getNamenodeHeartbeatServices(); for (NamenodeHeartbeatService heartbeatService : heartbeatServices) { - heartbeatService.getNamenodeStatusReport(); - } - if (HttpConfig.Policy.HTTPS_ONLY.name().equals(scheme)) { - assertEquals(2, appender.countLinesWithMessage("JMX URL: https://")); - assertEquals(0, appender.countLinesWithMessage("JMX URL: http://")); - } else { - assertEquals(2, appender.countLinesWithMessage("JMX URL: http://")); - assertEquals(0, appender.countLinesWithMessage("JMX URL: https://")); + for (int request = 0; request < requestsPerService; request++) { + heartbeatService.getNamenodeStatusReport(); + } } + assertEquals(httpsRequests * 2, appender.countLinesWithMessage("JMX URL: https://")); + assertEquals(httpRequests * 2, appender.countLinesWithMessage("JMX URL: http://")); } /**