From f9e18cf31b4feabb86b942f205eb4694f7824ccd Mon Sep 17 00:00:00 2001 From: Kiran Kumar Maturi Date: Tue, 18 Sep 2018 09:45:26 +0530 Subject: [PATCH] HBASE-20857 balancer status tag in jmx metrics Signed-off-by: Andrew Purtell --- .../balancer/MetricsBalancerSource.java | 3 + .../balancer/MetricsBalancerSourceImpl.java | 6 + .../rsgroup/RSGroupBasedLoadBalancer.java | 4 + .../hadoop/hbase/master/LoadBalancer.java | 3 + .../hbase/master/MasterRpcServices.java | 1 + .../master/balancer/BaseLoadBalancer.java | 7 ++ .../master/balancer/MetricsBalancer.java | 4 + .../balancer/MetricsStochasticBalancer.java | 8 ++ .../TestBalancerStatusTagInJMXMetrics.java | 115 ++++++++++++++++++ 9 files changed, 151 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerStatusTagInJMXMetrics.java diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSource.java index 5add25f4c04..6b8c40ba512 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSource.java @@ -41,6 +41,7 @@ public interface MetricsBalancerSource extends BaseSource { String BALANCE_CLUSTER = "balancerCluster"; String MISC_INVOATION_COUNT = "miscInvocationCount"; + String BALANCER_STATUS = "isBalancerActive"; /** * Description @@ -50,4 +51,6 @@ public interface MetricsBalancerSource extends BaseSource { void updateBalanceCluster(long time); void incrMiscInvocations(); + + void updateBalancerStatus(boolean status); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java index a10b3d71ebd..7bccbb70d58 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java @@ -36,6 +36,7 @@ public class MetricsBalancerSourceImpl extends BaseSourceImpl implements Metrics String metricsDescription, String metricsContext, String metricsJmxContext) { super(metricsName, metricsDescription, metricsContext, metricsJmxContext); + updateBalancerStatus(true); } @Override @@ -53,4 +54,9 @@ public class MetricsBalancerSourceImpl extends BaseSourceImpl implements Metrics public void incrMiscInvocations() { miscCount.incr(); } + + @Override + public void updateBalancerStatus(boolean status) { + metricsRegistry.tag(BALANCER_STATUS,"", String.valueOf(status), true); + } } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index 0e1dacdf194..ecd034e76f1 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -480,4 +480,8 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { public void postMasterStartupInitialize() { this.internalBalancer.postMasterStartupInitialize(); } + + public void updateBalancerStatus(boolean status) { + internalBalancer.updateBalancerStatus(status); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java index aa88f499ba2..9a894e1f096 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java @@ -164,6 +164,9 @@ public interface LoadBalancer extends Configurable, Stoppable, ConfigurationObse */ void postMasterStartupInitialize(); + /*Updates balancer status tag reported to JMX*/ + void updateBalancerStatus(boolean status); + /** * @return true if Master carries regions */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 62c80a6002e..4702ad9bb0a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -419,6 +419,7 @@ public class MasterRpcServices extends RSRpcServices if (master.cpHost != null) { master.cpHost.postBalanceSwitch(oldValue, newValue); } + master.getLoadBalancer().updateBalancerStatus(newValue); } catch (IOException ioe) { LOG.warn("Error flipping balance switch", ioe); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index 6cca59fa783..52a284e3c08 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -1554,6 +1554,13 @@ public abstract class BaseLoadBalancer implements LoadBalancer { stopped = true; } + /** + * Updates the balancer status tag reported to JMX + */ + public void updateBalancerStatus(boolean status) { + metricsBalancer.balancerStatus(status); + } + /** * Used to assign a single region to a random server. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancer.java index fdaf3cf0582..9bde5e20902 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancer.java @@ -48,4 +48,8 @@ public class MetricsBalancer { public void incrMiscInvocations() { source.incrMiscInvocations(); } + + public void balancerStatus(boolean status) { + source.updateBalancerStatus(status); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancer.java index ee24ff3fdb3..6c79f054ee4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancer.java @@ -56,6 +56,14 @@ public class MetricsStochasticBalancer extends MetricsBalancer { stochasticSource.incrMiscInvocations(); } + /** + * Updates the balancer status tag reported to JMX + */ + @Override + public void balancerStatus(boolean status) { + stochasticSource.updateBalancerStatus(status); + } + /** * Updates the number of metrics reported to JMX */ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerStatusTagInJMXMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerStatusTagInJMXMetrics.java new file mode 100644 index 00000000000..9f566210a87 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerStatusTagInJMXMetrics.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License + * for the specific language governing permissions and limitations under the License. + */ + +package org.apache.hadoop.hbase.master.balancer; + +import static org.junit.Assert.assertEquals; + +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ MediumTests.class }) +public class TestBalancerStatusTagInJMXMetrics extends BalancerTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestBalancerStatusTagInJMXMetrics.class); + + private static final Log LOG = LogFactory.getLog(TestBalancerStatusTagInJMXMetrics.class); + private static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static int connectorPort = 61120; + private static HMaster master; + private static MiniHBaseCluster cluster; + private static Configuration conf = null; + + /** + * Setup the environment for the test. + */ + @BeforeClass + public static void setupBeforeClass() throws Exception { + + conf = UTIL.getConfiguration(); + Random rand = new Random(); + for (int i = 0; i < 10; i++) { + do { + int sign = i % 2 == 0 ? 1 : -1; + connectorPort += sign * rand.nextInt(100); + } while (!HBaseTestingUtility.available(connectorPort)); + try { + conf.setInt("regionserver.rmi.registry.port", connectorPort); + cluster = UTIL.startMiniCluster(); + LOG.info("Waiting for active/ready master"); + cluster.waitForActiveAndReadyMaster(); + master = cluster.getMaster(); + break; + } catch (Exception e) { + LOG.debug("Encountered exception when starting mini cluster. Trying port " + connectorPort, + e); + try { + // this is to avoid "IllegalStateException: A mini-cluster is already running" + UTIL.shutdownMiniCluster(); + } catch (Exception ex) { + LOG.debug("Encountered exception shutting down cluster", ex); + } + } + } + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + /** + * Tests the status change using the Default Metrics System + */ + @Test + public void testJmxMetrics() throws Exception { + + assertEquals(getStatus(), "true"); + master.getLoadBalancer().updateBalancerStatus(false); + assertEquals(getStatus(), "false"); + + } + + /** + * Gets the balancer status tag from the Metrics registry + */ + public String getStatus() throws Exception { + MetricsSource source = + DefaultMetricsSystem.instance().getSource(MetricsBalancerSource.METRICS_JMX_CONTEXT); + if (source instanceof MetricsBalancerSourceImpl) { + MetricsTag status = ((MetricsBalancerSourceImpl) source).getMetricsRegistry() + .getTag(MetricsBalancerSource.BALANCER_STATUS); + return status.value(); + } else { + LOG.warn("Balancer JMX Metrics not registered"); + throw new Exception("MetricsBalancer JMX Context not found"); + } + } + +}