From 8ff17c68e2fb71c818cd16e6d3445d0c2658798b Mon Sep 17 00:00:00 2001 From: Baiqiang Zhao Date: Thu, 8 Apr 2021 02:11:46 +0800 Subject: [PATCH] =?UTF-8?q?HBASE-25687=20Backport=20"HBASE-25681=20Add=20a?= =?UTF-8?q?=20switch=20for=20server/table=20query=E2=80=A6=20(#3074)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: stack --- .../regionserver/MetricsTableQueryMeter.java | 3 ++ .../hbase/test/MetricsAssertHelper.java | 10 ++++ .../MetricsTableQueryMeterImpl.java | 3 -- .../hbase/test/MetricsAssertHelperImpl.java | 11 +++- .../regionserver/MetricsRegionServer.java | 29 +++++++--- .../RegionServerTableMetrics.java | 26 +++++---- .../regionserver/TestMetricsRegionServer.java | 28 +++++++++- .../TestMetricsTableLatencies.java | 53 ++++++++++++++++++- 8 files changed, 139 insertions(+), 24 deletions(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java index d0085ffb4a8..c3b819228fe 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeter.java @@ -25,6 +25,9 @@ import org.apache.yetus.audience.InterfaceAudience; @InterfaceAudience.Private public interface MetricsTableQueryMeter { + String TABLE_READ_QUERY_PER_SECOND = "tableReadQueryPerSecond"; + String TABLE_WRITE_QUERY_PER_SECOND = "tableWriteQueryPerSecond"; + /** * Update table read QPS * @param tableName The table the metric is for diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelper.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelper.java index 52e0d091279..0649fc997ad 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelper.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelper.java @@ -149,6 +149,16 @@ public interface MetricsAssertHelper { */ boolean checkCounterExists(String name, BaseSource source); + /** + * Check if a gauge exists. + * + * @param name name of the gauge. + * @param source The BaseSource{@link BaseSource} that will provide the tags, + * gauges, and counters. + * @return boolean true if gauge metric exists. + */ + boolean checkGaugeExists(String name, BaseSource source); + /** * Get the value of a gauge as a double. * diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeterImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeterImpl.java index 438310ff1d4..6b1d323dc19 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeterImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableQueryMeterImpl.java @@ -33,9 +33,6 @@ public class MetricsTableQueryMeterImpl implements MetricsTableQueryMeter { private final Map metersByTable = new ConcurrentHashMap<>(); private final MetricRegistry metricRegistry; - private final static String TABLE_READ_QUERY_PER_SECOND = "tableReadQueryPerSecond"; - private final static String TABLE_WRITE_QUERY_PER_SECOND = "tableWriteQueryPerSecond"; - public MetricsTableQueryMeterImpl(MetricRegistry metricRegistry) { this.metricRegistry = metricRegistry; } diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java index afe31f16667..c48cb807696 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java @@ -211,9 +211,16 @@ public class MetricsAssertHelperImpl implements MetricsAssertHelper { public boolean checkCounterExists(String name, BaseSource source) { getMetrics(source); String cName = canonicalizeMetricName(name); - return (counters.get(cName) != null) ? true : false; + return counters.get(cName) != null; } - + + @Override + public boolean checkGaugeExists(String name, BaseSource source) { + getMetrics(source); + String cName = canonicalizeMetricName(name); + return gauges.get(cName) != null; + } + @Override public double getGaugeDouble(String name, BaseSource source) { getMetrics(source); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java index 400b5fe4e0d..7ba1746ec28 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java @@ -40,6 +40,12 @@ public class MetricsRegionServer { public static final String RS_ENABLE_TABLE_METRICS_KEY = "hbase.regionserver.enable.table.latencies"; public static final boolean RS_ENABLE_TABLE_METRICS_DEFAULT = true; + public static final String RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY = + "hbase.regionserver.enable.server.query.meter"; + public static final boolean RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY_DEFAULT = true; + public static final String RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY = + "hbase.regionserver.enable.table.query.meter"; + public static final boolean RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY_DEFAULT = true; public static final String SLOW_METRIC_TIME = "hbase.ipc.slow.metric.time"; private final MetricsRegionServerSource serverSource; @@ -73,9 +79,12 @@ public class MetricsRegionServer { bulkLoadTimer = metricRegistry.timer("Bulkload"); slowMetricTime = conf.getLong(SLOW_METRIC_TIME, DEFAULT_SLOW_METRIC_TIME); - serverReadQueryMeter = metricRegistry.meter("ServerReadQueryPerSecond"); - serverWriteQueryMeter = metricRegistry.meter("ServerWriteQueryPerSecond"); quotaSource = CompatibilitySingletonFactory.getInstance(MetricsRegionServerQuotaSource.class); + if (conf.getBoolean(RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY, + RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY_DEFAULT)) { + serverReadQueryMeter = metricRegistry.meter("ServerReadQueryPerSecond"); + serverWriteQueryMeter = metricRegistry.meter("ServerWriteQueryPerSecond"); + } } MetricsRegionServer(MetricsRegionServerWrapper regionServerWrapper, @@ -93,7 +102,9 @@ public class MetricsRegionServer { */ static RegionServerTableMetrics createTableMetrics(Configuration conf) { if (conf.getBoolean(RS_ENABLE_TABLE_METRICS_KEY, RS_ENABLE_TABLE_METRICS_DEFAULT)) { - return new RegionServerTableMetrics(); + return new RegionServerTableMetrics( + conf.getBoolean(RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY, + RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY_DEFAULT)); } return null; } @@ -270,21 +281,27 @@ public class MetricsRegionServer { if (tableMetrics != null && tn != null) { tableMetrics.updateTableReadQueryMeter(tn, count); } - this.serverReadQueryMeter.mark(count); + if (serverReadQueryMeter != null) { + serverReadQueryMeter.mark(count); + } } public void updateWriteQueryMeter(TableName tn, long count) { if (tableMetrics != null && tn != null) { tableMetrics.updateTableWriteQueryMeter(tn, count); } - this.serverWriteQueryMeter.mark(count); + if (serverWriteQueryMeter != null) { + serverWriteQueryMeter.mark(count); + } } public void updateWriteQueryMeter(TableName tn) { if (tableMetrics != null && tn != null) { tableMetrics.updateTableWriteQueryMeter(tn); } - this.serverWriteQueryMeter.mark(); + if (serverWriteQueryMeter != null) { + serverWriteQueryMeter.mark(); + } } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerTableMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerTableMetrics.java index 812ae45e884..a9a7d75bfea 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerTableMetrics.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerTableMetrics.java @@ -29,12 +29,14 @@ import org.apache.yetus.audience.InterfaceAudience; public class RegionServerTableMetrics { private final MetricsTableLatencies latencies; - private final MetricsTableQueryMeter queryMeter; + private MetricsTableQueryMeter queryMeter; - public RegionServerTableMetrics() { + public RegionServerTableMetrics(boolean enableTableQueryMeter) { latencies = CompatibilitySingletonFactory.getInstance(MetricsTableLatencies.class); - queryMeter = new MetricsTableQueryMeterImpl(MetricRegistries.global(). - get(((MetricsTableLatenciesImpl) latencies).getMetricRegistryInfo()).get()); + if (enableTableQueryMeter) { + queryMeter = new MetricsTableQueryMeterImpl(MetricRegistries.global(). + get(((MetricsTableLatenciesImpl) latencies).getMetricRegistryInfo()).get()); + } } public void updatePut(TableName table, long time) { @@ -86,18 +88,20 @@ public class RegionServerTableMetrics { } public void updateTableReadQueryMeter(TableName table, long count) { - queryMeter.updateTableReadQueryMeter(table, count); - } - - public void updateTableReadQueryMeter(TableName table) { - queryMeter.updateTableReadQueryMeter(table); + if (queryMeter != null) { + queryMeter.updateTableReadQueryMeter(table, count); + } } public void updateTableWriteQueryMeter(TableName table, long count) { - queryMeter.updateTableWriteQueryMeter(table, count); + if (queryMeter != null) { + queryMeter.updateTableWriteQueryMeter(table, count); + } } public void updateTableWriteQueryMeter(TableName table) { - queryMeter.updateTableWriteQueryMeter(table); + if (queryMeter != null) { + queryMeter.updateTableWriteQueryMeter(table); + } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java index e8d7bfe4d75..3fc13a181c5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java @@ -17,11 +17,14 @@ */ package org.apache.hadoop.hbase.regionserver; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -57,7 +60,9 @@ public class TestMetricsRegionServer { @Before public void setUp() { wrapper = new MetricsRegionServerWrapperStub(); - rsm = new MetricsRegionServer(wrapper, new Configuration(false), null); + Configuration conf = new Configuration(false); + conf.setBoolean(MetricsRegionServer.RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY, false); + rsm = new MetricsRegionServer(wrapper, conf, null); serverSource = rsm.getMetricsSource(); } @@ -245,5 +250,26 @@ public class TestMetricsRegionServer { HELPER.assertCounter("pauseTimeWithGc_num_ops", 1, serverSource); } + @Test + public void testServerQueryMeterSwitch() { + TableName tn1 = TableName.valueOf("table1"); + // has been set disable in setUp() + rsm.updateReadQueryMeter(tn1, 500L); + assertFalse(HELPER.checkGaugeExists("ServerReadQueryPerSecond_count", serverSource)); + rsm.updateWriteQueryMeter(tn1, 500L); + assertFalse(HELPER.checkGaugeExists("ServerWriteQueryPerSecond_count", serverSource)); + + // enable + Configuration conf = new Configuration(false); + conf.setBoolean(MetricsRegionServer.RS_ENABLE_SERVER_QUERY_METER_METRICS_KEY, true); + rsm = new MetricsRegionServer(wrapper, conf, null); + serverSource = rsm.getMetricsSource(); + rsm.updateReadQueryMeter(tn1, 500L); + assertTrue(HELPER.checkGaugeExists("ServerWriteQueryPerSecond_count", serverSource)); + HELPER.assertGauge("ServerReadQueryPerSecond_count", 500L, serverSource); + assertTrue(HELPER.checkGaugeExists("ServerWriteQueryPerSecond_count", serverSource)); + rsm.updateWriteQueryMeter(tn1, 500L); + HELPER.assertGauge("ServerWriteQueryPerSecond_count", 500L, serverSource); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableLatencies.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableLatencies.java index f16086e4911..46592b33df3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableLatencies.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableLatencies.java @@ -17,10 +17,12 @@ */ package org.apache.hadoop.hbase.regionserver; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.HBaseClassTestRule; @@ -51,7 +53,7 @@ public class TestMetricsTableLatencies { assertTrue("'latencies' is actually " + latencies.getClass(), latencies instanceof MetricsTableLatenciesImpl); MetricsTableLatenciesImpl latenciesImpl = (MetricsTableLatenciesImpl) latencies; - RegionServerTableMetrics tableMetrics = new RegionServerTableMetrics(); + RegionServerTableMetrics tableMetrics = new RegionServerTableMetrics(false); // Metrics to each table should be disjoint // N.B. each call to assertGauge removes all previously acquired metrics so we have to @@ -72,4 +74,53 @@ public class TestMetricsTableLatencies { HELPER.assertGauge(MetricsTableLatenciesImpl.qualifyMetricsName( tn2, MetricsTableLatencies.PUT_TIME + "_" + "99th_percentile"), 75L, latenciesImpl); } + + @Test + public void testTableQueryMeterSwitch() { + TableName tn1 = TableName.valueOf("table1"); + MetricsTableLatencies latencies = CompatibilitySingletonFactory.getInstance( + MetricsTableLatencies.class); + assertTrue("'latencies' is actually " + latencies.getClass(), + latencies instanceof MetricsTableLatenciesImpl); + MetricsTableLatenciesImpl latenciesImpl = (MetricsTableLatenciesImpl) latencies; + + Configuration conf = new Configuration(); + conf.setBoolean(MetricsRegionServer.RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY, false); + boolean enableTableQueryMeter = conf.getBoolean( + MetricsRegionServer.RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY, + MetricsRegionServer.RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY_DEFAULT); + // disable + assertFalse(enableTableQueryMeter); + RegionServerTableMetrics tableMetrics = new RegionServerTableMetrics(enableTableQueryMeter); + tableMetrics.updateTableReadQueryMeter(tn1, 500L); + assertFalse(HELPER.checkGaugeExists(MetricsTableLatenciesImpl.qualifyMetricsName( + tn1, MetricsTableQueryMeterImpl.TABLE_READ_QUERY_PER_SECOND + "_" + "count"), + latenciesImpl)); + tableMetrics.updateTableWriteQueryMeter(tn1, 500L); + assertFalse(HELPER.checkGaugeExists(MetricsTableLatenciesImpl.qualifyMetricsName( + tn1, MetricsTableQueryMeterImpl.TABLE_WRITE_QUERY_PER_SECOND + "_" + "count"), + latenciesImpl)); + + // enable + conf.setBoolean(MetricsRegionServer.RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY, true); + enableTableQueryMeter = conf.getBoolean( + MetricsRegionServer.RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY, + MetricsRegionServer.RS_ENABLE_TABLE_QUERY_METER_METRICS_KEY_DEFAULT); + assertTrue(enableTableQueryMeter); + tableMetrics = new RegionServerTableMetrics(true); + tableMetrics.updateTableReadQueryMeter(tn1, 500L); + assertTrue(HELPER.checkGaugeExists(MetricsTableLatenciesImpl.qualifyMetricsName( + tn1, MetricsTableQueryMeterImpl.TABLE_READ_QUERY_PER_SECOND + "_" + "count"), + latenciesImpl)); + HELPER.assertGauge(MetricsTableLatenciesImpl.qualifyMetricsName( + tn1, MetricsTableQueryMeterImpl.TABLE_READ_QUERY_PER_SECOND + "_" + "count"), + 500L, latenciesImpl); + tableMetrics.updateTableWriteQueryMeter(tn1, 500L); + assertTrue(HELPER.checkGaugeExists(MetricsTableLatenciesImpl.qualifyMetricsName( + tn1, MetricsTableQueryMeterImpl.TABLE_WRITE_QUERY_PER_SECOND + "_" + "count"), + latenciesImpl)); + HELPER.assertGauge(MetricsTableLatenciesImpl.qualifyMetricsName( + tn1, MetricsTableQueryMeterImpl.TABLE_WRITE_QUERY_PER_SECOND + "_" + "count"), + 500L, latenciesImpl); + } }