From 9a0eb6f0c2a407aad3e2e966e0b44a584fb68b9a Mon Sep 17 00:00:00 2001
From: Nicolas Spiegelberg <nspiegelberg@apache.org>
Date: Tue, 20 Dec 2011 18:42:11 +0000
Subject: [PATCH] [jira] [HBASE-5072] Support Max Value for Per-Store Metrics

Summary:
We were bit in our multi-tenant cluster because one of our Stores
encountered a bug and grew its StoreFile count. We didn't notice this because
the StoreFile count currently reported by the RegionServer is an average of all
Stores in the region. For the per-Store metrics, we should also record the max
so we can notice outliers.

Test Plan: - mvn test -Dtest=TestRegionServerMetrics

Reviewers: JIRA, mbautin, Kannan

Reviewed By: Kannan

CC: stack, nspiegelberg, mbautin, Kannan

Differential Revision: 945

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1221419 13f79535-47bb-0310-9956-ffa450edef68
---
 .../regionserver/metrics/SchemaMetrics.java   | 27 ++++++++++++++-----
 .../regionserver/TestRegionServerMetrics.java | 10 +++++--
 2 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
index 3e587c30c7f..08c5cfb2d3e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java
@@ -245,6 +245,7 @@ public class SchemaMetrics {
 
   private final String[] bloomMetricNames = new String[2];
   private final String[] storeMetricNames = new String[NUM_STORE_METRIC_TYPES];
+  private final String[] storeMetricNamesMax = new String[NUM_STORE_METRIC_TYPES];
 
   private SchemaMetrics(final String tableName, final String cfName) {
     String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix(
@@ -290,8 +291,9 @@ public class SchemaMetrics {
     }
 
     for (StoreMetricType storeMetric : StoreMetricType.values()) {
-      storeMetricNames[storeMetric.ordinal()] = metricPrefix +
-        storeMetric.toString();
+      String coreName = metricPrefix + storeMetric.toString();
+      storeMetricNames[storeMetric.ordinal()] = coreName;
+      storeMetricNamesMax[storeMetric.ordinal()] = coreName + ".max";
     }
   }
 
@@ -390,13 +392,22 @@ public class SchemaMetrics {
   public void accumulateStoreMetric(final Map<String, MutableDouble> tmpMap,
       StoreMetricType storeMetricType, double val) {
     final String key = getStoreMetricName(storeMetricType);
-    if (tmpMap.get(key) != null) {
-      tmpMap.get(key).add(val);
-    } else {
+    if (tmpMap.get(key) == null) {
       tmpMap.put(key, new MutableDouble(val));
+    } else {
+      tmpMap.get(key).add(val);
     }
 
-    if (this != ALL_SCHEMA_METRICS) {
+    if (this == ALL_SCHEMA_METRICS) {
+      // also compute the max value across all Stores on this server
+      final String maxKey = getStoreMetricNameMax(storeMetricType);
+      MutableDouble cur = tmpMap.get(maxKey);
+      if (cur == null) {
+        tmpMap.put(maxKey, new MutableDouble(val));
+      } else if (cur.doubleValue() < val) {
+        cur.setValue(val);
+      }
+    } else {
       ALL_SCHEMA_METRICS.accumulateStoreMetric(tmpMap, storeMetricType, val);
     }
   }
@@ -405,6 +416,10 @@ public class SchemaMetrics {
     return storeMetricNames[storeMetricType.ordinal()];
   }
 
+  public String getStoreMetricNameMax(StoreMetricType storeMetricType) {
+    return storeMetricNamesMax[storeMetricType.ordinal()];
+  }
+
   /**
    * Update a metric that does not get reset on every poll.
    * @param storeMetricType the store metric to update
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
index 3447aeb6336..6560672abb0 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
@@ -116,9 +116,15 @@ public class TestRegionServerMetrics {
 
     for (String cf : FAMILIES) {
       SchemaMetrics schemaMetrics = SchemaMetrics.getInstance(TABLE_NAME, cf);
-      assertStoreMetricEquals(NUM_FLUSHES * NUM_REGIONS,
-          schemaMetrics, StoreMetricType.STORE_FILE_COUNT);
+      assertStoreMetricEquals(NUM_FLUSHES * NUM_REGIONS, schemaMetrics,
+          StoreMetricType.STORE_FILE_COUNT);
     }
+
+    // ensure that the max value is also maintained
+    final String storeMetricName = ALL_METRICS
+        .getStoreMetricNameMax(StoreMetricType.STORE_FILE_COUNT);
+    assertEquals("Invalid value for store metric " + storeMetricName,
+        NUM_FLUSHES, HRegion.getNumericMetric(storeMetricName));
   }