diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java index 623cbdb6be0..78b2bf0e180 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.util; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicLongArray; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; @@ -31,11 +30,20 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; @InterfaceAudience.Public @InterfaceStability.Evolving public class FastLongHistogram { + + /** + * Default number of bins. + */ + public static final int DEFAULT_NBINS = 255; + + public static final double[] DEFAULT_QUANTILES = + new double[]{0.25, 0.5, 0.75, 0.90, 0.95, 0.98, 0.99, 0.999}; + /** * Bins is a class containing a list of buckets(or bins) for estimation histogram of some data. */ private static class Bins { - private final AtomicLongArray counts; + private final Counter[] counts; // inclusive private final long binsMin; // exclusive @@ -43,6 +51,10 @@ public class FastLongHistogram { private final long bins10XMax; private final AtomicLong min = new AtomicLong(Long.MAX_VALUE); private final AtomicLong max = new AtomicLong(0L); + + private final Counter count = new Counter(0); + private final Counter total = new Counter(0); + // set to true when any of data has been inserted to the Bins. It is set after the counts are // updated. private final AtomicBoolean hasData = new AtomicBoolean(false); @@ -50,17 +62,18 @@ public class FastLongHistogram { /** * The constructor for creating a Bins without any prior data. */ - public Bins() { - this.counts = new AtomicLongArray(4); - this.binsMin = 0L; - this.binsMax = Long.MAX_VALUE; - this.bins10XMax = Long.MAX_VALUE; + public Bins(int numBins) { + counts = createCounters(numBins + 3); + this.binsMin = 1L; + + // These two numbers are total guesses + // and should be treated as highly suspect. + this.binsMax = 1000; + this.bins10XMax = binsMax * 10; } - + /** * The constructor for creating a Bins with last Bins. - * @param last the last Bins instance. - * @param quantiles the quantiles for creating the bins of the histogram. */ public Bins(Bins last, int numOfBins, double minQ, double maxQ) { long[] values = last.getQuantiles(new double[] { minQ, maxQ }); @@ -72,28 +85,51 @@ public class FastLongHistogram { this.binsMax = Math.max(binsMax, this.binsMin + numOfBins); this.bins10XMax = Math.max((long) (values[1] + (binsMax - 1) * 9), this.binsMax + 1); - this.counts = new AtomicLongArray(numOfBins + 3); + this.counts = createCounters(numOfBins + 3); + } + + private Counter[] createCounters(int num) { + Counter[] counters = new Counter[num]; + for (int i = 0; i < num; i++) { + counters[i] = new Counter(); + } + return counters; + } + + private int getIndex(long value) { + if (value < this.binsMin) { + return 0; + } else if (value > this.bins10XMax) { + return this.counts.length - 1; + } else if (value >= this.binsMax) { + return this.counts.length - 2; + } + // compute the position + return 1 + (int) ((value - this.binsMin) * (this.counts.length - 3) / + (this.binsMax - this.binsMin)); + } /** * Adds a value to the histogram. */ public void add(long value, long count) { + if (value < 0) { + // The whole computation is completely thrown off if there are negative numbers + // + // Normally we would throw an IllegalArgumentException however this is the metrics + // system and it should be completely safe at all times. + // So silently throw it away. + return; + } AtomicUtils.updateMin(min, value); AtomicUtils.updateMax(max, value); - if (value < this.binsMin) { - this.counts.addAndGet(0, count); - } else if (value > this.bins10XMax) { - this.counts.addAndGet(this.counts.length() - 1, count); - } else if (value >= this.binsMax) { - this.counts.addAndGet(this.counts.length() - 2, count); - } else { - // compute the position - int pos = - 1 + (int) ((value - this.binsMin) * (this.counts.length() - 3) / (this.binsMax - this.binsMin)); - this.counts.addAndGet(pos, count); - } + this.count.add(count); + this.total.add(value * count); + + int pos = getIndex(value); + this.counts[pos].add(count); // hasData needs to be updated as last this.hasData.set(true); @@ -101,7 +137,6 @@ public class FastLongHistogram { /** * Computes the quantiles give the ratios. - * @param smooth set to true to have a prior on the distribution. Used for recreating the bins. */ public long[] getQuantiles(double[] quantiles) { if (!this.hasData.get()) { @@ -112,10 +147,10 @@ public class FastLongHistogram { // Make a snapshot of lowerCounter, higherCounter and bins.counts to counts. // This is not synchronized, but since the counter are accumulating, the result is a good // estimation of a snapshot. - long[] counts = new long[this.counts.length()]; + long[] counts = new long[this.counts.length]; long total = 0L; - for (int i = 0; i < this.counts.length(); i++) { - counts[i] = this.counts.get(i); + for (int i = 0; i < this.counts.length; i++) { + counts[i] = this.counts[i].get(); total += counts[i]; } @@ -137,8 +172,8 @@ public class FastLongHistogram { mn = this.binsMax; mx = this.bins10XMax; } else { - mn = this.binsMin + (i - 1) * (this.binsMax - this.binsMin) / (this.counts.length() - 3); - mx = this.binsMin + i * (this.binsMax - this.binsMin) / (this.counts.length() - 3); + mn = this.binsMin + (i - 1) * (this.binsMax - this.binsMin) / (this.counts.length - 3); + mx = this.binsMin + i * (this.binsMax - this.binsMin) / (this.counts.length - 3); } if (mx < this.min.get()) { @@ -177,12 +212,27 @@ public class FastLongHistogram { return res; } + + + long getNumAtOrBelow(long val) { + final int targetIndex = getIndex(val); + long totalToCurrentIndex = 0; + for (int i = 0; i <= targetIndex; i++) { + totalToCurrentIndex += this.counts[i].get(); + } + return totalToCurrentIndex; + } } // The bins counting values. It is replaced with a new one in calling of reset(). - private volatile Bins bins = new Bins(); - // The quantiles for creating a Bins with last Bins. - private final int numOfBins; + private volatile Bins bins; + + /** + * Constructor. + */ + public FastLongHistogram() { + this(DEFAULT_NBINS); + } /** * Constructor. @@ -190,7 +240,7 @@ public class FastLongHistogram { * results but with lower efficiency, and vice versus. */ public FastLongHistogram(int numOfBins) { - this.numOfBins = numOfBins; + this.bins = new Bins(numOfBins); } /** @@ -202,10 +252,14 @@ public class FastLongHistogram { */ public FastLongHistogram(int numOfBins, long min, long max) { this(numOfBins); - Bins bins = new Bins(); + Bins bins = new Bins(numOfBins); bins.add(min, 1); bins.add(max, 1); - this.bins = new Bins(bins, numOfBins, 0.01, 0.99); + this.bins = new Bins(bins, numOfBins, 0.01, 0.999); + } + + private FastLongHistogram(Bins bins) { + this.bins = bins; } /** @@ -222,12 +276,46 @@ public class FastLongHistogram { return this.bins.getQuantiles(quantiles); } + public long[] getQuantiles() { + return this.bins.getQuantiles(DEFAULT_QUANTILES); + } + + public long getMin() { + return this.bins.min.get(); + } + + public long getMax() { + return this.bins.max.get(); + } + + public long getCount() { + return this.bins.count.get(); + } + + public long getMean() { + Bins bins = this.bins; + long count = bins.count.get(); + long total = bins.total.get(); + if (count == 0) { + return 0; + } + return total / count; + } + + public long getNumAtOrBelow(long value) { + return this.bins.getNumAtOrBelow(value); + } + /** * Resets the histogram for new counting. */ - public void reset() { + public FastLongHistogram reset() { if (this.bins.hasData.get()) { - this.bins = new Bins(this.bins, numOfBins, 0.01, 0.99); + Bins oldBins = this.bins; + this.bins = new Bins(this.bins, this.bins.counts.length - 3, 0.01, 0.99); + return new FastLongHistogram(oldBins); } + + return null; } } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java index e0437b5f3ba..7304e2db23e 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java @@ -20,15 +20,18 @@ package org.apache.hadoop.hbase.util; import java.util.Arrays; import java.util.Random; -import org.apache.hadoop.hbase.SmallTests; + +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; +import static org.junit.Assert.assertEquals; + /** * Testcases for FastLongHistogram. */ -@Category({ SmallTests.class }) +@Category(SmallTests.class) public class TestFastLongHistogram { private static void doTestUniform(FastLongHistogram hist) { @@ -87,6 +90,36 @@ public class TestFastLongHistogram { } } + + @Test + public void testGetNumAtOrBelow() { + long[] VALUES = { 1, 10, 20, 30, 40, 50 }; + + FastLongHistogram h = new FastLongHistogram(); + for (long v : VALUES) { + for (int i = 0; i < 100; i++) { + h.add(v, 1); + } + } + + h.add(Integer.MAX_VALUE, 1); + + h.reset(); + + for (long v : VALUES) { + for (int i = 0; i < 100; i++) { + h.add(v, 1); + } + } + // Add something way out there to make sure it doesn't throw off the counts. + h.add(Integer.MAX_VALUE, 1); + + assertEquals(100, h.getNumAtOrBelow(1)); + assertEquals(200, h.getNumAtOrBelow(11)); + assertEquals(601, h.getNumAtOrBelow(Long.MAX_VALUE)); + } + + @Test public void testSameValues() { FastLongHistogram hist = new FastLongHistogram(100); diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java index 3ab783af394..f79aa9f91e7 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java @@ -79,14 +79,6 @@ public interface BaseSource { void updateHistogram(String name, long value); - /** - * Add some value to a Quantile (An accurate histogram). - * - * @param name the name of the quantile - * @param value the value to add to the quantile - */ - void updateQuantile(String name, long value); - /** * Get the metrics context. For hadoop metrics2 system this is usually an all lowercased string. * eg. regionserver, master, thriftserver diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java index b759efb2e69..9d24a23f19d 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java @@ -30,10 +30,13 @@ public interface MetricHistogram { String MAX_METRIC_NAME = "_max"; String MEAN_METRIC_NAME = "_mean"; String MEDIAN_METRIC_NAME = "_median"; + String TWENTY_FIFTH_PERCENTILE_METRIC_NAME = "_25th_percentile"; String SEVENTY_FIFTH_PERCENTILE_METRIC_NAME = "_75th_percentile"; String NINETIETH_PERCENTILE_METRIC_NAME = "_90th_percentile"; String NINETY_FIFTH_PERCENTILE_METRIC_NAME = "_95th_percentile"; + String NINETY_EIGHTH_PERCENTILE_METRIC_NAME = "_98th_percentile"; String NINETY_NINETH_PERCENTILE_METRIC_NAME = "_99th_percentile"; + String NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME = "_99.9th_percentile"; /** * Add a single value to a histogram's stream of values. diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml index df77668b34d..386e570e481 100644 --- a/hbase-hadoop2-compat/pom.xml +++ b/hbase-hadoop2-compat/pom.xml @@ -160,6 +160,10 @@ limitations under the License. org.apache.hbase hbase-hadoop-compat + + org.apache.hbase + hbase-common + org.apache.hbase hbase-hadoop-compat @@ -181,10 +185,6 @@ limitations under the License. hadoop-common ${hadoop-two.version} - - com.yammer.metrics - metrics-core - commons-lang commons-lang diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java index 487f9f59157..48f57e976e4 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/ipc/MetricsHBaseServerSourceImpl.java @@ -21,11 +21,11 @@ package org.apache.hadoop.hbase.ipc; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.Interns; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.metrics2.lib.MutableHistogram; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; @InterfaceAudience.Private public class MetricsHBaseServerSourceImpl extends BaseSourceImpl @@ -33,29 +33,29 @@ public class MetricsHBaseServerSourceImpl extends BaseSourceImpl private final MetricsHBaseServerWrapper wrapper; - private final MutableCounterLong authorizationSuccesses; - private final MutableCounterLong authorizationFailures; - private final MutableCounterLong authenticationSuccesses; - private final MutableCounterLong authenticationFailures; - private final MutableCounterLong authenticationFallbacks; - private final MutableCounterLong sentBytes; - private final MutableCounterLong receivedBytes; + private final MutableFastCounter authorizationSuccesses; + private final MutableFastCounter authorizationFailures; + private final MutableFastCounter authenticationSuccesses; + private final MutableFastCounter authenticationFailures; + private final MutableFastCounter authenticationFallbacks; + private final MutableFastCounter sentBytes; + private final MutableFastCounter receivedBytes; - private final MutableCounterLong exceptions; - private final MutableCounterLong exceptionsOOO; - private final MutableCounterLong exceptionsBusy; - private final MutableCounterLong exceptionsUnknown; - private final MutableCounterLong exceptionsSanity; - private final MutableCounterLong exceptionsNSRE; - private final MutableCounterLong exceptionsMoved; - private final MutableCounterLong exceptionsMultiTooLarge; + private final MutableFastCounter exceptions; + private final MutableFastCounter exceptionsOOO; + private final MutableFastCounter exceptionsBusy; + private final MutableFastCounter exceptionsUnknown; + private final MutableFastCounter exceptionsSanity; + private final MutableFastCounter exceptionsNSRE; + private final MutableFastCounter exceptionsMoved; + private final MutableFastCounter exceptionsMultiTooLarge; - private MutableHistogram queueCallTime; - private MutableHistogram processCallTime; - private MutableHistogram totalCallTime; - private MutableHistogram requestSize; - private MutableHistogram responseSize; + private MetricHistogram queueCallTime; + private MetricHistogram processCallTime; + private MetricHistogram totalCallTime; + private MetricHistogram requestSize; + private MetricHistogram responseSize; public MetricsHBaseServerSourceImpl(String metricsName, String metricsDescription, diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java index ccf1c1d85ad..a2192d2a9f8 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java @@ -20,17 +20,19 @@ package org.apache.hadoop.hbase.master; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; -import org.apache.hadoop.metrics2.lib.MutableHistogram; @InterfaceAudience.Private -public class MetricsAssignmentManagerSourceImpl extends BaseSourceImpl implements MetricsAssignmentManagerSource { +public class MetricsAssignmentManagerSourceImpl + extends BaseSourceImpl + implements MetricsAssignmentManagerSource { private MutableGaugeLong ritGauge; private MutableGaugeLong ritCountOverThresholdGauge; private MutableGaugeLong ritOldestAgeGauge; - private MutableHistogram assignTimeHisto; - private MutableHistogram bulkAssignTimeHisto; + private MetricHistogram assignTimeHisto; + private MetricHistogram bulkAssignTimeHisto; public MetricsAssignmentManagerSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java index 28414ea71c3..7e8542c4bac 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterFilesystemSourceImpl.java @@ -20,15 +20,17 @@ package org.apache.hadoop.hbase.master; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.lib.MutableHistogram; +import org.apache.hadoop.metrics2.MetricHistogram; @InterfaceAudience.Private -public class MetricsMasterFilesystemSourceImpl extends BaseSourceImpl implements MetricsMasterFileSystemSource { +public class MetricsMasterFilesystemSourceImpl + extends BaseSourceImpl + implements MetricsMasterFileSystemSource { - private MutableHistogram splitSizeHisto; - private MutableHistogram splitTimeHisto; - private MutableHistogram metaSplitTimeHisto; - private MutableHistogram metaSplitSizeHisto; + private MetricHistogram splitSizeHisto; + private MetricHistogram splitTimeHisto; + private MetricHistogram metaSplitTimeHisto; + private MetricHistogram metaSplitSizeHisto; public MetricsMasterFilesystemSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java index b0ba66ef73d..ea21967cafb 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterSourceImpl.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hbase.metrics.BaseSourceImpl; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.Interns; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; /** * Hadoop2 implementation of MetricsMasterSource. @@ -35,7 +35,7 @@ public class MetricsMasterSourceImpl extends BaseSourceImpl implements MetricsMasterSource { private final MetricsMasterWrapper masterWrapper; - private MutableCounterLong clusterRequestsCounter; + private MutableFastCounter clusterRequestsCounter; public MetricsMasterSourceImpl(MetricsMasterWrapper masterWrapper) { this(METRICS_NAME, diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java index c2fc6b9fcb6..bcefda0c9c3 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsSnapshotSourceImpl.java @@ -20,14 +20,14 @@ package org.apache.hadoop.hbase.master; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.lib.MutableHistogram; +import org.apache.hadoop.metrics2.MetricHistogram; @InterfaceAudience.Private public class MetricsSnapshotSourceImpl extends BaseSourceImpl implements MetricsSnapshotSource { - private MutableHistogram snapshotTimeHisto; - private MutableHistogram snapshotCloneTimeHisto; - private MutableHistogram snapshotRestoreTimeHisto; + private MetricHistogram snapshotTimeHisto; + private MetricHistogram snapshotCloneTimeHisto; + private MetricHistogram snapshotRestoreTimeHisto; public MetricsSnapshotSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java index da34df23049..0a74630f92f 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancerSourceImpl.java @@ -20,14 +20,14 @@ package org.apache.hadoop.hbase.master.balancer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.metrics2.lib.MutableHistogram; +import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; @InterfaceAudience.Private public class MetricsBalancerSourceImpl extends BaseSourceImpl implements MetricsBalancerSource{ - private MutableHistogram blanceClusterHisto; - private MutableCounterLong miscCount; + private MetricHistogram blanceClusterHisto; + private MutableFastCounter miscCount; public MetricsBalancerSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java index 6756a21e1bc..f843ec23900 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java @@ -24,8 +24,7 @@ import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.impl.JmxCacheBuster; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry; -import org.apache.hadoop.metrics2.lib.MetricMutableQuantiles; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.metrics2.lib.MutableHistogram; import org.apache.hadoop.metrics2.source.JvmMetrics; @@ -88,7 +87,7 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { * @param value the new value of the gauge. */ public void setGauge(String gaugeName, long value) { - MutableGaugeLong gaugeInt = metricsRegistry.getLongGauge(gaugeName, value); + MutableGaugeLong gaugeInt = metricsRegistry.getGauge(gaugeName, value); gaugeInt.set(value); } @@ -99,7 +98,7 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { * @param delta The amount to increment the gauge by. */ public void incGauge(String gaugeName, long delta) { - MutableGaugeLong gaugeInt = metricsRegistry.getLongGauge(gaugeName, 0l); + MutableGaugeLong gaugeInt = metricsRegistry.getGauge(gaugeName, 0l); gaugeInt.incr(delta); } @@ -110,7 +109,7 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { * @param delta the ammount to subtract from a gauge value. */ public void decGauge(String gaugeName, long delta) { - MutableGaugeLong gaugeInt = metricsRegistry.getLongGauge(gaugeName, 0l); + MutableGaugeLong gaugeInt = metricsRegistry.getGauge(gaugeName, 0l); gaugeInt.decr(delta); } @@ -121,7 +120,7 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { * @param delta the ammount to increment */ public void incCounters(String key, long delta) { - MutableCounterLong counter = metricsRegistry.getLongCounter(key, 0l); + MutableFastCounter counter = metricsRegistry.getCounter(key, 0l); counter.incr(delta); } @@ -132,12 +131,6 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { histo.add(value); } - @Override - public void updateQuantile(String name, long value) { - MetricMutableQuantiles histo = metricsRegistry.getQuantile(name); - histo.add(value); - } - /** * Remove a named gauge. * diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index 9ee2b2942c7..4e66ac07636 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -24,7 +24,7 @@ import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.Interns; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; /** * Hadoop2 implementation of MetricsRegionServerSource. @@ -45,13 +45,13 @@ public class MetricsRegionServerSourceImpl private final MetricHistogram replayHisto; private final MetricHistogram scanNextHisto; - private final MutableCounterLong slowPut; - private final MutableCounterLong slowDelete; - private final MutableCounterLong slowGet; - private final MutableCounterLong slowIncrement; - private final MutableCounterLong slowAppend; - private final MutableCounterLong splitRequest; - private final MutableCounterLong splitSuccess; + private final MutableFastCounter slowPut; + private final MutableFastCounter slowDelete; + private final MutableFastCounter slowGet; + private final MutableFastCounter slowIncrement; + private final MutableFastCounter slowAppend; + private final MutableFastCounter splitRequest; + private final MutableFastCounter splitSuccess; private final MetricHistogram splitTimeHisto; private final MetricHistogram flushTimeHisto; diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index eefab08822e..494d3de1bc0 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -23,11 +23,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry; import org.apache.hadoop.metrics2.lib.Interns; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.metrics2.lib.MutableHistogram; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; @InterfaceAudience.Private public class MetricsRegionSourceImpl implements MetricsRegionSource { @@ -53,12 +53,12 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { private final String regionAppendKey; private final String regionScanNextKey; - private final MutableCounterLong regionPut; - private final MutableCounterLong regionDelete; - private final MutableCounterLong regionIncrement; - private final MutableCounterLong regionAppend; - private final MutableHistogram regionGet; - private final MutableHistogram regionScanNext; + private final MutableFastCounter regionPut; + private final MutableFastCounter regionDelete; + private final MutableFastCounter regionIncrement; + private final MutableFastCounter regionAppend; + private final MetricHistogram regionGet; + private final MetricHistogram regionScanNext; private final int hashCode; public MetricsRegionSourceImpl(MetricsRegionWrapper regionWrapper, @@ -80,16 +80,16 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { String suffix = "Count"; regionPutKey = regionNamePrefix + MetricsRegionServerSource.MUTATE_KEY + suffix; - regionPut = registry.getLongCounter(regionPutKey, 0L); + regionPut = registry.getCounter(regionPutKey, 0L); regionDeleteKey = regionNamePrefix + MetricsRegionServerSource.DELETE_KEY + suffix; - regionDelete = registry.getLongCounter(regionDeleteKey, 0L); + regionDelete = registry.getCounter(regionDeleteKey, 0L); regionIncrementKey = regionNamePrefix + MetricsRegionServerSource.INCREMENT_KEY + suffix; - regionIncrement = registry.getLongCounter(regionIncrementKey, 0L); + regionIncrement = registry.getCounter(regionIncrementKey, 0L); regionAppendKey = regionNamePrefix + MetricsRegionServerSource.APPEND_KEY + suffix; - regionAppend = registry.getLongCounter(regionAppendKey, 0L); + regionAppend = registry.getCounter(regionAppendKey, 0L); regionGetKey = regionNamePrefix + MetricsRegionServerSource.GET_KEY; regionGet = registry.newTimeHistogram(regionGetKey); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java index 90ac4c358c5..aec327830ca 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java @@ -21,7 +21,7 @@ package org.apache.hadoop.hbase.regionserver.wal; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; import org.apache.hadoop.metrics2.MetricHistogram; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; /** @@ -36,10 +36,10 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo private final MetricHistogram appendSizeHisto; private final MetricHistogram appendTimeHisto; private final MetricHistogram syncTimeHisto; - private final MutableCounterLong appendCount; - private final MutableCounterLong slowAppendCount; - private final MutableCounterLong logRollRequested; - private final MutableCounterLong lowReplicationLogRollRequested; + private final MutableFastCounter appendCount; + private final MutableFastCounter slowAppendCount; + private final MutableFastCounter logRollRequested; + private final MutableFastCounter lowReplicationLogRollRequested; public MetricsWALSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java index 392cd39c04e..93b10b604dd 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hbase.replication.regionserver; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; public class MetricsReplicationGlobalSourceSource implements MetricsReplicationSourceSource{ @@ -26,38 +26,38 @@ public class MetricsReplicationGlobalSourceSource implements MetricsReplicationS private final MutableGaugeLong ageOfLastShippedOpGauge; private final MutableGaugeLong sizeOfLogQueueGauge; - private final MutableCounterLong logReadInEditsCounter; - private final MutableCounterLong logEditsFilteredCounter; - private final MutableCounterLong shippedBatchesCounter; - private final MutableCounterLong shippedOpsCounter; - private final MutableCounterLong shippedKBsCounter; - private final MutableCounterLong logReadInBytesCounter; - private final MutableCounterLong shippedHFilesCounter; + private final MutableFastCounter logReadInEditsCounter; + private final MutableFastCounter logEditsFilteredCounter; + private final MutableFastCounter shippedBatchesCounter; + private final MutableFastCounter shippedOpsCounter; + private final MutableFastCounter shippedKBsCounter; + private final MutableFastCounter logReadInBytesCounter; + private final MutableFastCounter shippedHFilesCounter; private final MutableGaugeLong sizeOfHFileRefsQueueGauge; public MetricsReplicationGlobalSourceSource(MetricsReplicationSourceImpl rms) { this.rms = rms; - ageOfLastShippedOpGauge = rms.getMetricsRegistry().getLongGauge(SOURCE_AGE_OF_LAST_SHIPPED_OP, 0L); + ageOfLastShippedOpGauge = rms.getMetricsRegistry().getGauge(SOURCE_AGE_OF_LAST_SHIPPED_OP, 0L); - sizeOfLogQueueGauge = rms.getMetricsRegistry().getLongGauge(SOURCE_SIZE_OF_LOG_QUEUE, 0L); + sizeOfLogQueueGauge = rms.getMetricsRegistry().getGauge(SOURCE_SIZE_OF_LOG_QUEUE, 0L); - shippedBatchesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_BATCHES, 0L); + shippedBatchesCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_BATCHES, 0L); - shippedOpsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_OPS, 0L); + shippedOpsCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_OPS, 0L); - shippedKBsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_KBS, 0L); + shippedKBsCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_KBS, 0L); - logReadInBytesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_READ_IN_BYTES, 0L); + logReadInBytesCounter = rms.getMetricsRegistry().getCounter(SOURCE_LOG_READ_IN_BYTES, 0L); - logReadInEditsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_READ_IN_EDITS, 0L); + logReadInEditsCounter = rms.getMetricsRegistry().getCounter(SOURCE_LOG_READ_IN_EDITS, 0L); - logEditsFilteredCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_EDITS_FILTERED, 0L); + logEditsFilteredCounter = rms.getMetricsRegistry().getCounter(SOURCE_LOG_EDITS_FILTERED, 0L); - shippedHFilesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_HFILES, 0L); + shippedHFilesCounter = rms.getMetricsRegistry().getCounter(SOURCE_SHIPPED_HFILES, 0L); sizeOfHFileRefsQueueGauge = - rms.getMetricsRegistry().getLongGauge(SOURCE_SIZE_OF_HFILE_REFS_QUEUE, 0L); + rms.getMetricsRegistry().getGauge(SOURCE_SIZE_OF_HFILE_REFS_QUEUE, 0L); } @Override public void setLastShippedAge(long age) { diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java index 8f4a3375417..540212aad53 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java @@ -18,21 +18,21 @@ package org.apache.hadoop.hbase.replication.regionserver; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; public class MetricsReplicationSinkSourceImpl implements MetricsReplicationSinkSource { private final MutableGaugeLong ageGauge; - private final MutableCounterLong batchesCounter; - private final MutableCounterLong opsCounter; - private final MutableCounterLong hfilesCounter; + private final MutableFastCounter batchesCounter; + private final MutableFastCounter opsCounter; + private final MutableFastCounter hfilesCounter; public MetricsReplicationSinkSourceImpl(MetricsReplicationSourceImpl rms) { - ageGauge = rms.getMetricsRegistry().getLongGauge(SINK_AGE_OF_LAST_APPLIED_OP, 0L); - batchesCounter = rms.getMetricsRegistry().getLongCounter(SINK_APPLIED_BATCHES, 0L); - opsCounter = rms.getMetricsRegistry().getLongCounter(SINK_APPLIED_OPS, 0L); - hfilesCounter = rms.getMetricsRegistry().getLongCounter(SINK_APPLIED_HFILES, 0L); + ageGauge = rms.getMetricsRegistry().getGauge(SINK_AGE_OF_LAST_APPLIED_OP, 0L); + batchesCounter = rms.getMetricsRegistry().getCounter(SINK_APPLIED_BATCHES, 0L); + opsCounter = rms.getMetricsRegistry().getCounter(SINK_APPLIED_OPS, 0L); + hfilesCounter = rms.getMetricsRegistry().getCounter(SINK_APPLIED_HFILES, 0L); } @Override public void setLastAppliedOpAge(long age) { diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java index 217cc3e2a46..99417122b8c 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.replication.regionserver; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSourceSource { @@ -37,13 +37,13 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou private final MutableGaugeLong ageOfLastShippedOpGauge; private final MutableGaugeLong sizeOfLogQueueGauge; - private final MutableCounterLong logReadInEditsCounter; - private final MutableCounterLong logEditsFilteredCounter; - private final MutableCounterLong shippedBatchesCounter; - private final MutableCounterLong shippedOpsCounter; - private final MutableCounterLong shippedKBsCounter; - private final MutableCounterLong logReadInBytesCounter; - private final MutableCounterLong shippedHFilesCounter; + private final MutableFastCounter logReadInEditsCounter; + private final MutableFastCounter logEditsFilteredCounter; + private final MutableFastCounter shippedBatchesCounter; + private final MutableFastCounter shippedOpsCounter; + private final MutableFastCounter shippedKBsCounter; + private final MutableFastCounter logReadInBytesCounter; + private final MutableFastCounter shippedHFilesCounter; private final MutableGaugeLong sizeOfHFileRefsQueueGauge; public MetricsReplicationSourceSourceImpl(MetricsReplicationSourceImpl rms, String id) { @@ -51,34 +51,34 @@ public class MetricsReplicationSourceSourceImpl implements MetricsReplicationSou this.id = id; ageOfLastShippedOpKey = "source." + id + ".ageOfLastShippedOp"; - ageOfLastShippedOpGauge = rms.getMetricsRegistry().getLongGauge(ageOfLastShippedOpKey, 0L); + ageOfLastShippedOpGauge = rms.getMetricsRegistry().getGauge(ageOfLastShippedOpKey, 0L); sizeOfLogQueueKey = "source." + id + ".sizeOfLogQueue"; - sizeOfLogQueueGauge = rms.getMetricsRegistry().getLongGauge(sizeOfLogQueueKey, 0L); + sizeOfLogQueueGauge = rms.getMetricsRegistry().getGauge(sizeOfLogQueueKey, 0L); shippedBatchesKey = "source." + this.id + ".shippedBatches"; - shippedBatchesCounter = rms.getMetricsRegistry().getLongCounter(shippedBatchesKey, 0L); + shippedBatchesCounter = rms.getMetricsRegistry().getCounter(shippedBatchesKey, 0L); shippedOpsKey = "source." + this.id + ".shippedOps"; - shippedOpsCounter = rms.getMetricsRegistry().getLongCounter(shippedOpsKey, 0L); + shippedOpsCounter = rms.getMetricsRegistry().getCounter(shippedOpsKey, 0L); shippedKBsKey = "source." + this.id + ".shippedKBs"; - shippedKBsCounter = rms.getMetricsRegistry().getLongCounter(shippedKBsKey, 0L); + shippedKBsCounter = rms.getMetricsRegistry().getCounter(shippedKBsKey, 0L); logReadInBytesKey = "source." + this.id + ".logReadInBytes"; - logReadInBytesCounter = rms.getMetricsRegistry().getLongCounter(logReadInBytesKey, 0L); + logReadInBytesCounter = rms.getMetricsRegistry().getCounter(logReadInBytesKey, 0L); logReadInEditsKey = "source." + id + ".logEditsRead"; - logReadInEditsCounter = rms.getMetricsRegistry().getLongCounter(logReadInEditsKey, 0L); + logReadInEditsCounter = rms.getMetricsRegistry().getCounter(logReadInEditsKey, 0L); logEditsFilteredKey = "source." + id + ".logEditsFiltered"; - logEditsFilteredCounter = rms.getMetricsRegistry().getLongCounter(logEditsFilteredKey, 0L); + logEditsFilteredCounter = rms.getMetricsRegistry().getCounter(logEditsFilteredKey, 0L); shippedHFilesKey = "source." + this.id + ".shippedHFiles"; - shippedHFilesCounter = rms.getMetricsRegistry().getLongCounter(shippedHFilesKey, 0L); + shippedHFilesCounter = rms.getMetricsRegistry().getCounter(shippedHFilesKey, 0L); sizeOfHFileRefsQueueKey = "source." + id + ".sizeOfHFileRefsQueue"; - sizeOfHFileRefsQueueGauge = rms.getMetricsRegistry().getLongGauge(sizeOfHFileRefsQueueKey, 0L); + sizeOfHFileRefsQueueGauge = rms.getMetricsRegistry().getGauge(sizeOfHFileRefsQueueKey, 0L); } @Override public void setLastShippedAge(long age) { diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java index 9eae18b8c72..71755b742d1 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/rest/MetricsRESTSourceImpl.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.rest; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; /** * Hadoop Two implementation of a metrics2 source that will export metrics from the Rest server to @@ -31,15 +31,15 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong; @InterfaceAudience.Private public class MetricsRESTSourceImpl extends BaseSourceImpl implements MetricsRESTSource { - private MutableCounterLong request; - private MutableCounterLong sucGet; - private MutableCounterLong sucPut; - private MutableCounterLong sucDel; - private MutableCounterLong sucScan; - private MutableCounterLong fGet; - private MutableCounterLong fPut; - private MutableCounterLong fDel; - private MutableCounterLong fScan; + private MutableFastCounter request; + private MutableFastCounter sucGet; + private MutableFastCounter sucPut; + private MutableFastCounter sucDel; + private MutableFastCounter sucScan; + private MutableFastCounter fGet; + private MutableFastCounter fPut; + private MutableFastCounter fDel; + private MutableFastCounter fScan; public MetricsRESTSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, CONTEXT, JMX_CONTEXT); @@ -55,17 +55,17 @@ public class MetricsRESTSourceImpl extends BaseSourceImpl implements MetricsREST @Override public void init() { super.init(); - request = getMetricsRegistry().getLongCounter(REQUEST_KEY, 0l); + request = getMetricsRegistry().getCounter(REQUEST_KEY, 0l); - sucGet = getMetricsRegistry().getLongCounter(SUCCESSFUL_GET_KEY, 0l); - sucPut = getMetricsRegistry().getLongCounter(SUCCESSFUL_PUT_KEY, 0l); - sucDel = getMetricsRegistry().getLongCounter(SUCCESSFUL_DELETE_KEY, 0l); - sucScan = getMetricsRegistry().getLongCounter(SUCCESSFUL_SCAN_KEY, 0L); + sucGet = getMetricsRegistry().getCounter(SUCCESSFUL_GET_KEY, 0l); + sucPut = getMetricsRegistry().getCounter(SUCCESSFUL_PUT_KEY, 0l); + sucDel = getMetricsRegistry().getCounter(SUCCESSFUL_DELETE_KEY, 0l); + sucScan = getMetricsRegistry().getCounter(SUCCESSFUL_SCAN_KEY, 0L); - fGet = getMetricsRegistry().getLongCounter(FAILED_GET_KEY, 0l); - fPut = getMetricsRegistry().getLongCounter(FAILED_PUT_KEY, 0l); - fDel = getMetricsRegistry().getLongCounter(FAILED_DELETE_KEY, 0l); - fScan = getMetricsRegistry().getLongCounter(FAILED_SCAN_KEY, 0l); + fGet = getMetricsRegistry().getCounter(FAILED_GET_KEY, 0l); + fPut = getMetricsRegistry().getCounter(FAILED_PUT_KEY, 0l); + fDel = getMetricsRegistry().getCounter(FAILED_DELETE_KEY, 0l); + fScan = getMetricsRegistry().getCounter(FAILED_SCAN_KEY, 0l); } @Override diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java index f9612e5ba1d..f5d83b1e0ea 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceImpl.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.thrift; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; +import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.metrics2.lib.MutableHistogram; @@ -32,12 +33,12 @@ import org.apache.hadoop.metrics2.lib.MutableHistogram; public class MetricsThriftServerSourceImpl extends BaseSourceImpl implements MetricsThriftServerSource { - private MutableHistogram batchGetStat; - private MutableHistogram batchMutateStat; - private MutableHistogram queueTimeStat; + private MetricHistogram batchGetStat; + private MetricHistogram batchMutateStat; + private MetricHistogram queueTimeStat; - private MutableHistogram thriftCallStat; - private MutableHistogram thriftSlowCallStat; + private MetricHistogram thriftCallStat; + private MetricHistogram thriftSlowCallStat; private MutableGaugeLong callQueueLenGauge; @@ -56,7 +57,7 @@ public class MetricsThriftServerSourceImpl extends BaseSourceImpl implements queueTimeStat = getMetricsRegistry().newTimeHistogram(TIME_IN_QUEUE_KEY); thriftCallStat = getMetricsRegistry().newTimeHistogram(THRIFT_CALL_KEY); thriftSlowCallStat = getMetricsRegistry().newTimeHistogram(SLOW_THRIFT_CALL_KEY); - callQueueLenGauge = getMetricsRegistry().getLongGauge(CALL_QUEUE_LEN_KEY, 0); + callQueueLenGauge = getMetricsRegistry().getGauge(CALL_QUEUE_LEN_KEY, 0); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java index ee13c7634db..1f3da12566e 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java @@ -105,28 +105,6 @@ public class DynamicMetricsRegistry { return tagsMap.get(name); } - /** - * Create a mutable integer counter - * @param name of the metric - * @param desc metric description - * @param iVal initial value - * @return a new counter object - */ - public MutableCounterInt newCounter(String name, String desc, int iVal) { - return newCounter(new MetricsInfoImpl(name, desc), iVal); - } - - /** - * Create a mutable integer counter - * @param info metadata of the metric - * @param iVal initial value - * @return a new counter object - */ - public MutableCounterInt newCounter(MetricsInfo info, int iVal) { - MutableCounterInt ret = new MutableCounterInt(info, iVal); - return addNewMetricIfAbsent(info.name(), ret, MutableCounterInt.class); - } - /** * Create a mutable long integer counter * @param name of the metric @@ -134,7 +112,7 @@ public class DynamicMetricsRegistry { * @param iVal initial value * @return a new counter object */ - public MutableCounterLong newCounter(String name, String desc, long iVal) { + public MutableFastCounter newCounter(String name, String desc, long iVal) { return newCounter(new MetricsInfoImpl(name, desc), iVal); } @@ -144,30 +122,9 @@ public class DynamicMetricsRegistry { * @param iVal initial value * @return a new counter object */ - public MutableCounterLong newCounter(MetricsInfo info, long iVal) { - MutableCounterLong ret = new MutableCounterLong(info, iVal); - return addNewMetricIfAbsent(info.name(), ret, MutableCounterLong.class); - } - - /** - * Create a mutable integer gauge - * @param name of the metric - * @param desc metric description - * @param iVal initial value - * @return a new gauge object - */ - public MutableGaugeInt newGauge(String name, String desc, int iVal) { - return newGauge(new MetricsInfoImpl(name, desc), iVal); - } - /** - * Create a mutable integer gauge - * @param info metadata of the metric - * @param iVal initial value - * @return a new gauge object - */ - public MutableGaugeInt newGauge(MetricsInfo info, int iVal) { - MutableGaugeInt ret = new MutableGaugeInt(info, iVal); - return addNewMetricIfAbsent(info.name(), ret, MutableGaugeInt.class); + public MutableFastCounter newCounter(MetricsInfo info, long iVal) { + MutableFastCounter ret = new MutableFastCounter(info, iVal); + return addNewMetricIfAbsent(info.name(), ret, MutableFastCounter.class); } /** @@ -326,19 +283,6 @@ public class DynamicMetricsRegistry { return addNewMetricIfAbsent(name, histo, MutableSizeHistogram.class); } - /** - * Create a new MutableQuantile(A more accurate histogram). - * @param name The name of the histogram - * @return a new MutableQuantile - */ - public MetricMutableQuantiles newQuantile(String name) { - return newQuantile(name, ""); - } - - public MetricMutableQuantiles newQuantile(String name, String desc) { - MetricMutableQuantiles histo = new MetricMutableQuantiles(name, desc, "Ops", "", 60); - return addNewMetricIfAbsent(name, histo, MetricMutableQuantiles.class); - } synchronized void add(String name, MutableMetric metric) { addNewMetricIfAbsent(name, metric, MutableMetric.class); @@ -475,7 +419,7 @@ public class DynamicMetricsRegistry { * @param gaugeName name of the gauge to create or get. * @param potentialStartingValue value of the new gauge if we have to create it. */ - public MutableGaugeLong getLongGauge(String gaugeName, long potentialStartingValue) { + public MutableGaugeLong getGauge(String gaugeName, long potentialStartingValue) { //Try and get the guage. MutableMetric metric = metricsMap.get(gaugeName); @@ -510,12 +454,12 @@ public class DynamicMetricsRegistry { * @param counterName Name of the counter to get * @param potentialStartingValue starting value if we have to create a new counter */ - public MutableCounterLong getLongCounter(String counterName, long potentialStartingValue) { - //See getLongGauge for description on how this works. + public MutableFastCounter getCounter(String counterName, long potentialStartingValue) { + //See getGauge for description on how this works. MutableMetric counter = metricsMap.get(counterName); if (counter == null) { - MutableCounterLong newCounter = - new MutableCounterLong(new MetricsInfoImpl(counterName, ""), potentialStartingValue); + MutableFastCounter newCounter = + new MutableFastCounter(new MetricsInfoImpl(counterName, ""), potentialStartingValue); counter = metricsMap.putIfAbsent(counterName, newCounter); if (counter == null) { return newCounter; @@ -523,16 +467,16 @@ public class DynamicMetricsRegistry { } - if (!(counter instanceof MutableCounterLong)) { + if (!(counter instanceof MutableCounter)) { throw new MetricsException("Metric already exists in registry for metric name: " + - counterName + " and not of type MetricMutableCounterLong"); + counterName + " and not of type MutableCounter"); } - return (MutableCounterLong) counter; + return (MutableFastCounter) counter; } public MutableHistogram getHistogram(String histoName) { - //See getLongGauge for description on how this works. + //See getGauge for description on how this works. MutableMetric histo = metricsMap.get(histoName); if (histo == null) { MutableHistogram newCounter = @@ -552,27 +496,6 @@ public class DynamicMetricsRegistry { return (MutableHistogram) histo; } - public MetricMutableQuantiles getQuantile(String histoName) { - //See getLongGauge for description on how this works. - MutableMetric histo = metricsMap.get(histoName); - if (histo == null) { - MetricMutableQuantiles newCounter = - new MetricMutableQuantiles(histoName, "", "Ops", "", 60); - histo = metricsMap.putIfAbsent(histoName, newCounter); - if (histo == null) { - return newCounter; - } - } - - - if (!(histo instanceof MetricMutableQuantiles)) { - throw new MetricsException("Metric already exists in registry for metric name: " + - histoName + " and not of type MutableHistogram"); - } - - return (MetricMutableQuantiles) histo; - } - private T addNewMetricIfAbsent(String name, T ret, diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java deleted file mode 100644 index c03654b8d5b..00000000000 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metrics2.lib; - -import static org.apache.hadoop.metrics2.lib.Interns.info; - -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.metrics2.MetricHistogram; -import org.apache.hadoop.metrics2.MetricsExecutor; -import org.apache.hadoop.metrics2.MetricsInfo; -import org.apache.hadoop.metrics2.MetricsRecordBuilder; -import org.apache.hadoop.metrics2.util.MetricQuantile; -import org.apache.hadoop.metrics2.util.MetricSampleQuantiles; - -import com.google.common.annotations.VisibleForTesting; - -/** - * Watches a stream of long values, maintaining online estimates of specific quantiles with provably - * low error bounds. This is particularly useful for accurate high-percentile (e.g. 95th, 99th) - * latency metrics. - */ -@InterfaceAudience.Private -public class MetricMutableQuantiles extends MutableMetric implements MetricHistogram { - - static final MetricQuantile[] quantiles = {new MetricQuantile(0.50, 0.050), - new MetricQuantile(0.75, 0.025), new MetricQuantile(0.90, 0.010), - new MetricQuantile(0.95, 0.005), new MetricQuantile(0.99, 0.001)}; - - private final MetricsInfo numInfo; - private final MetricsInfo[] quantileInfos; - private final int interval; - - private MetricSampleQuantiles estimator; - private long previousCount = 0; - private MetricsExecutor executor; - - - @VisibleForTesting - protected Map previousSnapshot = null; - - /** - * Instantiates a new {@link MetricMutableQuantiles} for a metric that rolls itself over on the - * specified time interval. - * - * @param name of the metric - * @param description long-form textual description of the metric - * @param sampleName type of items in the stream (e.g., "Ops") - * @param valueName type of the values - * @param interval rollover interval (in seconds) of the estimator - */ - public MetricMutableQuantiles(String name, String description, String sampleName, - String valueName, int interval) { - String ucName = StringUtils.capitalize(name); - String usName = StringUtils.capitalize(sampleName); - String uvName = StringUtils.capitalize(valueName); - String desc = StringUtils.uncapitalize(description); - String lsName = StringUtils.uncapitalize(sampleName); - String lvName = StringUtils.uncapitalize(valueName); - - numInfo = info(ucName + "Num" + usName, String.format( - "Number of %s for %s with %ds interval", lsName, desc, interval)); - // Construct the MetricsInfos for the quantiles, converting to percentiles - quantileInfos = new MetricsInfo[quantiles.length]; - String nameTemplate = "%s%dthPercentile%dsInterval%s"; - String descTemplate = "%d percentile %s with %d second interval for %s"; - for (int i = 0; i < quantiles.length; i++) { - int percentile = (int) (100 * quantiles[i].quantile); - quantileInfos[i] = info(String.format(nameTemplate, ucName, percentile, interval, uvName), - String.format(descTemplate, percentile, lvName, interval, desc)); - } - - estimator = new MetricSampleQuantiles(quantiles); - executor = new MetricsExecutorImpl(); - this.interval = interval; - executor.getExecutor().scheduleAtFixedRate(new RolloverSample(this), - interval, - interval, - TimeUnit.SECONDS); - } - - @Override - public synchronized void snapshot(MetricsRecordBuilder builder, boolean all) { - if (all || changed()) { - builder.addGauge(numInfo, previousCount); - for (int i = 0; i < quantiles.length; i++) { - long newValue = 0; - // If snapshot is null, we failed to update since the window was empty - if (previousSnapshot != null) { - newValue = previousSnapshot.get(quantiles[i]); - } - builder.addGauge(quantileInfos[i], newValue); - } - if (changed()) { - clearChanged(); - } - } - } - - public synchronized void add(long value) { - estimator.insert(value); - } - - public int getInterval() { - return interval; - } - - /** Runnable used to periodically roll over the internal {@link org.apache.hadoop.metrics2.util.MetricSampleQuantiles} every interval. */ - private static class RolloverSample implements Runnable { - - MetricMutableQuantiles parent; - - public RolloverSample(MetricMutableQuantiles parent) { - this.parent = parent; - } - - @Override - public void run() { - synchronized (parent) { - try { - parent.previousCount = parent.estimator.getCount(); - parent.previousSnapshot = parent.estimator.snapshot(); - } catch (IOException e) { - // Couldn't get a new snapshot because the window was empty - parent.previousCount = 0; - parent.previousSnapshot = null; - } - parent.estimator.clear(); - } - parent.setChanged(); - } - - } -} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java index f70413ecfb1..c3816097416 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java @@ -28,7 +28,7 @@ import org.apache.hadoop.metrics2.MetricsExecutor; /** * Class to handle the ScheduledExecutorService{@link ScheduledExecutorService} used by - * MetricMutableQuantiles{@link MetricMutableQuantiles}, MetricsRegionAggregateSourceImpl, and + * MetricsRegionAggregateSourceImpl, and * JmxCacheBuster */ @InterfaceAudience.Private diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableFastCounter.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableFastCounter.java new file mode 100644 index 00000000000..3f6c4a77455 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableFastCounter.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.lib; + +import org.apache.hadoop.hbase.util.Counter; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +public class MutableFastCounter extends MutableCounter { + + private final Counter counter; + + protected MutableFastCounter(MetricsInfo info, long iVal) { + super(info); + counter = new Counter(iVal); + } + + @Override + public void incr() { + counter.increment(); + setChanged(); + } + + /** + * Increment the value by a delta + * @param delta of the increment + */ + public void incr(long delta) { + counter.add(delta); + setChanged(); + } + + @Override + public void snapshot(MetricsRecordBuilder builder, boolean all) { + if (all || changed()) { + builder.addCounter(info(), value()); + clearChanged(); + } + } + + public long value() { + return counter.get(); + } +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java index de299402827..a6f0d147af9 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java @@ -18,124 +18,81 @@ package org.apache.hadoop.metrics2.lib; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.Counter; +import org.apache.hadoop.hbase.util.FastLongHistogram; import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; -import com.yammer.metrics.stats.ExponentiallyDecayingSample; -import com.yammer.metrics.stats.Sample; -import com.yammer.metrics.stats.Snapshot; - /** * A histogram implementation that runs in constant space, and exports to hadoop2's metrics2 system. */ @InterfaceAudience.Private public class MutableHistogram extends MutableMetric implements MetricHistogram { - - private static final int DEFAULT_SAMPLE_SIZE = 2046; - // the bias towards sampling from more recent data. - // Per Cormode et al. an alpha of 0.015 strongly biases to the last 5 minutes - private static final double DEFAULT_ALPHA = 0.015; + // Double buffer the two FastLongHistograms. + // As they are reset they learn how the buckets should be spaced + // So keep two around and use them + protected final FastLongHistogram histogram; protected final String name; protected final String desc; - private final Sample sample; - private final AtomicLong min; - private final AtomicLong max; - private final AtomicLong sum; - private final AtomicLong count; + + protected final Counter counter = new Counter(0); public MutableHistogram(MetricsInfo info) { this(info.name(), info.description()); } public MutableHistogram(String name, String description) { + this(name, description, Integer.MAX_VALUE << 2); + } + + protected MutableHistogram(String name, String description, long maxExpected) { this.name = StringUtils.capitalize(name); this.desc = StringUtils.uncapitalize(description); - sample = new ExponentiallyDecayingSample(DEFAULT_SAMPLE_SIZE, DEFAULT_ALPHA); - count = new AtomicLong(); - min = new AtomicLong(Long.MAX_VALUE); - max = new AtomicLong(Long.MIN_VALUE); - sum = new AtomicLong(); + + this.histogram = new FastLongHistogram(FastLongHistogram.DEFAULT_NBINS, 1, maxExpected); } public void add(final long val) { - setChanged(); - count.incrementAndGet(); - sample.update(val); - setMax(val); - setMin(val); - sum.getAndAdd(val); - } - - private void setMax(final long potentialMax) { - boolean done = false; - while (!done) { - final long currentMax = max.get(); - done = currentMax >= potentialMax - || max.compareAndSet(currentMax, potentialMax); - } - } - - private void setMin(long potentialMin) { - boolean done = false; - while (!done) { - final long currentMin = min.get(); - done = currentMin <= potentialMin - || min.compareAndSet(currentMin, potentialMin); - } - } - - public long getMax() { - if (count.get() > 0) { - return max.get(); - } - return 0L; - } - - public long getMin() { - if (count.get() > 0) { - return min.get(); - } - return 0L; - } - - public double getMean() { - long cCount = count.get(); - if (cCount > 0) { - return sum.get() / (double) cCount; - } - return 0.0; + counter.increment(); + histogram.add(val, 1); } @Override - public void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { - if (all || changed()) { - clearChanged(); - updateSnapshotMetrics(metricsRecordBuilder); - } + public synchronized void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { + // Get a reference to the old histogram. + FastLongHistogram histo = histogram.reset(); + updateSnapshotMetrics(metricsRecordBuilder, histo); } - - public void updateSnapshotMetrics(MetricsRecordBuilder metricsRecordBuilder) { - final Snapshot s = sample.getSnapshot(); - metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), count.get()); - metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), getMin()); - metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), getMax()); - metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), getMean()); + protected void updateSnapshotMetrics(MetricsRecordBuilder metricsRecordBuilder, + FastLongHistogram histo) { + metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), counter.get()); + metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), histo.getMin()); + metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), histo.getMax()); + metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), histo.getMean()); - metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc), s.getMedian()); - metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), - s.get75thPercentile()); - metricsRecordBuilder.addGauge(Interns.info(name + NINETIETH_PERCENTILE_METRIC_NAME, desc), - s.getValue(0.90)); - metricsRecordBuilder.addGauge(Interns.info(name + NINETY_FIFTH_PERCENTILE_METRIC_NAME, desc), - s.get95thPercentile()); - metricsRecordBuilder.addGauge(Interns.info(name + NINETY_NINETH_PERCENTILE_METRIC_NAME, desc), - s.get99thPercentile()); + long[] percentiles = histo.getQuantiles(); + + metricsRecordBuilder.addGauge(Interns.info(name + TWENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), + percentiles[0]); + metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc), + percentiles[1]); + metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), + percentiles[2]); + metricsRecordBuilder.addGauge(Interns.info(name + NINETIETH_PERCENTILE_METRIC_NAME, desc), + percentiles[3]); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_FIFTH_PERCENTILE_METRIC_NAME, desc), + percentiles[4]); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_EIGHTH_PERCENTILE_METRIC_NAME, desc), + percentiles[5]); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_NINETH_PERCENTILE_METRIC_NAME, desc), + percentiles[6]); + metricsRecordBuilder.addGauge( + Interns.info(name + NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME, desc), + percentiles[7]); } } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java index ac1f497c98a..ac8aee0a8d5 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java @@ -18,26 +18,29 @@ package org.apache.hadoop.metrics2.lib; -import java.util.concurrent.atomic.AtomicLongArray; - import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; - /** * Extended histogram implementation with metric range counters. */ @InterfaceAudience.Private -public abstract class MutableRangeHistogram extends MutableHistogram { +public abstract class MutableRangeHistogram extends MutableHistogram implements MetricHistogram { public MutableRangeHistogram(MetricsInfo info) { this(info.name(), info.description()); } public MutableRangeHistogram(String name, String description) { - super(name, description); + this(name, description, Integer.MAX_VALUE << 2); } - + + public MutableRangeHistogram(String name, String description, long expectedMax) { + super(name, description, expectedMax); + } + /** * Returns the type of range histogram size or time */ @@ -46,49 +49,39 @@ public abstract class MutableRangeHistogram extends MutableHistogram { /** * Returns the ranges to be counted */ - public abstract long[] getRange(); - - /** - * Returns the range counts - */ - public abstract AtomicLongArray getRangeVals(); + public abstract long[] getRanges(); - @Override - public void add(final long val) { - super.add(val); - updateBand(val); - } - - private void updateBand(final long val) { - int i; - for (i=0; i getRange()[i]; i++); - getRangeVals().incrementAndGet(i); - } @Override - public void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { - if (all || changed()) { - clearChanged(); - updateSnapshotMetrics(metricsRecordBuilder); - updateSnapshotRangeMetrics(metricsRecordBuilder); - } + public synchronized void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { + // Get a reference to the old histogram. + FastLongHistogram histo = histogram.reset(); + updateSnapshotMetrics(metricsRecordBuilder, histo); + updateSnapshotRangeMetrics(metricsRecordBuilder, histo); } - - public void updateSnapshotRangeMetrics(MetricsRecordBuilder metricsRecordBuilder) { - long prior = 0; - for (int i = 0; i < getRange().length; i++) { - long val = getRangeVals().get(i); - if (val > 0) { + + public void updateSnapshotRangeMetrics(MetricsRecordBuilder metricsRecordBuilder, + FastLongHistogram histogram) { + long priorRange = 0; + long cumNum = 0; + + final long[] ranges = getRanges(); + final String rangeType = getRangeType(); + for (int i = 0; i < ranges.length - 1; i++) { + long val = histogram.getNumAtOrBelow(ranges[i]); + if (val - cumNum > 0) { metricsRecordBuilder.addCounter( - Interns.info(name + "_" + getRangeType() + "_" + prior + "-" + getRange()[i], desc), val); + Interns.info(name + "_" + rangeType + "_" + priorRange + "-" + ranges[i], desc), + val - cumNum); } - prior = getRange()[i]; + priorRange = ranges[i]; + cumNum = val; } - long val = getRangeVals().get(getRange().length); - if (val > 0) { + long val = histogram.getCount(); + if (val - cumNum > 0) { metricsRecordBuilder.addCounter( - Interns.info(name + "_" + getRangeType() + "_" + getRange()[getRange().length - 1] + "-inf", desc), - getRangeVals().get(getRange().length)); + Interns.info(name + "_" + rangeType + "_" + ranges[ranges.length - 1] + "-inf", desc), + val - cumNum); } } } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java index 2f1d57a4d2d..38e78a2324e 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableSizeHistogram.java @@ -18,8 +18,6 @@ package org.apache.hadoop.metrics2.lib; -import java.util.concurrent.atomic.AtomicLongArray; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsInfo; @@ -28,30 +26,29 @@ import org.apache.hadoop.metrics2.MetricsInfo; */ @InterfaceAudience.Private public class MutableSizeHistogram extends MutableRangeHistogram { - private final String rangeType = "SizeRangeCount"; - private final long[] ranges = {10,100,1000,10000,100000,1000000,10000000,100000000}; - private final AtomicLongArray rangeVals = new AtomicLongArray(getRange().length+1); + private final static String RANGE_TYPE = "SizeRangeCount"; + private final static long[] RANGES = {10,100,1000,10000,100000,1000000,10000000,100000000}; public MutableSizeHistogram(MetricsInfo info) { this(info.name(), info.description()); } public MutableSizeHistogram(String name, String description) { - super(name, description); + this(name, description, RANGES[RANGES.length-2]); + } + + public MutableSizeHistogram(String name, String description, long expectedMax) { + super(name, description, expectedMax); } @Override public String getRangeType() { - return rangeType; + return RANGE_TYPE; } @Override - public long[] getRange() { - return ranges; + public long[] getRanges() { + return RANGES; } - - @Override - public AtomicLongArray getRangeVals() { - return rangeVals; - } + } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java index 32d4fae4e93..aaf4359f18c 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableTimeHistogram.java @@ -18,8 +18,6 @@ package org.apache.hadoop.metrics2.lib; -import java.util.concurrent.atomic.AtomicLongArray; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsInfo; @@ -28,31 +26,30 @@ import org.apache.hadoop.metrics2.MetricsInfo; */ @InterfaceAudience.Private public class MutableTimeHistogram extends MutableRangeHistogram { - private final String rangeType = "TimeRangeCount"; - private final long[] ranges = + private final static String RANGE_TYPE = "TimeRangeCount"; + private final static long[] RANGES = { 1, 3, 10, 30, 100, 300, 1000, 3000, 10000, 30000, 60000, 120000, 300000, 600000 }; - private final AtomicLongArray rangeVals = new AtomicLongArray(ranges.length+1); public MutableTimeHistogram(MetricsInfo info) { this(info.name(), info.description()); } public MutableTimeHistogram(String name, String description) { - super(name, description); + this(name, description, RANGES[RANGES.length - 2]); + } + + public MutableTimeHistogram(String name, String description, long expectedMax) { + super(name, description, expectedMax); } @Override public String getRangeType() { - return rangeType; + return RANGE_TYPE; } @Override - public long[] getRange() { - return ranges; + public long[] getRanges() { + return RANGES; } - @Override - public AtomicLongArray getRangeVals() { - return rangeVals; - } } diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java index 7381fb9b870..2e374f7058a 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseSourceImpl.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.metrics; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.testclassification.MetricsTests; import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.junit.BeforeClass; import org.junit.Test; @@ -72,9 +73,9 @@ public class TestBaseSourceImpl { @Test public void testIncCounters() throws Exception { bmsi.incCounters("testinccounter", 100); - assertEquals(100, ((MutableCounterLong) bmsi.metricsRegistry.get("testinccounter")).value()); + assertEquals(100, ((MutableFastCounter) bmsi.metricsRegistry.get("testinccounter")).value()); bmsi.incCounters("testinccounter", 100); - assertEquals(200, ((MutableCounterLong) bmsi.metricsRegistry.get("testinccounter")).value()); + assertEquals(200, ((MutableFastCounter) bmsi.metricsRegistry.get("testinccounter")).value()); } diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon index 6986f129946..3dcd5e25f97 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon @@ -178,7 +178,6 @@ org.apache.hadoop.util.StringUtils; AgeSnapshot ageAtEvictionSnapshot = bc.getStats().getAgeAtEvictionSnapshot(); // Only show if non-zero mean and stddev as is the case in combinedblockcache double mean = ageAtEvictionSnapshot.getMean(); - double stddev = ageAtEvictionSnapshot.getStdDev(); Evicted @@ -197,13 +196,6 @@ org.apache.hadoop.util.StringUtils; Mean age of Blocks at eviction time (seconds) -<%if stddev > 0 %> - - StdDev - <% String.format("%,d", (long)(ageAtEvictionSnapshot.getStdDev()/1000000)) %> - Standard Deviation for age of Blocks at eviction time - - <%def hits_tmpl> diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon index 523d1b9beba..fa55f6a04ba 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon @@ -37,7 +37,6 @@ org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator; org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.Bucket; org.apache.hadoop.util.StringUtils; -com.yammer.metrics.stats.Snapshot; <%java> BlockCache bc = cacheConfig == null ? null : cacheConfig.getBlockCache(); diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon index ed0926fd321..e04756dd81d 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon @@ -34,7 +34,6 @@ org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo; org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad; org.apache.hadoop.hbase.util.DirectMemoryUtils; org.apache.hadoop.util.StringUtils; -com.yammer.metrics.stats.Snapshot; java.lang.management.ManagementFactory;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java index 24a4e320164..4c1ad23e46c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java @@ -17,58 +17,54 @@ */ package org.apache.hadoop.hbase.io.hfile; +import org.apache.hadoop.hbase.util.FastLongHistogram; import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import com.yammer.metrics.core.Histogram; -import com.yammer.metrics.stats.Snapshot; - /** * Snapshot of block cache age in cache. * This object is preferred because we can control how it is serialized out when JSON'ing. */ @JsonIgnoreProperties({"ageHistogram", "snapshot"}) public class AgeSnapshot { - private final Histogram ageHistogram; - private final Snapshot snapshot; - AgeSnapshot(final Histogram ageHistogram) { + private final FastLongHistogram ageHistogram; + private final long[] quantiles; + + AgeSnapshot(final FastLongHistogram ageHistogram) { this.ageHistogram = ageHistogram; - this.snapshot = ageHistogram.getSnapshot(); + this.quantiles = ageHistogram.getQuantiles(new double[]{0.75, 0.95, 0.98, 0.99, 0.999}); } public double get75thPercentile() { - return snapshot.get75thPercentile(); + return quantiles[0]; } public double get95thPercentile() { - return snapshot.get95thPercentile(); + return quantiles[1]; } public double get98thPercentile() { - return snapshot.get98thPercentile(); - } - - public double get999thPercentile() { - return snapshot.get999thPercentile(); + return quantiles[2]; } public double get99thPercentile() { - return snapshot.get99thPercentile(); + return quantiles[3]; } + public double get999thPercentile() { + return quantiles[4]; + } + + public double getMean() { - return this.ageHistogram.mean(); + return this.ageHistogram.getMean(); } public double getMax() { - return ageHistogram.max(); + return this.ageHistogram.getMax(); } public double getMin() { - return ageHistogram.min(); - } - - public double getStdDev() { - return ageHistogram.stdDev(); + return this.ageHistogram.getMin(); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java index 94638dacf3c..ff67337b32b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java @@ -25,27 +25,19 @@ import java.util.concurrent.ConcurrentSkipListSet; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.FastLongHistogram; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.SerializationConfig; -import com.yammer.metrics.core.Histogram; -import com.yammer.metrics.core.MetricsRegistry; -import com.yammer.metrics.stats.Snapshot; - /** * Utilty for aggregating counts in CachedBlocks and toString/toJSON CachedBlocks and BlockCaches. * No attempt has been made at making this thread safe. */ @InterfaceAudience.Private public class BlockCacheUtil { - /** - * Needed making histograms. - */ - private static final MetricsRegistry METRICS = new MetricsRegistry(); - /** * Needed generating JSON. */ @@ -204,7 +196,7 @@ public class BlockCacheUtil { */ private NavigableMap> cachedBlockByFile = new ConcurrentSkipListMap>(); - Histogram age = METRICS.newHistogram(CachedBlocksByFile.class, "age"); + FastLongHistogram hist = new FastLongHistogram(); /** * @param cb @@ -226,7 +218,7 @@ public class BlockCacheUtil { this.dataSize += cb.getSize(); } long age = this.now - cb.getCachedTime(); - this.age.update(age); + this.hist.add(age, 1); return false; } @@ -269,18 +261,22 @@ public class BlockCacheUtil { } public AgeSnapshot getAgeInCacheSnapshot() { - return new AgeSnapshot(this.age); + return new AgeSnapshot(this.hist); } @Override public String toString() { - Snapshot snapshot = this.age.getSnapshot(); - return "count=" + count + ", dataBlockCount=" + this.dataBlockCount + ", size=" + size + + AgeSnapshot snapshot = getAgeInCacheSnapshot(); + return "count=" + count + ", dataBlockCount=" + dataBlockCount + ", size=" + size + ", dataSize=" + getDataSize() + - ", mean age=" + this.age.mean() + ", stddev age=" + this.age.stdDev() + - ", min age=" + this.age.min() + ", max age=" + this.age.max() + - ", 95th percentile age=" + snapshot.get95thPercentile() + - ", 99th percentile age=" + snapshot.get99thPercentile(); + ", mean age=" + snapshot.getMean() + + ", min age=" + snapshot.getMin() + + ", max age=" + snapshot.getMax() + + ", 75th percentile age=" + snapshot.get75thPercentile() + + ", 95th percentile age=" + snapshot.get95thPercentile() + + ", 98th percentile age=" + snapshot.get98thPercentile() + + ", 99th percentile age=" + snapshot.get99thPercentile() + + ", 99.9th percentile age=" + snapshot.get99thPercentile(); } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java index fff6585aea5..b2a0d09a1e0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java @@ -22,18 +22,14 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import com.yammer.metrics.core.Histogram; -import com.yammer.metrics.core.MetricsRegistry; +import org.apache.hadoop.hbase.util.Counter; +import org.apache.hadoop.hbase.util.FastLongHistogram; /** * Class that implements cache metrics. */ @InterfaceAudience.Private public class CacheStats { - /** - * Needed making histograms. - */ - private static final MetricsRegistry METRICS = new MetricsRegistry(); /** Sliding window statistics. The number of metric periods to include in * sliding window hit ratio calculations. @@ -41,10 +37,10 @@ public class CacheStats { static final int DEFAULT_WINDOW_PERIODS = 5; /** The number of getBlock requests that were cache hits */ - private final AtomicLong hitCount = new AtomicLong(0); + private final Counter hitCount = new Counter(); /** The number of getBlock requests that were cache hits from primary replica */ - private final AtomicLong primaryHitCount = new AtomicLong(0); + private final Counter primaryHitCount = new Counter(); /** * The number of getBlock requests that were cache hits, but only from @@ -52,27 +48,27 @@ public class CacheStats { * attempt to read from the block cache even if they will not put new blocks * into the block cache. See HBASE-2253 for more information. */ - private final AtomicLong hitCachingCount = new AtomicLong(0); + private final Counter hitCachingCount = new Counter(); /** The number of getBlock requests that were cache misses */ - private final AtomicLong missCount = new AtomicLong(0); + private final Counter missCount = new Counter(); /** The number of getBlock requests for primary replica that were cache misses */ - private final AtomicLong primaryMissCount = new AtomicLong(0); + private final Counter primaryMissCount = new Counter(); /** * The number of getBlock requests that were cache misses, but only from * requests that were set to use the block cache. */ - private final AtomicLong missCachingCount = new AtomicLong(0); + private final Counter missCachingCount = new Counter(); /** The number of times an eviction has occurred */ - private final AtomicLong evictionCount = new AtomicLong(0); + private final Counter evictionCount = new Counter(); /** The total number of blocks that have been evicted */ - private final AtomicLong evictedBlockCount = new AtomicLong(0); + private final Counter evictedBlockCount = new Counter(); /** The total number of blocks for primary replica that have been evicted */ - private final AtomicLong primaryEvictedBlockCount = new AtomicLong(0); + private final Counter primaryEvictedBlockCount = new Counter(); /** The total number of blocks that were not inserted. */ private final AtomicLong failedInserts = new AtomicLong(0); @@ -100,7 +96,7 @@ public class CacheStats { /** * Keep running age at eviction time */ - private Histogram ageAtEviction; + private FastLongHistogram ageAtEviction; private long startTime = System.nanoTime(); public CacheStats(final String name) { @@ -113,7 +109,7 @@ public class CacheStats { this.hitCachingCounts = initializeZeros(numPeriodsInWindow); this.requestCounts = initializeZeros(numPeriodsInWindow); this.requestCachingCounts = initializeZeros(numPeriodsInWindow); - this.ageAtEviction = METRICS.newHistogram(CacheStats.class, name + ".ageAtEviction"); + this.ageAtEviction = new FastLongHistogram(); } @Override @@ -125,14 +121,13 @@ public class CacheStats { ", evictedBlockCount=" + getEvictedCount() + ", primaryMissCount=" + getPrimaryMissCount() + ", primaryHitCount=" + getPrimaryHitCount() + - ", evictedAgeMean=" + snapshot.getMean() + - ", evictedAgeStdDev=" + snapshot.getStdDev(); + ", evictedAgeMean=" + snapshot.getMean(); } public void miss(boolean caching, boolean primary) { - missCount.incrementAndGet(); - if (primary) primaryMissCount.incrementAndGet(); - if (caching) missCachingCount.incrementAndGet(); + missCount.increment(); + if (primary) primaryMissCount.increment(); + if (caching) missCachingCount.increment(); } public void hit(boolean caching) { @@ -140,20 +135,20 @@ public class CacheStats { } public void hit(boolean caching, boolean primary) { - hitCount.incrementAndGet(); - if (primary) primaryHitCount.incrementAndGet(); - if (caching) hitCachingCount.incrementAndGet(); + hitCount.increment(); + if (primary) primaryHitCount.increment(); + if (caching) hitCachingCount.increment(); } public void evict() { - evictionCount.incrementAndGet(); + evictionCount.increment(); } public void evicted(final long t, boolean primary) { - if (t > this.startTime) this.ageAtEviction.update(t - this.startTime); - this.evictedBlockCount.incrementAndGet(); + if (t > this.startTime) this.ageAtEviction.add(t - this.startTime,1); + this.evictedBlockCount.increment(); if (primary) { - primaryEvictedBlockCount.incrementAndGet(); + primaryEvictedBlockCount.increment(); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index a67bf8c06c3..3fb9554112e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -35,7 +35,6 @@ import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -62,6 +61,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HFileProtos; import org.apache.hadoop.hbase.util.BloomFilterWriter; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Counter; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.io.Writable; @@ -179,17 +179,19 @@ public class HFile { */ public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024; // For measuring number of checksum failures - static final AtomicLong checksumFailures = new AtomicLong(); + static final Counter checksumFailures = new Counter(); // for test purpose - public static final AtomicLong dataBlockReadCnt = new AtomicLong(0); + public static final Counter dataBlockReadCnt = new Counter(); /** * Number of checksum verification failures. It also * clears the counter. */ public static final long getChecksumFailuresCount() { - return checksumFailures.getAndSet(0); + long count = checksumFailures.get(); + checksumFailures.set(0); + return count; } /** API required to write an {@link HFile} */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java index edccfb59dbe..6cd7b20a1dc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java @@ -1523,7 +1523,7 @@ public class HFileBlock implements Cacheable { HFile.LOG.warn(msg); throw new IOException(msg); // cannot happen case here } - HFile.checksumFailures.incrementAndGet(); // update metrics + HFile.checksumFailures.increment(); // update metrics // If we have a checksum failure, we fall back into a mode where // the next few reads use HDFS level checksums. We aim to make the diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java index a1b4c340d69..50200dc123b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java @@ -417,7 +417,7 @@ public class HFileReaderV2 extends AbstractHFileReader { assert cachedBlock.isUnpacked() : "Packed block leak."; if (cachedBlock.getBlockType().isData()) { if (updateCacheMetrics) { - HFile.dataBlockReadCnt.incrementAndGet(); + HFile.dataBlockReadCnt.increment(); } // Validate encoding type for data blocks. We include encoding // type in the cache key, and we expect it to match on a cache hit. @@ -456,7 +456,7 @@ public class HFileReaderV2 extends AbstractHFileReader { } if (updateCacheMetrics && hfileBlock.getBlockType().isData()) { - HFile.dataBlockReadCnt.incrementAndGet(); + HFile.dataBlockReadCnt.increment(); } return unpacked; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java index 51e6268fc93..ec1d4d9f683 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java @@ -18,10 +18,9 @@ */ package org.apache.hadoop.hbase.io.hfile.bucket; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.hfile.CacheStats; +import org.apache.hadoop.hbase.util.Counter; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; /** @@ -29,8 +28,8 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; */ @InterfaceAudience.Private public class BucketCacheStats extends CacheStats { - private final AtomicLong ioHitCount = new AtomicLong(0); - private final AtomicLong ioHitTime = new AtomicLong(0); + private final Counter ioHitCount = new Counter(0); + private final Counter ioHitTime = new Counter(0); private final static int nanoTime = 1000000; private long lastLogTime = EnvironmentEdgeManager.currentTime(); @@ -45,8 +44,8 @@ public class BucketCacheStats extends CacheStats { } public void ioHit(long time) { - ioHitCount.incrementAndGet(); - ioHitTime.addAndGet(time); + ioHitCount.increment(); + ioHitTime.add(time); } public long getIOHitsPerSecond() {