From 7dc5908e6bcf37240bfd2c95fc1832999b03c476 Mon Sep 17 00:00:00 2001 From: eclark Date: Thu, 29 Nov 2012 01:12:55 +0000 Subject: [PATCH] HBASE-7220 Creating a table with 3000 regions on 2 nodes fails after 1 hour git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1415016 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/metrics2/impl/JmxCacheBuster.java | 39 +++++++++++++---- .../hadoop/metrics2/impl/JmxCacheBuster.java | 42 ++++++++++++++----- .../hadoop/hbase/regionserver/HRegion.java | 31 +++++++++----- 3 files changed, 84 insertions(+), 28 deletions(-) diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java index 1e2eb2f4fa6..741449c25ac 100644 --- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java @@ -20,7 +20,13 @@ package org.apache.hadoop.metrics2.impl; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.metrics2.MetricsExecutor; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricsExecutorImpl; + + +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; /** * JMX caches the beans that have been exported; even after the values are removed from hadoop's @@ -32,21 +38,38 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; */ public class JmxCacheBuster { private static final Log LOG = LogFactory.getLog(JmxCacheBuster.class); + private static Object lock = new Object(); + private static ScheduledFuture fut = null; + private static MetricsExecutor executor = new MetricsExecutorImpl(); /** * For JMX to forget about all previously exported metrics. */ public static void clearJmxCache() { - LOG.trace("Clearing JMX mbean cache."); - // This is pretty extreme but it's the best way that - // I could find to get metrics to be removed. + //If there are more then 100 ms before the executor will run then everything should be merged. + if (fut == null || (!fut.isDone() && fut.getDelay(TimeUnit.MILLISECONDS) > 100)) return; - try { - DefaultMetricsSystem.INSTANCE.stop(); - DefaultMetricsSystem.INSTANCE.start(); - } catch (Exception exception ) { - LOG.debug("error clearing the jmx it appears the metrics system hasn't been started", exception); + synchronized (lock) { + fut = executor.getExecutor().schedule(new JmxCacheBusterRunnable(), 5, TimeUnit.SECONDS); + } + } + + static class JmxCacheBusterRunnable implements Runnable { + + @Override + public void run() { + LOG.trace("Clearing JMX mbean cache."); + + // This is pretty extreme but it's the best way that + // I could find to get metrics to be removed. + + try { + DefaultMetricsSystem.INSTANCE.stop(); + DefaultMetricsSystem.INSTANCE.start(); + } catch (Exception exception ) { + LOG.debug("error clearing the jmx it appears the metrics system hasn't been started", exception); + } } } } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java index ce5b9e2eec7..9fa9fa1096d 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java @@ -20,7 +20,12 @@ package org.apache.hadoop.metrics2.impl; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.metrics2.MetricsExecutor; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricsExecutorImpl; + +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; /** * JMX caches the beans that have been exported; even after the values are removed from hadoop's @@ -32,23 +37,40 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; */ public class JmxCacheBuster { private static final Log LOG = LogFactory.getLog(JmxCacheBuster.class); + private static Object lock = new Object(); + private static ScheduledFuture fut = null; + private static MetricsExecutor executor = new MetricsExecutorImpl(); /** * For JMX to forget about all previously exported metrics. */ public static void clearJmxCache() { - LOG.trace("Clearing JMX mbean cache."); - // This is pretty extreme but it's the best way that - // I could find to get metrics to be removed. - try { - if (DefaultMetricsSystem.instance() != null ) { - DefaultMetricsSystem.instance().stop(); - DefaultMetricsSystem.instance().start(); + //If there are more then 100 ms before the executor will run then everything should be merged. + if (fut == null || (!fut.isDone() && fut.getDelay(TimeUnit.MILLISECONDS) > 100)) return; + + synchronized (lock) { + fut = executor.getExecutor().schedule(new JmxCacheBusterRunnable(), 5, TimeUnit.SECONDS); + } + } + + static class JmxCacheBusterRunnable implements Runnable { + + @Override + public void run() { + LOG.trace("Clearing JMX mbean cache."); + + // This is pretty extreme but it's the best way that + // I could find to get metrics to be removed. + try { + if (DefaultMetricsSystem.instance() != null ) { + DefaultMetricsSystem.instance().stop(); + DefaultMetricsSystem.instance().start(); + } + + } catch (Exception exception ) { + LOG.debug("error clearing the jmx it appears the metrics system hasn't been started", exception); } - - } catch (Exception exception ) { - LOG.debug("error clearing the jmx it appears the metrics system hasn't been started", exception); } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index c01a1a118c8..a4a292862ec 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -451,8 +451,6 @@ public class HRegion implements HeapSize { // , Writable{ this.regiondir = getRegionDir(this.tableDir, encodedNameStr); this.scannerReadPoints = new ConcurrentHashMap(); - this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this)); - /* * timestamp.slop provides a server-side constraint on the timestamp. This * assumes that you base your TS around currentTimeMillis(). In this case, @@ -475,6 +473,9 @@ public class HRegion implements HeapSize { // , Writable{ // don't initialize coprocessors if not running within a regionserver // TODO: revisit if coprocessors should load in other cases this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf); + this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this)); + } else { + this.metricsRegion = null; } if (LOG.isDebugEnabled()) { // Write out region name as string and its encoded name. @@ -1024,7 +1025,9 @@ public class HRegion implements HeapSize { // , Writable{ status.setStatus("Running coprocessor post-close hooks"); this.coprocessorHost.postClose(abort); } - this.metricsRegion.close(); + if ( this.metricsRegion != null) { + this.metricsRegion.close(); + } status.markComplete("Closed"); LOG.info("Closed " + this); return result; @@ -2331,11 +2334,15 @@ public class HRegion implements HeapSize { // , Writable{ if (noOfPuts > 0) { // There were some Puts in the batch. double noOfMutations = noOfPuts + noOfDeletes; - this.metricsRegion.updatePut(); + if (this.metricsRegion != null) { + this.metricsRegion.updatePut(); + } } if (noOfDeletes > 0) { // There were some Deletes in the batch. - this.metricsRegion.updateDelete(); + if (this.metricsRegion != null) { + this.metricsRegion.updateDelete(); + } } if (!success) { for (int i = firstIndex; i < lastIndexExclusive; i++) { @@ -4269,8 +4276,9 @@ public class HRegion implements HeapSize { // , Writable{ } // do after lock - - this.metricsRegion.updateGet(); + if (this.metricsRegion != null) { + this.metricsRegion.updateGet(); + } return results; } @@ -4657,8 +4665,9 @@ public class HRegion implements HeapSize { // , Writable{ closeRegionOperation(); } - this.metricsRegion.updateAppend(); - + if (this.metricsRegion != null) { + this.metricsRegion.updateAppend(); + } if (flush) { // Request a cache flush. Do it outside update lock. @@ -4795,7 +4804,9 @@ public class HRegion implements HeapSize { // , Writable{ mvcc.completeMemstoreInsert(w); } closeRegionOperation(); - this.metricsRegion.updateIncrement(); + if (this.metricsRegion != null) { + this.metricsRegion.updateIncrement(); + } } if (flush) {