Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-7220 Creating a table with 3000 regions on 2 nodes fails after 1 hour 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1415016 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
eclark 2012-11-29 01:12:55 +00:00
parent eded2c4881
commit 7dc5908e6b
3 changed files with 84 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java
View File

@ -20,7 +20,13 @@ package org.apache.hadoop.metrics2.impl;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.metrics2.MetricsExecutor;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MetricsExecutorImpl;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
/**
* JMX caches the beans that have been exported; even after the values are removed from hadoop's
@ -32,21 +38,38 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
*/
public class JmxCacheBuster {
private static final Log LOG = LogFactory.getLog(JmxCacheBuster.class);
private static Object lock = new Object();
private static ScheduledFuture fut = null;
private static MetricsExecutor executor = new MetricsExecutorImpl();
/**
* For JMX to forget about all previously exported metrics.
*/
public static void clearJmxCache() {
LOG.trace("Clearing JMX mbean cache.");
// This is pretty extreme but it's the best way that
// I could find to get metrics to be removed.
//If there are more then 100 ms before the executor will run then everything should be merged.
if (fut == null || (!fut.isDone() && fut.getDelay(TimeUnit.MILLISECONDS) > 100)) return;
try {
DefaultMetricsSystem.INSTANCE.stop();
DefaultMetricsSystem.INSTANCE.start();
} catch (Exception exception ) {
LOG.debug("error clearing the jmx it appears the metrics system hasn't been started", exception);
synchronized (lock) {
fut = executor.getExecutor().schedule(new JmxCacheBusterRunnable(), 5, TimeUnit.SECONDS);
}
}
static class JmxCacheBusterRunnable implements Runnable {
@Override
public void run() {
LOG.trace("Clearing JMX mbean cache.");
// This is pretty extreme but it's the best way that
// I could find to get metrics to be removed.
try {
DefaultMetricsSystem.INSTANCE.stop();
DefaultMetricsSystem.INSTANCE.start();
} catch (Exception exception ) {
LOG.debug("error clearing the jmx it appears the metrics system hasn't been started", exception);
}
}
}
}

42
hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java
View File

@ -20,7 +20,12 @@ package org.apache.hadoop.metrics2.impl;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.metrics2.MetricsExecutor;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.MetricsExecutorImpl;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
/**
* JMX caches the beans that have been exported; even after the values are removed from hadoop's
@ -32,23 +37,40 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
*/
public class JmxCacheBuster {
private static final Log LOG = LogFactory.getLog(JmxCacheBuster.class);
private static Object lock = new Object();
private static ScheduledFuture fut = null;
private static MetricsExecutor executor = new MetricsExecutorImpl();
/**
* For JMX to forget about all previously exported metrics.
*/
public static void clearJmxCache() {
LOG.trace("Clearing JMX mbean cache.");
// This is pretty extreme but it's the best way that
// I could find to get metrics to be removed.
try {
if (DefaultMetricsSystem.instance() != null ) {
DefaultMetricsSystem.instance().stop();
DefaultMetricsSystem.instance().start();
//If there are more then 100 ms before the executor will run then everything should be merged.
if (fut == null || (!fut.isDone() && fut.getDelay(TimeUnit.MILLISECONDS) > 100)) return;
synchronized (lock) {
fut = executor.getExecutor().schedule(new JmxCacheBusterRunnable(), 5, TimeUnit.SECONDS);
}
}
static class JmxCacheBusterRunnable implements Runnable {
@Override
public void run() {
LOG.trace("Clearing JMX mbean cache.");
// This is pretty extreme but it's the best way that
// I could find to get metrics to be removed.
try {
if (DefaultMetricsSystem.instance() != null ) {
DefaultMetricsSystem.instance().stop();
DefaultMetricsSystem.instance().start();
}
} catch (Exception exception ) {
LOG.debug("error clearing the jmx it appears the metrics system hasn't been started", exception);
}
} catch (Exception exception ) {
LOG.debug("error clearing the jmx it appears the metrics system hasn't been started", exception);
}
}
}

31
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -451,8 +451,6 @@ public class HRegion implements HeapSize { // , Writable{
this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
this.scannerReadPoints = new ConcurrentHashMap<RegionScanner, Long>();
this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
/*
* timestamp.slop provides a server-side constraint on the timestamp. This
* assumes that you base your TS around currentTimeMillis(). In this case,
@ -475,6 +473,9 @@ public class HRegion implements HeapSize { // , Writable{
// don't initialize coprocessors if not running within a regionserver
// TODO: revisit if coprocessors should load in other cases
this.coprocessorHost = new RegionCoprocessorHost(this, rsServices, conf);
this.metricsRegion = new MetricsRegion(new MetricsRegionWrapperImpl(this));
} else {
this.metricsRegion = null;
}
if (LOG.isDebugEnabled()) {
// Write out region name as string and its encoded name.
@ -1024,7 +1025,9 @@ public class HRegion implements HeapSize { // , Writable{
status.setStatus("Running coprocessor post-close hooks");
this.coprocessorHost.postClose(abort);
}
this.metricsRegion.close();
if ( this.metricsRegion != null) {
this.metricsRegion.close();
}
status.markComplete("Closed");
LOG.info("Closed " + this);
return result;
@ -2331,11 +2334,15 @@ public class HRegion implements HeapSize { // , Writable{
if (noOfPuts > 0) {
// There were some Puts in the batch.
double noOfMutations = noOfPuts + noOfDeletes;
this.metricsRegion.updatePut();
if (this.metricsRegion != null) {
this.metricsRegion.updatePut();
}
}
if (noOfDeletes > 0) {
// There were some Deletes in the batch.
this.metricsRegion.updateDelete();
if (this.metricsRegion != null) {
this.metricsRegion.updateDelete();
}
}
if (!success) {
for (int i = firstIndex; i < lastIndexExclusive; i++) {
@ -4269,8 +4276,9 @@ public class HRegion implements HeapSize { // , Writable{
}
// do after lock
this.metricsRegion.updateGet();
if (this.metricsRegion != null) {
this.metricsRegion.updateGet();
}
return results;
}
@ -4657,8 +4665,9 @@ public class HRegion implements HeapSize { // , Writable{
closeRegionOperation();
}
this.metricsRegion.updateAppend();
if (this.metricsRegion != null) {
this.metricsRegion.updateAppend();
}
if (flush) {
// Request a cache flush. Do it outside update lock.
@ -4795,7 +4804,9 @@ public class HRegion implements HeapSize { // , Writable{
mvcc.completeMemstoreInsert(w);
}
closeRegionOperation();
this.metricsRegion.updateIncrement();
if (this.metricsRegion != null) {
this.metricsRegion.updateIncrement();
}
}
if (flush) {