Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-9653 Add compaction metrics to trunk 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1526314 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
eclark 2013-09-25 21:44:33 +00:00
parent e0768e0814
commit 22b77dc791
16 changed files with 158 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
View File

@ -155,6 +155,8 @@ public interface MetricsRegionServerSource extends BaseSource {
String PERCENT_FILES_LOCAL_DESC =
"The percent of HFiles that are stored on the local hdfs data node.";
String COMPACTION_QUEUE_LENGTH = "compactionQueueLength";
String LARGE_COMPACTION_QUEUE_LENGTH = "largeCompactionQueueLength";
String SMALL_COMPACTION_QUEUE_LENGTH = "smallCompactionQueueLength";
String COMPACTION_QUEUE_LENGTH_DESC = "Length of the queue for compactions.";
String FLUSH_QUEUE_LENGTH = "flushQueueLength";
String FLUSH_QUEUE_LENGTH_DESC = "Length of the queue for region flushes";

4
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
View File

@ -148,6 +148,10 @@ public interface MetricsRegionServerWrapper {
*/
int getCompactionQueueSize();
int getSmallCompactionQueueSize();
int getLargeCompactionQueueSize();
/**
* Get the size of the flush queue.
*/

8
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
View File

@ -27,6 +27,14 @@ public interface MetricsRegionSource extends Comparable<MetricsRegionSource> {
String OPS_SAMPLE_NAME = "ops";
String SIZE_VALUE_NAME = "size";
String COMPACTIONS_COMPLETED_COUNT = "compactionsCompletedCount";
String NUM_BYTES_COMPACTED_COUNT = "numBytesCompactedCount";
String NUM_FILES_COMPACTED_COUNT = "numFilesCompactedCount";
String COMPACTIONS_COMPLETED_DESC = "Number of compactions that have completed.";
String NUM_BYTES_COMPACTED_DESC =
"Sum of filesize on all files entering a finished, successful or aborted, compaction";
String NUM_FILES_COMPACTED_DESC =
"Number of files that were input for finished, successful or aborted, compactions";
/**
* Close the region's metrics as this region is closing.

5
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
View File

@ -74,4 +74,9 @@ public interface MetricsRegionWrapper {
*/
long getWriteRequestCount();
long getNumFilesCompacted();
long getNumBytesCompacted();
long getNumCompactionsCompleted();
}

6
hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
View File

@ -174,6 +174,12 @@ public class MetricsRegionServerSourceImpl
.addGauge(COMPACTION_QUEUE_LENGTH,
COMPACTION_QUEUE_LENGTH_DESC,
rsWrap.getCompactionQueueSize())
.addGauge(LARGE_COMPACTION_QUEUE_LENGTH,
COMPACTION_QUEUE_LENGTH_DESC,
rsWrap.getLargeCompactionQueueSize())
.addGauge(SMALL_COMPACTION_QUEUE_LENGTH,
COMPACTION_QUEUE_LENGTH_DESC,
rsWrap.getSmallCompactionQueueSize())
.addGauge(FLUSH_QUEUE_LENGTH, FLUSH_QUEUE_LENGTH_DESC, rsWrap.getFlushQueueSize())
.addGauge(BLOCK_CACHE_FREE_SIZE, BLOCK_CACHE_FREE_DESC, rsWrap.getBlockCacheFreeSize())
.addGauge(BLOCK_CACHE_COUNT, BLOCK_CACHE_COUNT_DESC, rsWrap.getBlockCacheCount())

10
hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
View File

@ -179,6 +179,16 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
mrb.addCounter(regionNamePrefix + MetricsRegionServerSource.WRITE_REQUEST_COUNT,
MetricsRegionServerSource.WRITE_REQUEST_COUNT_DESC,
this.regionWrapper.getWriteRequestCount());
mrb.addCounter(regionNamePrefix + MetricsRegionSource.COMPACTIONS_COMPLETED_COUNT,
MetricsRegionSource.COMPACTIONS_COMPLETED_DESC,
this.regionWrapper.getNumCompactionsCompleted());
mrb.addCounter(regionNamePrefix + MetricsRegionSource.NUM_BYTES_COMPACTED_COUNT,
MetricsRegionSource.NUM_BYTES_COMPACTED_DESC,
this.regionWrapper.getNumBytesCompacted());
mrb.addCounter(regionNamePrefix + MetricsRegionSource.NUM_FILES_COMPACTED_COUNT,
MetricsRegionSource.NUM_FILES_COMPACTED_DESC,
this.regionWrapper.getNumFilesCompacted());
}
}

15
hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
View File

@ -102,5 +102,20 @@ public class TestMetricsRegionSourceImpl {
public long getWriteRequestCount() {
return 0;
}
@Override
public long getNumFilesCompacted() {
return 0;
}
@Override
public long getNumBytesCompacted() {
return 0;
}
@Override
public long getNumCompactionsCompleted() {
return 0;
}
}
}

10
hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
View File

@ -180,5 +180,15 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE,
MetricsRegionServerSource.STOREFILE_SIZE_DESC),
this.regionWrapper.getStoreFileSize());
mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.COMPACTIONS_COMPLETED_COUNT,
MetricsRegionSource.COMPACTIONS_COMPLETED_DESC),
this.regionWrapper.getNumCompactionsCompleted());
mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.NUM_BYTES_COMPACTED_COUNT,
MetricsRegionSource.NUM_BYTES_COMPACTED_DESC),
this.regionWrapper.getNumBytesCompacted());
mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.NUM_FILES_COMPACTED_COUNT,
MetricsRegionSource.NUM_FILES_COMPACTED_DESC),
this.regionWrapper.getNumFilesCompacted());
}
}

15
hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
View File

@ -102,5 +102,20 @@ public class TestMetricsRegionSourceImpl {
public long getWriteRequestCount() {
return 0;
}
@Override
public long getNumFilesCompacted() {
return 0;
}
@Override
public long getNumBytesCompacted() {
return 0;
}
@Override
public long getNumCompactionsCompleted() {
return 0;
}
}
}

10
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
View File

@ -381,6 +381,16 @@ public class CompactSplitThread implements CompactionRequestor {
return largeCompactions.getQueue().size() + smallCompactions.getQueue().size();
}
public int getLargeCompactionQueueSize() {
return largeCompactions.getQueue().size();
}
public int getSmallCompactionQueueSize() {
return smallCompactions.getQueue().size();
}
private boolean shouldSplitRegion() {
return (regionSplitLimit > server.getNumberOfOnlineRegions());
}

17
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -250,8 +250,11 @@ public class HRegion implements HeapSize { // , Writable{
final Counter readRequestsCount = new Counter();
final Counter writeRequestsCount = new Counter();
//How long operations were blocked by a memstore over highwater.
final Counter updatesBlockedMs = new Counter();
// Compaction counters
final AtomicLong compactionsFinished = new AtomicLong(0L);
final AtomicLong compactionNumFilesCompacted = new AtomicLong(0L);
final AtomicLong compactionNumBytesCompacted = new AtomicLong(0L);
private final HLog log;
private final HRegionFileSystem fs;
@ -4927,7 +4930,7 @@ public class HRegion implements HeapSize { // , Writable{
public static final long FIXED_OVERHEAD = ClassSize.align(
ClassSize.OBJECT +
ClassSize.ARRAY +
38 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
40 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT +
(11 * Bytes.SIZEOF_LONG) +
4 * Bytes.SIZEOF_BOOLEAN);
@ -5496,8 +5499,14 @@ public class HRegion implements HeapSize { // , Writable{
(isMajor ? majorInProgress : minorInProgress).incrementAndGet();
}
public void reportCompactionRequestEnd(boolean isMajor){
public void reportCompactionRequestEnd(boolean isMajor, int numFiles, long filesSizeCompacted){
int newValue = (isMajor ? majorInProgress : minorInProgress).decrementAndGet();
// metrics
compactionsFinished.incrementAndGet();
compactionNumFilesCompacted.addAndGet(numFiles);
compactionNumBytesCompacted.addAndGet(filesSizeCompacted);
assert newValue >= 0;
}

3
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
View File

@ -985,6 +985,7 @@ public class HStore implements Store {
try {
// Commence the compaction.
List<Path> newFiles = compaction.compact();
// TODO: get rid of this!
if (!this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
LOG.warn("hbase.hstore.compaction.complete is set to false");
@ -1333,7 +1334,7 @@ public class HStore implements Store {
}
private void finishCompactionRequest(CompactionRequest cr) {
this.region.reportCompactionRequestEnd(cr.isMajor());
this.region.reportCompactionRequestEnd(cr.isMajor(), cr.getFiles().size(), cr.getSize());
if (cr.isOffPeak()) {
offPeakCompactionTracker.set(false);
cr.setOffPeak(false);

18
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
View File

@ -155,6 +155,24 @@ class MetricsRegionServerWrapperImpl
return this.regionServer.compactSplitThread.getCompactionQueueSize();
}
@Override
public int getSmallCompactionQueueSize() {
//The thread could be zero. if so assume there is no queue.
if (this.regionServer.compactSplitThread == null) {
return 0;
}
return this.regionServer.compactSplitThread.getSmallCompactionQueueSize();
}
@Override
public int getLargeCompactionQueueSize() {
//The thread could be zero. if so assume there is no queue.
if (this.regionServer.compactSplitThread == null) {
return 0;
}
return this.regionServer.compactSplitThread.getLargeCompactionQueueSize();
}
@Override
public int getFlushQueueSize() {
//If there is no flusher there should be no queue.

15
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
View File

@ -114,6 +114,21 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
return this.region.getWriteRequestsCount();
}
@Override
public long getNumFilesCompacted() {
return this.region.compactionNumFilesCompacted.get();
}
@Override
public long getNumBytesCompacted() {
return this.region.compactionNumBytesCompacted.get();
}
@Override
public long getNumCompactionsCompleted() {
return this.region.compactionsFinished.get();
}
public class HRegionMetricsWrapperRunnable implements Runnable {
@Override

10
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
View File

@ -135,6 +135,16 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
return 411;
}
@Override
public int getSmallCompactionQueueSize() {
return 0;
}
@Override
public int getLargeCompactionQueueSize() {
return 0;
}
@Override
public int getFlushQueueSize() {
return 412;

15
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
View File

@ -64,4 +64,19 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper {
public long getWriteRequestCount() {
return 106;
}
@Override
public long getNumFilesCompacted() {
return 0;
}
@Override
public long getNumBytesCompacted() {
return 0;
}
@Override
public long getNumCompactionsCompleted() {
return 0;
}
}