Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-10116 SlabCache metrics improvements 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1549935 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
ndimiduk 2013-12-10 18:58:47 +00:00
parent 63aff6d35c
commit a604c4224a
9 changed files with 75 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
View File

@ -31,6 +31,7 @@ org.apache.hadoop.hbase.protobuf.ProtobufUtil;
org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad;
org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
org.apache.hadoop.hbase.util.DirectMemoryUtils;
org.apache.hadoop.util.StringUtils;
com.yammer.metrics.stats.Snapshot;
java.lang.management.ManagementFactory;
@ -100,6 +101,8 @@ MetricsRegionServerWrapper mWrap;
<tr>
<th>Used Heap</th>
<th>Max Heap</th>
<th>Direct Memory Used</th>
<th>Direct Memory Configured</th>
<th>Memstore Size</th>
</tr>
</tr>
@ -110,6 +113,12 @@ MetricsRegionServerWrapper mWrap;
<td>
<% StringUtils.humanReadableInt(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()) %>
</td>
<td>
<% StringUtils.humanReadableInt(DirectMemoryUtils.getDirectMemoryUsage()) %>
</td>
<td>
<% StringUtils.humanReadableInt(DirectMemoryUtils.getDirectMemorySize()) %>
</td>
<td><% StringUtils.humanReadableInt(mWrap.getMemstoreSize()) %></td>
</tr>
</table>

4
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
View File

@ -100,8 +100,8 @@ public interface BlockCache {
long getCurrentSize();
/**
* Returns the number of evictions that have occurred.
* @return number of evictions
* Returns the number of blocks that have been evicted.
* @return number of evicted blocks
*/
long getEvictedCount();

6
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
View File

@ -55,7 +55,6 @@ public class CombinedBlockCache implements BlockCache, HeapSize {
return lruCache.heapSize() + bucketCache.heapSize();
}
@Override
public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
boolean isMetaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA;
@ -66,7 +65,6 @@ public class CombinedBlockCache implements BlockCache, HeapSize {
}
}
@Override
public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
cacheBlock(cacheKey, buf, false);
@ -79,7 +77,6 @@ public class CombinedBlockCache implements BlockCache, HeapSize {
return lruCache.getBlock(cacheKey, caching, repeat);
}
return bucketCache.getBlock(cacheKey, caching, repeat);
}
@Override
@ -102,7 +99,6 @@ public class CombinedBlockCache implements BlockCache, HeapSize {
public void shutdown() {
lruCache.shutdown();
bucketCache.shutdown();
}
@Override
@ -209,7 +205,5 @@ public class CombinedBlockCache implements BlockCache, HeapSize {
.getSumRequestCachingCountsPastNPeriods()));
return Double.isNaN(ratio) ? 0 : ratio;
}
}
}

20
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
View File

@ -603,26 +603,17 @@ public class LruBlockCache implements BlockCache, HeapSize {
return this.maxSize;
}
/**
* Get the current size of this cache.
* @return current size in bytes
*/
@Override
public long getCurrentSize() {
return this.size.get();
}
/**
* Get the current size of this cache.
* @return current size in bytes
*/
@Override
public long getFreeSize() {
return getMaxSize() - getCurrentSize();
}
/**
* Get the size of this cache (number of cached blocks)
* @return number of cached blocks
*/
@Override
public long size() {
return this.elements.get();
}
@ -639,10 +630,7 @@ public class LruBlockCache implements BlockCache, HeapSize {
return this.stats.getEvictionCount();
}
/**
* Get the number of blocks that have been evicted during the lifetime
* of this cache.
*/
@Override
public long getEvictedCount() {
return this.stats.getEvictedCount();
}

13
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
View File

@ -893,14 +893,11 @@ public class BucketCache implements BlockCache, HeapSize {
return this.bucketAllocator;
}
@Override
public long heapSize() {
return this.heapSize.get();
}
/**
* Returns the total size of the block cache, in bytes.
* @return size of cache, in bytes
*/
@Override
public long size() {
return this.realCacheSize.get();
@ -916,10 +913,6 @@ public class BucketCache implements BlockCache, HeapSize {
return this.blockNumber.get();
}
/**
* Returns the occupied size of the block cache, in bytes.
* @return occupied space in cache, in bytes
*/
@Override
public long getCurrentSize() {
return this.bucketAllocator.getUsedSize();
@ -931,10 +924,10 @@ public class BucketCache implements BlockCache, HeapSize {
}
/**
* Evicts all blocks for a specific HFile.
* Evicts all blocks for a specific HFile.
* <p>
* This is used for evict-on-close to remove all blocks of a specific HFile.
*
*
* @return the number of blocks evicted
*/
@Override

7
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java
View File

@ -91,8 +91,7 @@ public class SingleSizeCache implements BlockCache, HeapSize {
this.timeSinceLastAccess = new AtomicLong();
// This evictionListener is called whenever the cache automatically
// evicts
// something.
// evicts something.
RemovalListener<BlockCacheKey, CacheablePair> listener =
new RemovalListener<BlockCacheKey, CacheablePair>() {
@Override
@ -116,8 +115,6 @@ public class SingleSizeCache implements BlockCache, HeapSize {
.removalListener(listener)
.<BlockCacheKey, CacheablePair>build()
.asMap();
}
@Override
@ -138,7 +135,6 @@ public class SingleSizeCache implements BlockCache, HeapSize {
synchronized (this) {
CacheablePair alreadyCached = backingMap.putIfAbsent(blockName, newEntry);
if (alreadyCached != null) {
backingStore.free(storedBlock);
@ -193,7 +189,6 @@ public class SingleSizeCache implements BlockCache, HeapSize {
doEviction(key, evictedBlock);
}
return evictedBlock != null;
}
private void doEviction(BlockCacheKey key, CacheablePair evictedBlock) {

12
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java
View File

@ -179,7 +179,7 @@ public class SlabCache implements SlabItemActionWatcher, BlockCache, HeapSize {
private void addSlab(int blockSize, int numBlocks) {
LOG.info("Creating a slab of blockSize " + blockSize + " with " + numBlocks
+ " blocks.");
+ " blocks, " + StringUtils.humanReadableInt(blockSize * numBlocks) + "bytes.");
sizer.put(blockSize, new SingleSizeCache(blockSize, numBlocks, this));
}
@ -229,8 +229,6 @@ public class SlabCache implements SlabItemActionWatcher, BlockCache, HeapSize {
/**
* Get the buffer of the block with the specified name.
* @param caching
* @param key
*
* @return buffer of specified block name, or null if not in cache
*/
@ -301,13 +299,17 @@ public class SlabCache implements SlabItemActionWatcher, BlockCache, HeapSize {
}
public long getFreeSize() {
return 0; // this cache, by default, allocates all its space.
long childFreeSize = 0;
for (SingleSizeCache s : sizer.values()) {
childFreeSize += s.getFreeSize();
}
return childFreeSize;
}
@Override
public long getBlockCount() {
long count = 0;
for (SingleSizeCache cache : backingStore.values()) {
for (SingleSizeCache cache : sizer.values()) {
count += cache.getBlockCount();
}
return count;

2
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
View File

@ -198,7 +198,7 @@ class MetricsRegionServerWrapperImpl
if (this.blockCache == null) {
return 0;
}
return this.blockCache.size();
return this.blockCache.getBlockCount();
}
@Override

53
hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java
View File

@ -26,22 +26,53 @@ import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import com.google.common.base.Preconditions;
import javax.management.JMException;
import javax.management.MBeanServer;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
/**
* Utilities for interacting with and monitoring DirectByteBuffer allocations.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class DirectMemoryUtils {
private static final Log LOG = LogFactory.getLog(DirectMemoryUtils.class);
private static final MBeanServer beanServer;
private static final ObjectName nioDirectPool;
static {
// initialize singletons. Only maintain a reference to the MBeanServer if
// we're able to consume it -- hence convoluted logic.
ObjectName n = null;
MBeanServer s = null;
try {
n = new ObjectName("java.nio:type=BufferPool,name=direct");
} catch (MalformedObjectNameException e) {
LOG.warn("Unable to initialize ObjectName for DirectByteBuffer allocations.");
} finally {
nioDirectPool = n;
}
if (nioDirectPool != null) {
s = ManagementFactory.getPlatformMBeanServer();
}
beanServer = s;
}
/**
* @return the setting of -XX:MaxDirectMemorySize as a long. Returns 0 if
* -XX:MaxDirectMemorySize is not set.
*/
public static long getDirectMemorySize() {
RuntimeMXBean RuntimemxBean = ManagementFactory.getRuntimeMXBean();
List<String> arguments = RuntimemxBean.getInputArguments();
RuntimeMXBean runtimemxBean = ManagementFactory.getRuntimeMXBean();
List<String> arguments = runtimemxBean.getInputArguments();
long multiplier = 1; //for the byte case.
for (String s : arguments) {
if (s.contains("-XX:MaxDirectMemorySize=")) {
@ -64,11 +95,24 @@ public class DirectMemoryUtils {
long retValue = Long.parseLong(memSize);
return retValue * multiplier;
}
}
return 0;
}
/**
* @return the current amount of direct memory used.
*/
public static long getDirectMemoryUsage() {
if (beanServer == null || nioDirectPool == null) return 0;
try {
Long value = (Long) beanServer.getAttribute(nioDirectPool, "MemoryUsed");
return value == null ? 0 : value;
} catch (JMException e) {
LOG.debug("Failed to retrieve nio.BufferPool direct MemoryUsed");
return 0;
}
}
/**
* DirectByteBuffers are garbage collected by using a phantom reference and a
* reference queue. Every once a while, the JVM checks the reference queue and
@ -94,6 +138,5 @@ public class DirectMemoryUtils {
Method cleanMethod = cleaner.getClass().getMethod("clean");
cleanMethod.setAccessible(true);
cleanMethod.invoke(cleaner);
}
}