diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon index 3e60ee8ba78..bf33d464449 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon @@ -31,6 +31,7 @@ org.apache.hadoop.hbase.protobuf.ProtobufUtil; org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo; org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionLoad; org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram; +org.apache.hadoop.hbase.util.DirectMemoryUtils; org.apache.hadoop.util.StringUtils; com.yammer.metrics.stats.Snapshot; java.lang.management.ManagementFactory; @@ -100,6 +101,8 @@ MetricsRegionServerWrapper mWrap; Used Heap Max Heap + Direct Memory Used + Direct Memory Configured Memstore Size @@ -110,6 +113,12 @@ MetricsRegionServerWrapper mWrap; <% StringUtils.humanReadableInt(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()) %> + + <% StringUtils.humanReadableInt(DirectMemoryUtils.getDirectMemoryUsage()) %> + + + <% StringUtils.humanReadableInt(DirectMemoryUtils.getDirectMemorySize()) %> + <% StringUtils.humanReadableInt(mWrap.getMemstoreSize()) %> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java index df3b6964419..461e00909ee 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java @@ -100,8 +100,8 @@ public interface BlockCache { long getCurrentSize(); /** - * Returns the number of evictions that have occurred. - * @return number of evictions + * Returns the number of blocks that have been evicted. + * @return number of evicted blocks */ long getEvictedCount(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 063a03b302e..2aae578242c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -55,7 +55,6 @@ public class CombinedBlockCache implements BlockCache, HeapSize { return lruCache.heapSize() + bucketCache.heapSize(); } - @Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { boolean isMetaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA; @@ -66,7 +65,6 @@ public class CombinedBlockCache implements BlockCache, HeapSize { } } - @Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { cacheBlock(cacheKey, buf, false); @@ -79,7 +77,6 @@ public class CombinedBlockCache implements BlockCache, HeapSize { return lruCache.getBlock(cacheKey, caching, repeat); } return bucketCache.getBlock(cacheKey, caching, repeat); - } @Override @@ -102,7 +99,6 @@ public class CombinedBlockCache implements BlockCache, HeapSize { public void shutdown() { lruCache.shutdown(); bucketCache.shutdown(); - } @Override @@ -209,7 +205,5 @@ public class CombinedBlockCache implements BlockCache, HeapSize { .getSumRequestCachingCountsPastNPeriods())); return Double.isNaN(ratio) ? 0 : ratio; } - } - } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index 51ef024dd8e..7f7b1722228 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -603,26 +603,17 @@ public class LruBlockCache implements BlockCache, HeapSize { return this.maxSize; } - /** - * Get the current size of this cache. - * @return current size in bytes - */ + @Override public long getCurrentSize() { return this.size.get(); } - /** - * Get the current size of this cache. - * @return current size in bytes - */ + @Override public long getFreeSize() { return getMaxSize() - getCurrentSize(); } - /** - * Get the size of this cache (number of cached blocks) - * @return number of cached blocks - */ + @Override public long size() { return this.elements.get(); } @@ -639,10 +630,7 @@ public class LruBlockCache implements BlockCache, HeapSize { return this.stats.getEvictionCount(); } - /** - * Get the number of blocks that have been evicted during the lifetime - * of this cache. - */ + @Override public long getEvictedCount() { return this.stats.getEvictedCount(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index bb225443623..cea808b3e83 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -893,14 +893,11 @@ public class BucketCache implements BlockCache, HeapSize { return this.bucketAllocator; } + @Override public long heapSize() { return this.heapSize.get(); } - /** - * Returns the total size of the block cache, in bytes. - * @return size of cache, in bytes - */ @Override public long size() { return this.realCacheSize.get(); @@ -916,10 +913,6 @@ public class BucketCache implements BlockCache, HeapSize { return this.blockNumber.get(); } - /** - * Returns the occupied size of the block cache, in bytes. - * @return occupied space in cache, in bytes - */ @Override public long getCurrentSize() { return this.bucketAllocator.getUsedSize(); @@ -931,10 +924,10 @@ public class BucketCache implements BlockCache, HeapSize { } /** - * Evicts all blocks for a specific HFile. + * Evicts all blocks for a specific HFile. *

* This is used for evict-on-close to remove all blocks of a specific HFile. - * + * * @return the number of blocks evicted */ @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java index 3b2d85eb621..7e713d60cbc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java @@ -91,8 +91,7 @@ public class SingleSizeCache implements BlockCache, HeapSize { this.timeSinceLastAccess = new AtomicLong(); // This evictionListener is called whenever the cache automatically - // evicts - // something. + // evicts something. RemovalListener listener = new RemovalListener() { @Override @@ -116,8 +115,6 @@ public class SingleSizeCache implements BlockCache, HeapSize { .removalListener(listener) .build() .asMap(); - - } @Override @@ -138,7 +135,6 @@ public class SingleSizeCache implements BlockCache, HeapSize { synchronized (this) { CacheablePair alreadyCached = backingMap.putIfAbsent(blockName, newEntry); - if (alreadyCached != null) { backingStore.free(storedBlock); @@ -193,7 +189,6 @@ public class SingleSizeCache implements BlockCache, HeapSize { doEviction(key, evictedBlock); } return evictedBlock != null; - } private void doEviction(BlockCacheKey key, CacheablePair evictedBlock) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java index 0711fa61571..32757e2d436 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java @@ -179,7 +179,7 @@ public class SlabCache implements SlabItemActionWatcher, BlockCache, HeapSize { private void addSlab(int blockSize, int numBlocks) { LOG.info("Creating a slab of blockSize " + blockSize + " with " + numBlocks - + " blocks."); + + " blocks, " + StringUtils.humanReadableInt(blockSize * numBlocks) + "bytes."); sizer.put(blockSize, new SingleSizeCache(blockSize, numBlocks, this)); } @@ -229,8 +229,6 @@ public class SlabCache implements SlabItemActionWatcher, BlockCache, HeapSize { /** * Get the buffer of the block with the specified name. - * @param caching - * @param key * * @return buffer of specified block name, or null if not in cache */ @@ -301,13 +299,17 @@ public class SlabCache implements SlabItemActionWatcher, BlockCache, HeapSize { } public long getFreeSize() { - return 0; // this cache, by default, allocates all its space. + long childFreeSize = 0; + for (SingleSizeCache s : sizer.values()) { + childFreeSize += s.getFreeSize(); + } + return childFreeSize; } @Override public long getBlockCount() { long count = 0; - for (SingleSizeCache cache : backingStore.values()) { + for (SingleSizeCache cache : sizer.values()) { count += cache.getBlockCount(); } return count; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 59948f25b15..9f88e705d91 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -198,7 +198,7 @@ class MetricsRegionServerWrapperImpl if (this.blockCache == null) { return 0; } - return this.blockCache.size(); + return this.blockCache.getBlockCount(); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java index 2c28125cc50..bfaa6f24534 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/DirectMemoryUtils.java @@ -26,22 +26,53 @@ import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import com.google.common.base.Preconditions; +import javax.management.JMException; +import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; + +/** + * Utilities for interacting with and monitoring DirectByteBuffer allocations. + */ @InterfaceAudience.Private @InterfaceStability.Evolving public class DirectMemoryUtils { + private static final Log LOG = LogFactory.getLog(DirectMemoryUtils.class); + private static final MBeanServer beanServer; + private static final ObjectName nioDirectPool; + + static { + // initialize singletons. Only maintain a reference to the MBeanServer if + // we're able to consume it -- hence convoluted logic. + ObjectName n = null; + MBeanServer s = null; + try { + n = new ObjectName("java.nio:type=BufferPool,name=direct"); + } catch (MalformedObjectNameException e) { + LOG.warn("Unable to initialize ObjectName for DirectByteBuffer allocations."); + } finally { + nioDirectPool = n; + } + if (nioDirectPool != null) { + s = ManagementFactory.getPlatformMBeanServer(); + } + beanServer = s; + } + /** * @return the setting of -XX:MaxDirectMemorySize as a long. Returns 0 if * -XX:MaxDirectMemorySize is not set. */ - public static long getDirectMemorySize() { - RuntimeMXBean RuntimemxBean = ManagementFactory.getRuntimeMXBean(); - List arguments = RuntimemxBean.getInputArguments(); + RuntimeMXBean runtimemxBean = ManagementFactory.getRuntimeMXBean(); + List arguments = runtimemxBean.getInputArguments(); long multiplier = 1; //for the byte case. for (String s : arguments) { if (s.contains("-XX:MaxDirectMemorySize=")) { @@ -64,11 +95,24 @@ public class DirectMemoryUtils { long retValue = Long.parseLong(memSize); return retValue * multiplier; } - } return 0; } + /** + * @return the current amount of direct memory used. + */ + public static long getDirectMemoryUsage() { + if (beanServer == null || nioDirectPool == null) return 0; + try { + Long value = (Long) beanServer.getAttribute(nioDirectPool, "MemoryUsed"); + return value == null ? 0 : value; + } catch (JMException e) { + LOG.debug("Failed to retrieve nio.BufferPool direct MemoryUsed"); + return 0; + } + } + /** * DirectByteBuffers are garbage collected by using a phantom reference and a * reference queue. Every once a while, the JVM checks the reference queue and @@ -94,6 +138,5 @@ public class DirectMemoryUtils { Method cleanMethod = cleaner.getClass().getMethod("clean"); cleanMethod.setAccessible(true); cleanMethod.invoke(cleaner); - } }