HBASE-22412 Improve the metrics in ByteBuffAllocator

hbase-common/src/main/java/org/apache/hadoop/hbase/io/ByteBuffAllocator.java

@@ -97,12 +97,12 @@ public class ByteBuffAllocator {
 
   private final Queue<ByteBuffer> buffers = new ConcurrentLinkedQueue<>();
 
-  // Metrics to track the pool allocation number and heap allocation number. If heap allocation
+  // Metrics to track the pool allocation bytes and heap allocation bytes. If heap allocation
-  // number is increasing so much, then we may need to increase the max.buffer.count .
+  // bytes is increasing so much, then we may need to increase the max.buffer.count .
-  private final LongAdder poolAllocationNum = new LongAdder();
+  private final LongAdder poolAllocationBytes = new LongAdder();
-  private final LongAdder heapAllocationNum = new LongAdder();
+  private final LongAdder heapAllocationBytes = new LongAdder();
-  private long lastPoolAllocationNum = 0;
+  private long lastPoolAllocationBytes = 0;
-  private long lastHeapAllocationNum = 0;
+  private long lastHeapAllocationBytes = 0;
 
   /**
    * Initialize an {@link ByteBuffAllocator} which will try to allocate ByteBuffers from off-heap if
@@ -161,14 +161,26 @@ public class ByteBuffAllocator {
     return reservoirEnabled;
   }
 
-  public long getHeapAllocationNum() {
+  public long getHeapAllocationBytes() {
-    return heapAllocationNum.sum();
+    return heapAllocationBytes.sum();
   }
 
-  public long getPoolAllocationNum() {
+  public long getPoolAllocationBytes() {
-    return poolAllocationNum.sum();
+    return poolAllocationBytes.sum();
   }
 
+  public int getBufferSize() {
+    return this.bufSize;
+  }
+
+  public int getUsedBufferCount() {
+    return this.usedBufCount.intValue();
+  }
+
+  /**
+   * The {@link ConcurrentLinkedQueue#size()} is O(N) complexity and time-consuming, so DO NOT use
+   * the method except in UT.
+   */
   @VisibleForTesting
   public int getFreeBufferCount() {
     return this.buffers.size();
@@ -179,15 +191,15 @@ public class ByteBuffAllocator {
   }
 
   public double getHeapAllocationRatio() {
-    long heapAllocNum = heapAllocationNum.sum(), poolAllocNum = poolAllocationNum.sum();
+    long heapAllocBytes = heapAllocationBytes.sum(), poolAllocBytes = poolAllocationBytes.sum();
-    double heapDelta = heapAllocNum - lastHeapAllocationNum;
+    double heapDelta = heapAllocBytes - lastHeapAllocationBytes;
-    double poolDelta = poolAllocNum - lastPoolAllocationNum;
+    double poolDelta = poolAllocBytes - lastPoolAllocationBytes;
-    lastHeapAllocationNum = heapAllocNum;
+    lastHeapAllocationBytes = heapAllocBytes;
-    lastPoolAllocationNum = poolAllocNum;
+    lastPoolAllocationBytes = poolAllocBytes;
     if (Math.abs(heapDelta + poolDelta) < 1e-3) {
       return 0.0;
     }
-    return heapDelta / (heapDelta + poolDelta) * 100;
+    return heapDelta / (heapDelta + poolDelta);
   }
 
   /**
@@ -208,7 +220,7 @@ public class ByteBuffAllocator {
   }
 
   private ByteBuffer allocateOnHeap(int size) {
-    heapAllocationNum.increment();
+    heapAllocationBytes.add(size);
     return ByteBuffer.allocate(size);
   }
 
@@ -282,7 +294,7 @@ public class ByteBuffAllocator {
     if (bb != null) {
       // To reset the limit to capacity and position to 0, must clear here.
       bb.clear();
-      poolAllocationNum.increment();
+      poolAllocationBytes.add(bufSize);
       return bb;
     }
     while (true) {
@@ -299,7 +311,7 @@ public class ByteBuffAllocator {
       if (!this.usedBufCount.compareAndSet(c, c + 1)) {
         continue;
       }
-      poolAllocationNum.increment();
+      poolAllocationBytes.add(bufSize);
       return ByteBuffer.allocateDirect(bufSize);
     }
   }

hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestByteBuffAllocator.java

@@ -47,9 +47,12 @@ public class TestByteBuffAllocator {
     int maxBuffersInPool = 10;
     int bufSize = 6 * 1024;
     ByteBuffAllocator alloc = new ByteBuffAllocator(true, maxBuffersInPool, bufSize, bufSize / 6);
+    assertEquals(0, alloc.getUsedBufferCount());
+
     ByteBuff buff = alloc.allocate(10 * bufSize);
-    assertEquals(10, alloc.getPoolAllocationNum());
+    assertEquals(61440, alloc.getPoolAllocationBytes());
-    assertEquals(0, alloc.getHeapAllocationNum());
+    assertEquals(0, alloc.getHeapAllocationBytes());
+    assertEquals(10, alloc.getUsedBufferCount());
     buff.release();
     // When the request size is less than 1/6th of the pool buffer size. We should use on demand
     // created on heap Buffer
@@ -57,15 +60,17 @@ public class TestByteBuffAllocator {
     assertTrue(buff.hasArray());
     assertEquals(maxBuffersInPool, alloc.getFreeBufferCount());
     assertEquals(maxBuffersInPool, alloc.getTotalBufferCount());
-    assertEquals(10, alloc.getPoolAllocationNum());
+    assertEquals(61440, alloc.getPoolAllocationBytes());
-    assertEquals(1, alloc.getHeapAllocationNum());
+    assertEquals(200, alloc.getHeapAllocationBytes());
+    assertEquals(10, alloc.getUsedBufferCount());
     buff.release();
     // When the request size is > 1/6th of the pool buffer size.
     buff = alloc.allocate(1024);
     assertFalse(buff.hasArray());
     assertEquals(maxBuffersInPool - 1, alloc.getFreeBufferCount());
-    assertEquals(11, alloc.getPoolAllocationNum());
+    assertEquals(67584, alloc.getPoolAllocationBytes());
-    assertEquals(1, alloc.getHeapAllocationNum());
+    assertEquals(200, alloc.getHeapAllocationBytes());
+    assertEquals(10, alloc.getUsedBufferCount());
     buff.release();// ByteBuff Recycler#free should put back the BB to pool.
     assertEquals(maxBuffersInPool, alloc.getFreeBufferCount());
     // Request size> pool buffer size
@@ -79,8 +84,9 @@ public class TestByteBuffAllocator {
     assertEquals(6 * 1024, bbs[0].limit());
     assertEquals(1024, bbs[1].limit());
     assertEquals(maxBuffersInPool - 2, alloc.getFreeBufferCount());
-    assertEquals(13, alloc.getPoolAllocationNum());
+    assertEquals(79872, alloc.getPoolAllocationBytes());
-    assertEquals(1, alloc.getHeapAllocationNum());
+    assertEquals(200, alloc.getHeapAllocationBytes());
+    assertEquals(10, alloc.getUsedBufferCount());
     buff.release();
     assertEquals(maxBuffersInPool, alloc.getFreeBufferCount());
 
@@ -94,14 +100,16 @@ public class TestByteBuffAllocator {
     assertEquals(6 * 1024, bbs[0].limit());
     assertEquals(200, bbs[1].limit());
     assertEquals(maxBuffersInPool - 1, alloc.getFreeBufferCount());
-    assertEquals(14, alloc.getPoolAllocationNum());
+    assertEquals(86016, alloc.getPoolAllocationBytes());
-    assertEquals(2, alloc.getHeapAllocationNum());
+    assertEquals(400, alloc.getHeapAllocationBytes());
+    assertEquals(10, alloc.getUsedBufferCount());
     buff.release();
     assertEquals(maxBuffersInPool, alloc.getFreeBufferCount());
 
     alloc.allocate(bufSize * (maxBuffersInPool - 1));
-    assertEquals(23, alloc.getPoolAllocationNum());
+    assertEquals(141312, alloc.getPoolAllocationBytes());
-    assertEquals(2, alloc.getHeapAllocationNum());
+    assertEquals(400, alloc.getHeapAllocationBytes());
+    assertEquals(10, alloc.getUsedBufferCount());
 
     buff = alloc.allocate(20 * 1024);
     assertFalse(buff.hasArray());
@@ -113,21 +121,24 @@ public class TestByteBuffAllocator {
     assertEquals(6 * 1024, bbs[0].limit());
     assertEquals(14 * 1024, bbs[1].limit());
     assertEquals(0, alloc.getFreeBufferCount());
-    assertEquals(24, alloc.getPoolAllocationNum());
+    assertEquals(147456, alloc.getPoolAllocationBytes());
-    assertEquals(3, alloc.getHeapAllocationNum());
+    assertEquals(14736, alloc.getHeapAllocationBytes());
+    assertEquals(10, alloc.getUsedBufferCount());
 
     buff.release();
     assertEquals(1, alloc.getFreeBufferCount());
     alloc.allocateOneBuffer();
-    assertEquals(25, alloc.getPoolAllocationNum());
+    assertEquals(153600, alloc.getPoolAllocationBytes());
-    assertEquals(3, alloc.getHeapAllocationNum());
+    assertEquals(14736, alloc.getHeapAllocationBytes());
+    assertEquals(10, alloc.getUsedBufferCount());
 
     buff = alloc.allocate(7 * 1024);
     assertTrue(buff.hasArray());
     assertTrue(buff instanceof SingleByteBuff);
     assertEquals(7 * 1024, buff.nioByteBuffers()[0].limit());
-    assertEquals(25, alloc.getPoolAllocationNum());
+    assertEquals(153600, alloc.getPoolAllocationBytes());
-    assertEquals(4, alloc.getHeapAllocationNum());
+    assertEquals(21904, alloc.getHeapAllocationBytes());
+    assertEquals(10, alloc.getUsedBufferCount());
     buff.release();
   }
 
@@ -142,7 +153,7 @@ public class TestByteBuffAllocator {
       // expected exception
     }
     ByteBuff bb = allocator.allocate(0);
-    assertEquals(1, allocator.getHeapAllocationNum());
+    assertEquals(0, allocator.getHeapAllocationBytes());
     bb.release();
   }
 

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -554,17 +554,17 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
       "Average region size over the RegionServer including memstore and storefile sizes.";
 
   /** Metrics for {@link org.apache.hadoop.hbase.io.ByteBuffAllocator} **/
-  String BYTE_BUFF_ALLOCATOR_HEAP_ALLOCATION_NUM = "ByteBuffAllocatorHeapAllocationNum";
+  String BYTE_BUFF_ALLOCATOR_HEAP_ALLOCATION_BYTES = "ByteBuffAllocatorHeapAllocationBytes";
-  String BYTE_BUFF_ALLOCATOR_HEAP_ALLOCATION_NUM_DESC =
+  String BYTE_BUFF_ALLOCATOR_HEAP_ALLOCATION_BYTES_DESC =
-      "Number of heap allocation from ByteBuffAllocator";
+      "Bytes of heap allocation from ByteBuffAllocator";
-  String BYTE_BUFF_ALLOCATOR_POOL_ALLOCATION_NUM = "ByteBuffAllocatorPoolAllocationNum";
+  String BYTE_BUFF_ALLOCATOR_POOL_ALLOCATION_BYTES = "ByteBuffAllocatorPoolAllocationBytes";
-  String BYTE_BUFF_ALLOCATOR_POOL_ALLOCATION_NUM_DESC =
+  String BYTE_BUFF_ALLOCATOR_POOL_ALLOCATION_BYTES_DESC =
-      "Number of pool allocation from ByteBuffAllocator";
+      "Bytes of pool allocation from ByteBuffAllocator";
-  String BYTE_BUFF_ALLOCATOR_HEAP_ALLOACTION_RATIO = "ByteBuffAllocatorHeapAllocationRatio";
+  String BYTE_BUFF_ALLOCATOR_HEAP_ALLOCATION_RATIO = "ByteBuffAllocatorHeapAllocationRatio";
-  String BYTE_BUFF_ALLOCATOR_HEAP_ALLOACTION_RATIO_DESC =
+  String BYTE_BUFF_ALLOCATOR_HEAP_ALLOCATION_RATIO_DESC =
       "Ratio of heap allocation from ByteBuffAllocator, means heapAllocation/totalAllocation";
   String BYTE_BUFF_ALLOCATOR_TOTAL_BUFFER_COUNT = "ByteBuffAllocatorTotalBufferCount";
   String BYTE_BUFF_ALLOCATOR_TOTAL_BUFFER_COUNT_DESC = "Total buffer count in ByteBuffAllocator";
-  String BYTE_BUFF_ALLOCATOR_FREE_BUFFER_COUNT = "ByteBuffAllocatorFreeBufferCount";
+  String BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT = "ByteBuffAllocatorUsedBufferCount";
-  String BYTE_BUFF_ALLOCATOR_FREE_BUFFER_COUNT_DESC = "Free buffer count in ByteBuffAllocator";
+  String BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT_DESC = "Used buffer count in ByteBuffAllocator";
 }

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java

@@ -514,13 +514,13 @@ public interface MetricsRegionServerWrapper {
 
   long getTotalRowActionRequestCount();
 
-  long getByteBuffAllocatorHeapAllocationNum();
+  long getByteBuffAllocatorHeapAllocationBytes();
 
-  long getByteBuffAllocatorPoolAllocationNum();
+  long getByteBuffAllocatorPoolAllocationBytes();
 
   double getByteBuffAllocatorHeapAllocRatio();
 
   long getByteBuffAllocatorTotalBufferCount();
 
-  long getByteBuffAllocatorFreeBufferCount();
+  long getByteBuffAllocatorUsedBufferCount();
 }

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java

@@ -550,21 +550,21 @@ public class MetricsRegionServerSourceImpl
                     rsWrap.getReadRequestsRatePerSecond())
             .addGauge(Interns.info(WRITE_REQUEST_RATE_PER_SECOND, WRITE_REQUEST_RATE_DESC),
                     rsWrap.getWriteRequestsRatePerSecond())
-            .addGauge(Interns.info(BYTE_BUFF_ALLOCATOR_HEAP_ALLOCATION_NUM,
+            .addGauge(Interns.info(BYTE_BUFF_ALLOCATOR_HEAP_ALLOCATION_BYTES,
-                BYTE_BUFF_ALLOCATOR_HEAP_ALLOCATION_NUM_DESC),
+                  BYTE_BUFF_ALLOCATOR_HEAP_ALLOCATION_BYTES_DESC),
-                rsWrap.getByteBuffAllocatorHeapAllocationNum())
+                rsWrap.getByteBuffAllocatorHeapAllocationBytes())
-            .addGauge(Interns.info(BYTE_BUFF_ALLOCATOR_POOL_ALLOCATION_NUM,
+            .addGauge(Interns.info(BYTE_BUFF_ALLOCATOR_POOL_ALLOCATION_BYTES,
-                BYTE_BUFF_ALLOCATOR_POOL_ALLOCATION_NUM_DESC),
+                  BYTE_BUFF_ALLOCATOR_POOL_ALLOCATION_BYTES_DESC),
-                rsWrap.getByteBuffAllocatorPoolAllocationNum())
+                rsWrap.getByteBuffAllocatorPoolAllocationBytes())
-            .addGauge(Interns.info(BYTE_BUFF_ALLOCATOR_HEAP_ALLOACTION_RATIO,
+            .addGauge(Interns.info(BYTE_BUFF_ALLOCATOR_HEAP_ALLOCATION_RATIO,
-                BYTE_BUFF_ALLOCATOR_HEAP_ALLOACTION_RATIO_DESC),
+                  BYTE_BUFF_ALLOCATOR_HEAP_ALLOCATION_RATIO_DESC),
                 rsWrap.getByteBuffAllocatorHeapAllocRatio())
             .addGauge(Interns.info(BYTE_BUFF_ALLOCATOR_TOTAL_BUFFER_COUNT,
                 BYTE_BUFF_ALLOCATOR_TOTAL_BUFFER_COUNT_DESC),
                 rsWrap.getByteBuffAllocatorTotalBufferCount())
-            .addGauge(Interns.info(BYTE_BUFF_ALLOCATOR_FREE_BUFFER_COUNT,
+            .addGauge(Interns.info(BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT,
-                BYTE_BUFF_ALLOCATOR_FREE_BUFFER_COUNT_DESC),
+                BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT_DESC),
-                rsWrap.getByteBuffAllocatorFreeBufferCount());
+                rsWrap.getByteBuffAllocatorUsedBufferCount());
   }
 
   @Override

hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon

@@ -238,18 +238,20 @@ ByteBuffAllocator bbAllocator;
 </%args>
 <table class="table table-striped">
 <tr>
-    <th>Number of Heap Allocation</th>
+    <th>Total Heap Allocation(Bytes)</th>
-    <th>Number of Pool Allocation</th>
+    <th>Total Pool Allocation(Bytes)</th>
     <th>Heap Allocation Ratio</th>
     <th>Total Buffer Count</th>
-    <th>Free Buffer Count</th>
+    <th>Used Buffer Count</th>
+    <th>Buffer Size(Bytes)</th>
 </tr>
 <tr>
-    <td><% bbAllocator.getHeapAllocationNum() %></td>
+    <td><% bbAllocator.getHeapAllocationBytes() %></td>
-    <td><% bbAllocator.getPoolAllocationNum() %></td>
+    <td><% bbAllocator.getPoolAllocationBytes() %></td>
-    <td><% bbAllocator.getHeapAllocationRatio() %>%</td>
+    <td><% String.format("%.3f", bbAllocator.getHeapAllocationRatio() * 100) %><% "%" %></td>
     <td><% bbAllocator.getTotalBufferCount() %></td>
-    <td><% bbAllocator.getFreeBufferCount() %></td>
+    <td><% bbAllocator.getUsedBufferCount() %></td>
+    <td><% bbAllocator.getBufferSize() %></td>
 </tr>
 </table>
 </%def>

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

@@ -993,13 +993,13 @@ class MetricsRegionServerWrapperImpl
   }
 
   @Override
-  public long getByteBuffAllocatorHeapAllocationNum() {
+  public long getByteBuffAllocatorHeapAllocationBytes() {
-    return this.allocator.getHeapAllocationNum();
+    return this.allocator.getHeapAllocationBytes();
   }
 
   @Override
-  public long getByteBuffAllocatorPoolAllocationNum() {
+  public long getByteBuffAllocatorPoolAllocationBytes() {
-    return this.allocator.getPoolAllocationNum();
+    return this.allocator.getPoolAllocationBytes();
   }
 
   @Override
@@ -1013,7 +1013,7 @@ class MetricsRegionServerWrapperImpl
   }
 
   @Override
-  public long getByteBuffAllocatorFreeBufferCount() {
+  public long getByteBuffAllocatorUsedBufferCount() {
-    return this.allocator.getFreeBufferCount();
+    return this.allocator.getUsedBufferCount();
   }
 }

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java

@@ -111,12 +111,12 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
   }
 
   @Override
-  public long getByteBuffAllocatorHeapAllocationNum() {
+  public long getByteBuffAllocatorHeapAllocationBytes() {
     return 0;
   }
 
   @Override
-  public long getByteBuffAllocatorPoolAllocationNum() {
+  public long getByteBuffAllocatorPoolAllocationBytes() {
     return 0;
   }
 
@@ -131,7 +131,7 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
   }
 
   @Override
-  public long getByteBuffAllocatorFreeBufferCount() {
+  public long getByteBuffAllocatorUsedBufferCount() {
     return 0;
   }