Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-21029 Miscount of memstore's heap/offheap size if same cell was put

Browse Source
This commit is contained in:
Allan Yang 2018-08-13 20:30:23 +08:00
parent 26828b1860
commit 161c018927
2 changed files with 22 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
View File

@ -299,11 +299,15 @@ public abstract class Segment implements MemStoreSizing {
// If there's already a same cell in the CellSet and we are using MSLAB, we must count in the // If there's already a same cell in the CellSet and we are using MSLAB, we must count in the
// MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger // MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger
// than the counted number) // than the counted number)
if (succ || mslabUsed) { boolean sizeChanged = succ || mslabUsed;
if (sizeChanged) {
cellSize = getCellLength(cellToAdd); cellSize = getCellLength(cellToAdd);
} }
long heapSize = heapSizeChange(cellToAdd, succ); // same as above, if MSLAB is used, we need to inc the heap/offheap size, otherwise there will
long offHeapSize = offHeapSizeChange(cellToAdd, succ); // be a memory miscount. Since we are now use heapSize + offHeapSize to decide whether a flush
// is needed.
long heapSize = heapSizeChange(cellToAdd, sizeChanged);
long offHeapSize = offHeapSizeChange(cellToAdd, sizeChanged);
incMemStoreSize(cellSize, heapSize, offHeapSize); incMemStoreSize(cellSize, heapSize, offHeapSize);
if (memstoreSizing != null) { if (memstoreSizing != null) {
memstoreSizing.incMemStoreSize(cellSize, heapSize, offHeapSize); memstoreSizing.incMemStoreSize(cellSize, heapSize, offHeapSize);

15
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
View File

@ -138,6 +138,21 @@ public class TestDefaultMemStore {
Segment segment = this.memstore.getActive(); Segment segment = this.memstore.getActive();
MemStoreLAB msLab = segment.getMemStoreLAB(); MemStoreLAB msLab = segment.getMemStoreLAB();
if (msLab != null) { if (msLab != null) {
if (msLab.isOnHeap()) {
assertTrue("HeapSize should always bigger or equal than data size",
sizeChangeForFirstCell.getHeapSize() >= sizeChangeForFirstCell
.getDataSize());
assertTrue("HeapSize should always bigger or equal than data size",
sizeChangeForSecondCell.getHeapSize() >= sizeChangeForSecondCell
.getDataSize());
} else {
assertTrue("OffHeapSize should always bigger or equal than data size",
sizeChangeForFirstCell.getOffHeapSize() >= sizeChangeForFirstCell
.getDataSize());
assertTrue("OffHeapSize should always bigger or equal than data size",
sizeChangeForSecondCell.getOffHeapSize() >= sizeChangeForSecondCell
.getDataSize());
}
// make sure memstore size increased even when writing the same cell, if using MSLAB // make sure memstore size increased even when writing the same cell, if using MSLAB
assertEquals(Segment.getCellLength(kv), assertEquals(Segment.getCellLength(kv),
sizeChangeForSecondCell.getMemStoreSize().getDataSize()); sizeChangeForSecondCell.getMemStoreSize().getDataSize());