Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-17338 Treat Cell data size under global memstore heap size only when that Cell can not be copied to MSLAB.

This commit is contained in:
anoopsamjohn 2017-03-10 11:12:59 +05:30
parent c63a871176
commit ab5970773a
30 changed files with 289 additions and 379 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java
View File

@ -40,7 +40,7 @@ public class ByteBufferKeyValue extends ByteBufferCell implements ExtendedCell {
protected final int length; protected final int length;
private long seqId = 0; private long seqId = 0;
private static final int FIXED_OVERHEAD = ClassSize.OBJECT + ClassSize.REFERENCE public static final int FIXED_OVERHEAD = ClassSize.OBJECT + ClassSize.REFERENCE
+ (2 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_LONG; + (2 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_LONG;
public ByteBufferKeyValue(ByteBuffer buf, int offset, int length, long seqId) { public ByteBufferKeyValue(ByteBuffer buf, int offset, int length, long seqId) {
@ -301,11 +301,6 @@ public class ByteBufferKeyValue extends ByteBufferCell implements ExtendedCell {
Bytes.SIZEOF_LONG); Bytes.SIZEOF_LONG);
} }
@Override
public long heapOverhead() {
return FIXED_OVERHEAD;
}
@Override @Override
public Cell deepClone() { public Cell deepClone() {
byte[] copy = new byte[this.length]; byte[] copy = new byte[this.length];

36
hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
View File

@ -621,15 +621,6 @@ public final class CellUtil {
} }
} }
@Override
public long heapOverhead() {
long overhead = ((ExtendedCell) this.cell).heapOverhead() + HEAP_SIZE_OVERHEAD;
if (this.tags != null) {
overhead += ClassSize.ARRAY;
}
return overhead;
}
@Override @Override
public Cell deepClone() { public Cell deepClone() {
Cell clonedBaseCell = ((ExtendedCell) this.cell).deepClone(); Cell clonedBaseCell = ((ExtendedCell) this.cell).deepClone();
@ -811,15 +802,6 @@ public final class CellUtil {
} }
} }
@Override
public long heapOverhead() {
long overhead = ((ExtendedCell) this.cell).heapOverhead() + HEAP_SIZE_OVERHEAD;
if (this.tags != null) {
overhead += ClassSize.ARRAY;
}
return overhead;
}
@Override @Override
public Cell deepClone() { public Cell deepClone() {
Cell clonedBaseCell = ((ExtendedCell) this.cell).deepClone(); Cell clonedBaseCell = ((ExtendedCell) this.cell).deepClone();
@ -963,15 +945,6 @@ public final class CellUtil {
} }
} }
@Override
public long heapOverhead() {
long overhead = super.heapOverhead() + ClassSize.REFERENCE;
if (this.value != null) {
overhead += ClassSize.ARRAY;
}
return overhead;
}
@Override @Override
public Cell deepClone() { public Cell deepClone() {
Cell clonedBaseCell = ((ExtendedCell) this.cell).deepClone(); Cell clonedBaseCell = ((ExtendedCell) this.cell).deepClone();
@ -1038,15 +1011,6 @@ public final class CellUtil {
ValueAndTagRewriteCell.write(buf, offset, this.cell, this.value, this.tags); ValueAndTagRewriteCell.write(buf, offset, this.cell, this.value, this.tags);
} }
@Override
public long heapOverhead() {
long overhead = super.heapOverhead() + ClassSize.REFERENCE;
if (this.value != null) {
overhead += ClassSize.ARRAY;
}
return overhead;
}
@Override @Override
public Cell deepClone() { public Cell deepClone() {
Cell clonedBaseCell = ((ExtendedCell) this.cell).deepClone(); Cell clonedBaseCell = ((ExtendedCell) this.cell).deepClone();

5
hbase-common/src/main/java/org/apache/hadoop/hbase/ExtendedCell.java
View File

@ -68,11 +68,6 @@ public interface ExtendedCell extends Cell, SettableSequenceId, SettableTimestam
*/ */
void write(ByteBuffer buf, int offset); void write(ByteBuffer buf, int offset);
/**
* @return The heap size overhead associated with this Cell.
*/
long heapOverhead();
/** /**
* Does a deep copy of the contents to a new memory area and returns it as a new cell. * Does a deep copy of the contents to a new memory area and returns it as a new cell.
* @return The deep cloned cell * @return The deep cloned cell

3
hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java
View File

@ -120,8 +120,7 @@ public class IndividualBytesFieldCell implements ExtendedCell {
getValueLength(), getTagsLength(), withTags); getValueLength(), getTagsLength(), withTags);
} }
@Override private long heapOverhead() {
public long heapOverhead() {
return FIXED_OVERHEAD return FIXED_OVERHEAD
+ ClassSize.ARRAY // row , can not be null + ClassSize.ARRAY // row , can not be null
+ ((family == null) ? 0 : ClassSize.ARRAY) // family , can be null + ((family == null) ? 0 : ClassSize.ARRAY) // family , can be null

10
hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
View File

@ -2804,16 +2804,6 @@ public class KeyValue implements ExtendedCell {
// of Cell to be returned back over the RPC // of Cell to be returned back over the RPC
throw new IllegalStateException("A reader should never return this type of a Cell"); throw new IllegalStateException("A reader should never return this type of a Cell");
} }
@Override
public long heapOverhead() {
return super.heapOverhead() + Bytes.SIZEOF_SHORT;
}
}
@Override
public long heapOverhead() {
return FIXED_OVERHEAD;
} }
@Override @Override

5
hbase-common/src/main/java/org/apache/hadoop/hbase/SizeCachedKeyValue.java
View File

@ -62,9 +62,4 @@ public class SizeCachedKeyValue extends KeyValue {
public long heapSize() { public long heapSize() {
return super.heapSize() + FIXED_OVERHEAD; return super.heapSize() + FIXED_OVERHEAD;
} }
@Override
public long heapOverhead() {
return super.heapOverhead() + FIXED_OVERHEAD;
}
} }

10
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
View File

@ -469,11 +469,6 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override
public long heapOverhead() {
return FIXED_OVERHEAD;
}
@Override @Override
public Cell deepClone() { public Cell deepClone() {
// This is not used in actual flow. Throwing UnsupportedOperationException // This is not used in actual flow. Throwing UnsupportedOperationException
@ -719,11 +714,6 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
@Override
public long heapOverhead() {
return FIXED_OVERHEAD;
}
@Override @Override
public Cell deepClone() { public Cell deepClone() {
// This is not used in actual flow. Throwing UnsupportedOperationException // This is not used in actual flow. Throwing UnsupportedOperationException

6
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
View File

@ -153,7 +153,7 @@ public abstract class AbstractMemStore implements MemStore {
@Override @Override
public MemstoreSize getSnapshotSize() { public MemstoreSize getSnapshotSize() {
return new MemstoreSize(this.snapshot.keySize(), this.snapshot.heapOverhead()); return new MemstoreSize(this.snapshot.keySize(), this.snapshot.heapSize());
} }
@Override @Override
@ -281,10 +281,10 @@ public abstract class AbstractMemStore implements MemStore {
protected abstract long keySize(); protected abstract long keySize();
/** /**
* @return The total heap overhead of cells in this memstore. We will not consider cells in the * @return The total heap size of cells in this memstore. We will not consider cells in the
* snapshot * snapshot
*/ */
protected abstract long heapOverhead(); protected abstract long heapSize();
protected CellComparator getComparator() { protected CellComparator getComparator() {
return comparator; return comparator;

14
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
View File

@ -123,9 +123,9 @@ public class CompactingMemStore extends AbstractMemStore {
@Override @Override
public MemstoreSize size() { public MemstoreSize size() {
MemstoreSize memstoreSize = new MemstoreSize(); MemstoreSize memstoreSize = new MemstoreSize();
memstoreSize.incMemstoreSize(this.active.keySize(), this.active.heapOverhead()); memstoreSize.incMemstoreSize(this.active.keySize(), this.active.heapSize());
for (Segment item : pipeline.getSegments()) { for (Segment item : pipeline.getSegments()) {
memstoreSize.incMemstoreSize(item.keySize(), item.heapOverhead()); memstoreSize.incMemstoreSize(item.keySize(), item.heapSize());
} }
return memstoreSize; return memstoreSize;
} }
@ -196,13 +196,13 @@ public class CompactingMemStore extends AbstractMemStore {
// if snapshot is empty the tail of the pipeline (or everything in the memstore) is flushed // if snapshot is empty the tail of the pipeline (or everything in the memstore) is flushed
if (compositeSnapshot) { if (compositeSnapshot) {
snapshotSize = pipeline.getPipelineSize(); snapshotSize = pipeline.getPipelineSize();
snapshotSize.incMemstoreSize(this.active.keySize(), this.active.heapOverhead()); snapshotSize.incMemstoreSize(this.active.keySize(), this.active.heapSize());
} else { } else {
snapshotSize = pipeline.getTailSize(); snapshotSize = pipeline.getTailSize();
} }
} }
return snapshotSize.getDataSize() > 0 ? snapshotSize return snapshotSize.getDataSize() > 0 ? snapshotSize
: new MemstoreSize(this.active.keySize(), this.active.heapOverhead()); : new MemstoreSize(this.active.keySize(), this.active.heapSize());
} }
@Override @Override
@ -216,11 +216,11 @@ public class CompactingMemStore extends AbstractMemStore {
} }
@Override @Override
protected long heapOverhead() { protected long heapSize() {
// Need to consider heapOverhead of all segments in pipeline and active // Need to consider heapOverhead of all segments in pipeline and active
long h = this.active.heapOverhead(); long h = this.active.heapSize();
for (Segment segment : this.pipeline.getSegments()) { for (Segment segment : this.pipeline.getSegments()) {
h += segment.heapOverhead(); h += segment.heapSize();
} }
return h; return h;
} }

30
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
View File

@ -141,24 +141,24 @@ public class CompactionPipeline {
long newDataSize = 0; long newDataSize = 0;
if(segment != null) newDataSize = segment.keySize(); if(segment != null) newDataSize = segment.keySize();
long dataSizeDelta = suffixDataSize - newDataSize; long dataSizeDelta = suffixDataSize - newDataSize;
long suffixHeapOverhead = getSegmentsHeapOverhead(suffix); long suffixHeapSize = getSegmentsHeapSize(suffix);
long newHeapOverhead = 0; long newHeapSize = 0;
if(segment != null) newHeapOverhead = segment.heapOverhead(); if(segment != null) newHeapSize = segment.heapSize();
long heapOverheadDelta = suffixHeapOverhead - newHeapOverhead; long heapSizeDelta = suffixHeapSize - newHeapSize;
region.addMemstoreSize(new MemstoreSize(-dataSizeDelta, -heapOverheadDelta)); region.addMemstoreSize(new MemstoreSize(-dataSizeDelta, -heapSizeDelta));
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Suffix data size: " + suffixDataSize + " new segment data size: " LOG.debug("Suffix data size: " + suffixDataSize + " new segment data size: "
+ newDataSize + ". Suffix heap overhead: " + suffixHeapOverhead + newDataSize + ". Suffix heap size: " + suffixHeapSize
+ " new segment heap overhead: " + newHeapOverhead); + " new segment heap size: " + newHeapSize);
} }
} }
return true; return true;
} }
private static long getSegmentsHeapOverhead(List<? extends Segment> list) { private static long getSegmentsHeapSize(List<? extends Segment> list) {
long res = 0; long res = 0;
for (Segment segment : list) { for (Segment segment : list) {
res += segment.heapOverhead(); res += segment.heapSize();
} }
return res; return res;
} }
@ -234,20 +234,20 @@ public class CompactionPipeline {
public MemstoreSize getTailSize() { public MemstoreSize getTailSize() {
LinkedList<? extends Segment> localCopy = readOnlyCopy; LinkedList<? extends Segment> localCopy = readOnlyCopy;
if (localCopy.isEmpty()) return MemstoreSize.EMPTY_SIZE; if (localCopy.isEmpty()) return new MemstoreSize(true);
return new MemstoreSize(localCopy.peekLast().keySize(), localCopy.peekLast().heapOverhead()); return new MemstoreSize(localCopy.peekLast().keySize(), localCopy.peekLast().heapSize());
} }
public MemstoreSize getPipelineSize() { public MemstoreSize getPipelineSize() {
long keySize = 0; long keySize = 0;
long heapOverhead = 0; long heapSize = 0;
LinkedList<? extends Segment> localCopy = readOnlyCopy; LinkedList<? extends Segment> localCopy = readOnlyCopy;
if (localCopy.isEmpty()) return MemstoreSize.EMPTY_SIZE; if (localCopy.isEmpty()) return new MemstoreSize(true);
for (Segment segment : localCopy) { for (Segment segment : localCopy) {
keySize += segment.keySize(); keySize += segment.keySize();
heapOverhead += segment.heapOverhead(); heapSize += segment.heapSize();
} }
return new MemstoreSize(keySize, heapOverhead); return new MemstoreSize(keySize, heapSize);
} }
private void swapSuffix(List<? extends Segment> suffix, ImmutableSegment segment, private void swapSuffix(List<? extends Segment> suffix, ImmutableSegment segment,

56
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java
View File

@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Scan;
@ -43,7 +42,6 @@ import java.util.SortedSet;
public class CompositeImmutableSegment extends ImmutableSegment { public class CompositeImmutableSegment extends ImmutableSegment {
private final List<ImmutableSegment> segments; private final List<ImmutableSegment> segments;
private final CellComparator comparator;
// CompositeImmutableSegment is used for snapshots and snapshot should // CompositeImmutableSegment is used for snapshots and snapshot should
// support getTimeRangeTracker() interface. // support getTimeRangeTracker() interface.
// Thus we hold a constant TRT build in the construction time from TRT of the given segments. // Thus we hold a constant TRT build in the construction time from TRT of the given segments.
@ -53,7 +51,6 @@ public class CompositeImmutableSegment extends ImmutableSegment {
public CompositeImmutableSegment(CellComparator comparator, List<ImmutableSegment> segments) { public CompositeImmutableSegment(CellComparator comparator, List<ImmutableSegment> segments) {
super(comparator); super(comparator);
this.comparator = comparator;
this.segments = segments; this.segments = segments;
this.timeRangeTracker = new TimeRangeTracker(); this.timeRangeTracker = new TimeRangeTracker();
for (ImmutableSegment s : segments) { for (ImmutableSegment s : segments) {
@ -64,10 +61,12 @@ public class CompositeImmutableSegment extends ImmutableSegment {
} }
@VisibleForTesting @VisibleForTesting
@Override
public List<Segment> getAllSegments() { public List<Segment> getAllSegments() {
return new LinkedList<>(segments); return new LinkedList<>(segments);
} }
@Override
public int getNumOfSegments() { public int getNumOfSegments() {
return segments.size(); return segments.size();
} }
@ -77,6 +76,7 @@ public class CompositeImmutableSegment extends ImmutableSegment {
* general segment scanner. * general segment scanner.
* @return a special scanner for the MemStoreSnapshot object * @return a special scanner for the MemStoreSnapshot object
*/ */
@Override
public KeyValueScanner getSnapshotScanner() { public KeyValueScanner getSnapshotScanner() {
return getScanner(Long.MAX_VALUE, Long.MAX_VALUE); return getScanner(Long.MAX_VALUE, Long.MAX_VALUE);
} }
@ -84,6 +84,7 @@ public class CompositeImmutableSegment extends ImmutableSegment {
/** /**
* @return whether the segment has any cells * @return whether the segment has any cells
*/ */
@Override
public boolean isEmpty() { public boolean isEmpty() {
for (ImmutableSegment s : segments) { for (ImmutableSegment s : segments) {
if (!s.isEmpty()) return false; if (!s.isEmpty()) return false;
@ -94,6 +95,7 @@ public class CompositeImmutableSegment extends ImmutableSegment {
/** /**
* @return number of cells in segment * @return number of cells in segment
*/ */
@Override
public int getCellsCount() { public int getCellsCount() {
int result = 0; int result = 0;
for (ImmutableSegment s : segments) { for (ImmutableSegment s : segments) {
@ -105,6 +107,7 @@ public class CompositeImmutableSegment extends ImmutableSegment {
/** /**
* @return the first cell in the segment that has equal or greater key than the given cell * @return the first cell in the segment that has equal or greater key than the given cell
*/ */
@Override
public Cell getFirstAfter(Cell cell) { public Cell getFirstAfter(Cell cell) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@ -112,6 +115,7 @@ public class CompositeImmutableSegment extends ImmutableSegment {
/** /**
* Closing a segment before it is being discarded * Closing a segment before it is being discarded
*/ */
@Override
public void close() { public void close() {
for (ImmutableSegment s : segments) { for (ImmutableSegment s : segments) {
s.close(); s.close();
@ -123,14 +127,17 @@ public class CompositeImmutableSegment extends ImmutableSegment {
* otherwise the given cell is returned * otherwise the given cell is returned
* @return either the given cell or its clone * @return either the given cell or its clone
*/ */
@Override
public Cell maybeCloneWithAllocator(Cell cell) { public Cell maybeCloneWithAllocator(Cell cell) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@Override
public boolean shouldSeek(Scan scan, long oldestUnexpiredTS){ public boolean shouldSeek(Scan scan, long oldestUnexpiredTS){
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@Override
public long getMinTimestamp(){ public long getMinTimestamp(){
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@ -139,6 +146,7 @@ public class CompositeImmutableSegment extends ImmutableSegment {
* Creates the scanner for the given read point * Creates the scanner for the given read point
* @return a scanner for the given read point * @return a scanner for the given read point
*/ */
@Override
public KeyValueScanner getScanner(long readPoint) { public KeyValueScanner getScanner(long readPoint) {
// Long.MAX_VALUE is DEFAULT_SCANNER_ORDER // Long.MAX_VALUE is DEFAULT_SCANNER_ORDER
return getScanner(readPoint,Long.MAX_VALUE); return getScanner(readPoint,Long.MAX_VALUE);
@ -148,6 +156,7 @@ public class CompositeImmutableSegment extends ImmutableSegment {
* Creates the scanner for the given read point, and a specific order in a list * Creates the scanner for the given read point, and a specific order in a list
* @return a scanner for the given read point * @return a scanner for the given read point
*/ */
@Override
public KeyValueScanner getScanner(long readPoint, long order) { public KeyValueScanner getScanner(long readPoint, long order) {
KeyValueScanner resultScanner; KeyValueScanner resultScanner;
List<KeyValueScanner> list = new ArrayList<>(segments.size()); List<KeyValueScanner> list = new ArrayList<>(segments.size());
@ -164,6 +173,7 @@ public class CompositeImmutableSegment extends ImmutableSegment {
return resultScanner; return resultScanner;
} }
@Override
public boolean isTagsPresent() { public boolean isTagsPresent() {
for (ImmutableSegment s : segments) { for (ImmutableSegment s : segments) {
if (s.isTagsPresent()) return true; if (s.isTagsPresent()) return true;
@ -171,10 +181,12 @@ public class CompositeImmutableSegment extends ImmutableSegment {
return false; return false;
} }
@Override
public void incScannerCount() { public void incScannerCount() {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@Override
public void decScannerCount() { public void decScannerCount() {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@ -184,7 +196,7 @@ public class CompositeImmutableSegment extends ImmutableSegment {
* immutable CellSet after its creation in immutable segment constructor * immutable CellSet after its creation in immutable segment constructor
* @return this object * @return this object
*/ */
@Override
protected CompositeImmutableSegment setCellSet(CellSet cellSetOld, CellSet cellSetNew) { protected CompositeImmutableSegment setCellSet(CellSet cellSetOld, CellSet cellSetNew) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@ -192,17 +204,19 @@ public class CompositeImmutableSegment extends ImmutableSegment {
/** /**
* @return Sum of all cell sizes. * @return Sum of all cell sizes.
*/ */
@Override
public long keySize() { public long keySize() {
return this.keySize; return this.keySize;
} }
/** /**
* @return The heap overhead of this segment. * @return The heap size of this segment.
*/ */
public long heapOverhead() { @Override
public long heapSize() {
long result = 0; long result = 0;
for (ImmutableSegment s : segments) { for (ImmutableSegment s : segments) {
result += s.heapOverhead(); result += s.heapSize();
} }
return result; return result;
} }
@ -210,39 +224,43 @@ public class CompositeImmutableSegment extends ImmutableSegment {
/** /**
* Updates the heap size counter of the segment by the given delta * Updates the heap size counter of the segment by the given delta
*/ */
@Override
protected void incSize(long delta, long heapOverhead) { protected void incSize(long delta, long heapOverhead) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
protected void incHeapOverheadSize(long delta) { @Override
throw new IllegalStateException("Not supported by CompositeImmutableScanner");
}
public long getMinSequenceId() { public long getMinSequenceId() {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@Override
public TimeRangeTracker getTimeRangeTracker() { public TimeRangeTracker getTimeRangeTracker() {
return this.timeRangeTracker; return this.timeRangeTracker;
} }
//*** Methods for SegmentsScanner //*** Methods for SegmentsScanner
@Override
public Cell last() { public Cell last() {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@Override
public Iterator<Cell> iterator() { public Iterator<Cell> iterator() {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@Override
public SortedSet<Cell> headSet(Cell firstKeyOnRow) { public SortedSet<Cell> headSet(Cell firstKeyOnRow) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@Override
public int compare(Cell left, Cell right) { public int compare(Cell left, Cell right) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@Override
public int compareRows(Cell left, Cell right) { public int compareRows(Cell left, Cell right) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@ -250,36 +268,28 @@ public class CompositeImmutableSegment extends ImmutableSegment {
/** /**
* @return a set of all cells in the segment * @return a set of all cells in the segment
*/ */
@Override
protected CellSet getCellSet() { protected CellSet getCellSet() {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
/** @Override
* Returns the Cell comparator used by this segment
* @return the Cell comparator used by this segment
*/
protected CellComparator getComparator() {
return comparator;
}
protected void internalAdd(Cell cell, boolean mslabUsed, MemstoreSize memstoreSize) { protected void internalAdd(Cell cell, boolean mslabUsed, MemstoreSize memstoreSize) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
@Override
protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed, protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed,
MemstoreSize memstoreSize) { MemstoreSize memstoreSize) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }
protected long heapOverheadChange(Cell cell, boolean succ) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner");
}
/** /**
* Returns a subset of the segment cell set, which starts with the given cell * Returns a subset of the segment cell set, which starts with the given cell
* @param firstCell a cell in the segment * @param firstCell a cell in the segment
* @return a subset of the segment cell set, which starts with the given cell * @return a subset of the segment cell set, which starts with the given cell
*/ */
@Override
protected SortedSet<Cell> tailSet(Cell firstCell) { protected SortedSet<Cell> tailSet(Cell firstCell) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner"); throw new IllegalStateException("Not supported by CompositeImmutableScanner");
} }

14
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
View File

@ -111,7 +111,7 @@ public class DefaultMemStore extends AbstractMemStore {
public MemstoreSize getFlushableSize() { public MemstoreSize getFlushableSize() {
MemstoreSize snapshotSize = getSnapshotSize(); MemstoreSize snapshotSize = getSnapshotSize();
return snapshotSize.getDataSize() > 0 ? snapshotSize return snapshotSize.getDataSize() > 0 ? snapshotSize
: new MemstoreSize(keySize(), heapOverhead()); : new MemstoreSize(keySize(), heapSize());
} }
@Override @Override
@ -120,8 +120,8 @@ public class DefaultMemStore extends AbstractMemStore {
} }
@Override @Override
protected long heapOverhead() { protected long heapSize() {
return this.active.heapOverhead(); return this.active.heapSize();
} }
@Override @Override
@ -160,7 +160,7 @@ public class DefaultMemStore extends AbstractMemStore {
@Override @Override
public MemstoreSize size() { public MemstoreSize size() {
return new MemstoreSize(this.active.keySize(), this.active.heapOverhead()); return new MemstoreSize(this.active.keySize(), this.active.heapSize());
} }
/** /**
@ -205,12 +205,12 @@ public class DefaultMemStore extends AbstractMemStore {
memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memstoreSize); memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memstoreSize);
} }
LOG.info("memstore1 estimated size=" LOG.info("memstore1 estimated size="
+ (memstoreSize.getDataSize() + memstoreSize.getHeapOverhead())); + (memstoreSize.getDataSize() + memstoreSize.getHeapSize()));
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memstoreSize); memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memstoreSize);
} }
LOG.info("memstore1 estimated size (2nd loading of same data)=" LOG.info("memstore1 estimated size (2nd loading of same data)="
+ (memstoreSize.getDataSize() + memstoreSize.getHeapOverhead())); + (memstoreSize.getDataSize() + memstoreSize.getHeapSize()));
// Make a variably sized memstore. // Make a variably sized memstore.
DefaultMemStore memstore2 = new DefaultMemStore(); DefaultMemStore memstore2 = new DefaultMemStore();
memstoreSize = new MemstoreSize(); memstoreSize = new MemstoreSize();
@ -218,7 +218,7 @@ public class DefaultMemStore extends AbstractMemStore {
memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, new byte[i]), memstoreSize); memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, new byte[i]), memstoreSize);
} }
LOG.info("memstore2 estimated size=" LOG.info("memstore2 estimated size="
+ (memstoreSize.getDataSize() + memstoreSize.getHeapOverhead())); + (memstoreSize.getDataSize() + memstoreSize.getHeapSize()));
final int seconds = 30; final int seconds = 30;
LOG.info("Waiting " + seconds + " seconds while heap dump is taken"); LOG.info("Waiting " + seconds + " seconds while heap dump is taken");
LOG.info("Exiting."); LOG.info("Exiting.");

10
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -1689,7 +1689,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
this.closed.set(true); this.closed.set(true);
if (!canFlush) { if (!canFlush) {
this.decrMemstoreSize(new MemstoreSize(memstoreDataSize.get(), getMemstoreHeapOverhead())); this.decrMemstoreSize(new MemstoreSize(memstoreDataSize.get(), getMemstoreHeapSize()));
} else if (memstoreDataSize.get() != 0) { } else if (memstoreDataSize.get() != 0) {
LOG.error("Memstore size is " + memstoreDataSize.get()); LOG.error("Memstore size is " + memstoreDataSize.get());
} }
@ -1713,12 +1713,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
} }
} }
private long getMemstoreHeapOverhead() { private long getMemstoreHeapSize() {
long overhead = 0; long size = 0;
for (Store s : this.stores.values()) { for (Store s : this.stores.values()) {
overhead += s.getSizeOfMemStore().getHeapOverhead(); size += s.getSizeOfMemStore().getHeapSize();
} }
return overhead; return size;
} }
@Override @Override

8
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
View File

@ -380,7 +380,7 @@ public class HStore implements Store {
@Deprecated @Deprecated
public long getFlushableSize() { public long getFlushableSize() {
MemstoreSize size = getSizeToFlush(); MemstoreSize size = getSizeToFlush();
return size.getDataSize() + size.getHeapOverhead(); return size.getHeapSize();
} }
@Override @Override
@ -392,7 +392,7 @@ public class HStore implements Store {
@Deprecated @Deprecated
public long getSnapshotSize() { public long getSnapshotSize() {
MemstoreSize size = getSizeOfSnapshot(); MemstoreSize size = getSizeOfSnapshot();
return size.getDataSize() + size.getHeapOverhead(); return size.getHeapSize();
} }
@Override @Override
@ -2067,7 +2067,7 @@ public class HStore implements Store {
@Deprecated @Deprecated
public long getMemStoreSize() { public long getMemStoreSize() {
MemstoreSize size = getSizeOfMemStore(); MemstoreSize size = getSizeOfMemStore();
return size.getDataSize() + size.getHeapOverhead(); return size.getHeapSize();
} }
@Override @Override
@ -2298,7 +2298,7 @@ public class HStore implements Store {
@Override @Override
public long heapSize() { public long heapSize() {
MemstoreSize memstoreSize = this.memstore.size(); MemstoreSize memstoreSize = this.memstore.size();
return DEEP_OVERHEAD + memstoreSize.getDataSize() + memstoreSize.getHeapOverhead(); return DEEP_OVERHEAD + memstoreSize.getHeapSize();
} }
@Override @Override

4
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
View File

@ -20,8 +20,6 @@ package org.apache.hadoop.hbase.regionserver;
import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY; import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryType;
import java.lang.management.MemoryUsage; import java.lang.management.MemoryUsage;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -326,7 +324,7 @@ public class HeapMemoryManager {
// TODO : add support for offheap metrics // TODO : add support for offheap metrics
tunerContext.setCurBlockCacheUsed((float) blockCache.getCurrentSize() / maxHeapSize); tunerContext.setCurBlockCacheUsed((float) blockCache.getCurrentSize() / maxHeapSize);
metricsHeapMemoryManager.setCurBlockCacheSizeGauge(blockCache.getCurrentSize()); metricsHeapMemoryManager.setCurBlockCacheSizeGauge(blockCache.getCurrentSize());
long globalMemstoreHeapSize = regionServerAccounting.getGlobalMemstoreSize(); long globalMemstoreHeapSize = regionServerAccounting.getGlobalMemstoreHeapSize();
tunerContext.setCurMemStoreUsed((float) globalMemstoreHeapSize / maxHeapSize); tunerContext.setCurMemStoreUsed((float) globalMemstoreHeapSize / maxHeapSize);
metricsHeapMemoryManager.setCurMemStoreSizeGauge(globalMemstoreHeapSize); metricsHeapMemoryManager.setCurMemStoreSizeGauge(globalMemstoreHeapSize);
tunerContext.setCurBlockCacheSize(blockCachePercent); tunerContext.setCurBlockCacheSize(blockCachePercent);

16
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java
View File

@ -21,8 +21,7 @@ package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Scan;
@ -119,14 +118,13 @@ public class ImmutableSegment extends Segment {
super(new CellSet(comparator), // initiailize the CellSet with empty CellSet super(new CellSet(comparator), // initiailize the CellSet with empty CellSet
comparator, memStoreLAB); comparator, memStoreLAB);
type = Type.SKIPLIST_MAP_BASED; type = Type.SKIPLIST_MAP_BASED;
MemstoreSize memstoreSize = new MemstoreSize();
while (iterator.hasNext()) { while (iterator.hasNext()) {
Cell c = iterator.next(); Cell c = iterator.next();
// The scanner is doing all the elimination logic // The scanner is doing all the elimination logic
// now we just copy it to the new segment // now we just copy it to the new segment
Cell newKV = maybeCloneWithAllocator(c); Cell newKV = maybeCloneWithAllocator(c);
boolean usedMSLAB = (newKV != c); boolean usedMSLAB = (newKV != c);
internalAdd(newKV, usedMSLAB, memstoreSize); internalAdd(newKV, usedMSLAB, null);
} }
this.timeRange = this.timeRangeTracker == null ? null : this.timeRangeTracker.toTimeRange(); this.timeRange = this.timeRangeTracker == null ? null : this.timeRangeTracker.toTimeRange();
} }
@ -226,17 +224,13 @@ public class ImmutableSegment extends Segment {
} }
@Override @Override
protected long heapOverheadChange(Cell cell, boolean succ) { protected long heapSizeChange(Cell cell, boolean succ) {
if (succ) { if (succ) {
switch (this.type) { switch (this.type) {
case SKIPLIST_MAP_BASED: case SKIPLIST_MAP_BASED:
return super.heapOverheadChange(cell, succ); return super.heapSizeChange(cell, succ);
case ARRAY_MAP_BASED: case ARRAY_MAP_BASED:
if (cell instanceof ExtendedCell) { return ClassSize.align(ClassSize.CELL_ARRAY_MAP_ENTRY + CellUtil.estimatedHeapSizeOf(cell));
return ClassSize
.align(ClassSize.CELL_ARRAY_MAP_ENTRY + ((ExtendedCell) cell).heapOverhead());
}
return ClassSize.align(ClassSize.CELL_ARRAY_MAP_ENTRY + KeyValue.FIXED_OVERHEAD);
} }
} }
return 0; return 0;

30
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
View File

@ -186,11 +186,11 @@ class MemStoreFlusher implements FlushRequester {
(bestRegionReplica.getMemstoreSize() (bestRegionReplica.getMemstoreSize()
> secondaryMultiplier * regionToFlush.getMemstoreSize()))) { > secondaryMultiplier * regionToFlush.getMemstoreSize()))) {
LOG.info("Refreshing storefiles of region " + bestRegionReplica LOG.info("Refreshing storefiles of region " + bestRegionReplica
+ " due to global heap pressure. Total memstore size=" + " due to global heap pressure. Total memstore datasize="
+ StringUtils + StringUtils
.humanReadableInt(server.getRegionServerAccounting().getGlobalMemstoreDataSize()) .humanReadableInt(server.getRegionServerAccounting().getGlobalMemstoreDataSize())
+ " memstore heap overhead=" + StringUtils.humanReadableInt( + " memstore heap size=" + StringUtils.humanReadableInt(
server.getRegionServerAccounting().getGlobalMemstoreHeapOverhead())); server.getRegionServerAccounting().getGlobalMemstoreHeapSize()));
flushedOne = refreshStoreFilesAndReclaimMemory(bestRegionReplica); flushedOne = refreshStoreFilesAndReclaimMemory(bestRegionReplica);
if (!flushedOne) { if (!flushedOne) {
LOG.info("Excluding secondary region " + bestRegionReplica + LOG.info("Excluding secondary region " + bestRegionReplica +
@ -587,14 +587,20 @@ class MemStoreFlusher implements FlushRequester {
if (!blocked) { if (!blocked) {
startTime = EnvironmentEdgeManager.currentTime(); startTime = EnvironmentEdgeManager.currentTime();
if (!server.getRegionServerAccounting().isOffheap()) { if (!server.getRegionServerAccounting().isOffheap()) {
logMsg("the global memstore size", "global memstore heap overhead"); logMsg("global memstore heapsize",
server.getRegionServerAccounting().getGlobalMemstoreHeapSize(),
server.getRegionServerAccounting().getGlobalMemstoreLimit());
} else { } else {
switch (flushType) { switch (flushType) {
case ABOVE_OFFHEAP_HIGHER_MARK: case ABOVE_OFFHEAP_HIGHER_MARK:
logMsg("the global offheap memstore size", "global memstore heap overhead"); logMsg("the global offheap memstore datasize",
server.getRegionServerAccounting().getGlobalMemstoreDataSize(),
server.getRegionServerAccounting().getGlobalMemstoreLimit());
break; break;
case ABOVE_ONHEAP_HIGHER_MARK: case ABOVE_ONHEAP_HIGHER_MARK:
logMsg("global memstore heap overhead", ""); logMsg("global memstore heapsize",
server.getRegionServerAccounting().getGlobalMemstoreHeapSize(),
server.getRegionServerAccounting().getGlobalOnHeapMemstoreLimit());
break; break;
default: default:
break; break;
@ -635,16 +641,10 @@ class MemStoreFlusher implements FlushRequester {
scope.close(); scope.close();
} }
private void logMsg(String string1, String string2) { private void logMsg(String string1, long val, long max) {
LOG.info("Blocking updates on " + server.toString() + ": " + string1 + " " LOG.info("Blocking updates on " + server.toString() + ": " + string1 + " "
+ TraditionalBinaryPrefix + TraditionalBinaryPrefix.long2String(val, "", 1) + " is >= than blocking "
.long2String(server.getRegionServerAccounting().getGlobalMemstoreDataSize(), "", 1) + TraditionalBinaryPrefix.long2String(max, "", 1) + " size");
+ " + " + string2 + " "
+ TraditionalBinaryPrefix
.long2String(server.getRegionServerAccounting().getGlobalMemstoreHeapOverhead(), "", 1)
+ " is >= than blocking " + TraditionalBinaryPrefix.long2String(
server.getRegionServerAccounting().getGlobalMemstoreLimit(), "", 1)
+ " size");
} }
@Override @Override

8
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java
View File

@ -29,7 +29,7 @@ public class MemStoreSnapshot {
private final long id; private final long id;
private final int cellsCount; private final int cellsCount;
private final long dataSize; private final long dataSize;
private final long heapOverhead; private final long heapSize;
private final TimeRangeTracker timeRangeTracker; private final TimeRangeTracker timeRangeTracker;
private final KeyValueScanner scanner; private final KeyValueScanner scanner;
private final boolean tagsPresent; private final boolean tagsPresent;
@ -38,7 +38,7 @@ public class MemStoreSnapshot {
this.id = id; this.id = id;
this.cellsCount = snapshot.getCellsCount(); this.cellsCount = snapshot.getCellsCount();
this.dataSize = snapshot.keySize(); this.dataSize = snapshot.keySize();
this.heapOverhead = snapshot.heapOverhead(); this.heapSize = snapshot.heapSize();
this.timeRangeTracker = snapshot.getTimeRangeTracker(); this.timeRangeTracker = snapshot.getTimeRangeTracker();
this.scanner = snapshot.getSnapshotScanner(); this.scanner = snapshot.getSnapshotScanner();
this.tagsPresent = snapshot.isTagsPresent(); this.tagsPresent = snapshot.isTagsPresent();
@ -65,8 +65,8 @@ public class MemStoreSnapshot {
return dataSize; return dataSize;
} }
public long getHeapOverhead() { public long getHeapSize() {
return this.heapOverhead; return this.heapSize;
} }
/** /**

55
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreSize.java
View File

@ -20,26 +20,29 @@ package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceAudience;
/** /**
* Wraps the data size part and heap overhead of the memstore. * Wraps the data size part and total heap space occupied by the memstore.
*/ */
@InterfaceAudience.Private @InterfaceAudience.Private
public class MemstoreSize { public class MemstoreSize {
// 'dataSize' tracks the Cell's data bytes size alone (Key bytes, value bytes). A cell's data can
// be in on heap or off heap area depending on the MSLAB and its configuration to be using on heap
// or off heap LABs
private long dataSize; private long dataSize;
private long heapOverhead; // 'heapSize' tracks all Cell's heap size occupancy. This will include Cell POJO heap overhead.
// When Cells in on heap area, this will include the cells data size as well.
private long heapSize;
final private boolean isEmpty; final private boolean isEmpty;
static final MemstoreSize EMPTY_SIZE = new MemstoreSize(true);
public MemstoreSize() { public MemstoreSize() {
dataSize = 0; dataSize = 0;
heapOverhead = 0; heapSize = 0;
isEmpty = false; isEmpty = false;
} }
public MemstoreSize(boolean isEmpty) { public MemstoreSize(boolean isEmpty) {
dataSize = 0; dataSize = 0;
heapOverhead = 0; heapSize = 0;
this.isEmpty = isEmpty; this.isEmpty = isEmpty;
} }
@ -47,40 +50,38 @@ public class MemstoreSize {
return isEmpty; return isEmpty;
} }
public MemstoreSize(long dataSize, long heapOverhead) { public MemstoreSize(long dataSize, long heapSize) {
this.dataSize = dataSize; this.dataSize = dataSize;
this.heapOverhead = heapOverhead; this.heapSize = heapSize;
this.isEmpty = false; this.isEmpty = false;
} }
public void incMemstoreSize(long dataSize, long heapOverhead) { public void incMemstoreSize(long dataSizeDelta, long heapSizeDelta) {
this.dataSize += dataSize; this.dataSize += dataSizeDelta;
this.heapOverhead += heapOverhead; this.heapSize += heapSizeDelta;
} }
public void incMemstoreSize(MemstoreSize size) { public void incMemstoreSize(MemstoreSize delta) {
this.dataSize += size.dataSize; this.dataSize += delta.dataSize;
this.heapOverhead += size.heapOverhead; this.heapSize += delta.heapSize;
} }
public void decMemstoreSize(long dataSize, long heapOverhead) { public void decMemstoreSize(long dataSizeDelta, long heapSizeDelta) {
this.dataSize -= dataSize; this.dataSize -= dataSizeDelta;
this.heapOverhead -= heapOverhead; this.heapSize -= heapSizeDelta;
} }
public void decMemstoreSize(MemstoreSize size) { public void decMemstoreSize(MemstoreSize delta) {
this.dataSize -= size.dataSize; this.dataSize -= delta.dataSize;
this.heapOverhead -= size.heapOverhead; this.heapSize -= delta.heapSize;
} }
public long getDataSize() { public long getDataSize() {
return isEmpty ? 0 : dataSize; return isEmpty ? 0 : dataSize;
} }
public long getHeapOverhead() { public long getHeapSize() {
return isEmpty ? 0 : heapSize;
return isEmpty ? 0 : heapOverhead;
} }
@Override @Override
@ -89,18 +90,18 @@ public class MemstoreSize {
return false; return false;
} }
MemstoreSize other = (MemstoreSize) obj; MemstoreSize other = (MemstoreSize) obj;
return getDataSize() == other.dataSize && getHeapOverhead() == other.heapOverhead; return this.dataSize == other.dataSize && this.heapSize == other.heapSize;
} }
@Override @Override
public int hashCode() { public int hashCode() {
long h = 13 * this.dataSize; long h = 13 * this.dataSize;
h = h + 14 * this.heapOverhead; h = h + 14 * this.heapSize;
return (int) h; return (int) h;
} }
@Override @Override
public String toString() { public String toString() {
return "dataSize=" + this.dataSize + " , heapOverhead=" + this.heapOverhead; return "dataSize=" + this.dataSize + " , heapSize=" + this.heapSize;
} }
} }

6
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
View File

@ -85,10 +85,10 @@ public class MutableSegment extends Segment {
// decreased there as 0 only. Just keeping it as existing code now. We need to know the // decreased there as 0 only. Just keeping it as existing code now. We need to know the
// removed cell is from MSLAB or not. Will do once HBASE-16438 is in // removed cell is from MSLAB or not. Will do once HBASE-16438 is in
int cellLen = getCellLength(cur); int cellLen = getCellLength(cur);
long heapOverheadDelta = heapOverheadChange(cur, true); long heapSize = heapSizeChange(cur, true);
this.incSize(-cellLen, -heapOverheadDelta); this.incSize(-cellLen, -heapSize);
if (memstoreSize != null) { if (memstoreSize != null) {
memstoreSize.decMemstoreSize(cellLen, heapOverheadDelta); memstoreSize.decMemstoreSize(cellLen, heapSize);
} }
it.remove(); it.remove();
} else { } else {

56
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
View File

@ -39,16 +39,16 @@ public class RegionServerAccounting {
// memstore data size // memstore data size
private final AtomicLong globalMemstoreDataSize = new AtomicLong(0); private final AtomicLong globalMemstoreDataSize = new AtomicLong(0);
// memstore heap over head size // memstore heap size. When off heap MSLAB in place, this will be only heap overhead of the Cell
private final AtomicLong globalMemstoreHeapOverhead = new AtomicLong(0); // POJOs and entry overhead of them onto memstore. When on heap MSLAB, this will be include heap
// overhead as well as the cell data size. Ya cell data is in on heap area only then.
private final AtomicLong globalMemstoreHeapSize = new AtomicLong(0);
// Store the edits size during replaying WAL. Use this to roll back the // Store the edits size during replaying WAL. Use this to roll back the
// global memstore size once a region opening failed. // global memstore size once a region opening failed.
private final ConcurrentMap<byte[], MemstoreSize> replayEditsPerRegion = private final ConcurrentMap<byte[], MemstoreSize> replayEditsPerRegion =
new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR); new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
private final Configuration conf;
private long globalMemStoreLimit; private long globalMemStoreLimit;
private final float globalMemStoreLimitLowMarkPercent; private final float globalMemStoreLimitLowMarkPercent;
private long globalMemStoreLimitLowMark; private long globalMemStoreLimitLowMark;
@ -57,7 +57,6 @@ public class RegionServerAccounting {
private long globalOnHeapMemstoreLimitLowMark; private long globalOnHeapMemstoreLimitLowMark;
public RegionServerAccounting(Configuration conf) { public RegionServerAccounting(Configuration conf) {
this.conf = conf;
Pair<Long, MemoryType> globalMemstoreSizePair = MemorySizeUtil.getGlobalMemstoreSize(conf); Pair<Long, MemoryType> globalMemstoreSizePair = MemorySizeUtil.getGlobalMemstoreSize(conf);
this.globalMemStoreLimit = globalMemstoreSizePair.getFirst(); this.globalMemStoreLimit = globalMemstoreSizePair.getFirst();
this.memType = globalMemstoreSizePair.getSecond(); this.memType = globalMemstoreSizePair.getSecond();
@ -79,16 +78,16 @@ public class RegionServerAccounting {
(long) (this.globalOnHeapMemstoreLimit * this.globalMemStoreLimitLowMarkPercent); (long) (this.globalOnHeapMemstoreLimit * this.globalMemStoreLimitLowMarkPercent);
} }
public long getGlobalMemstoreLimit() { long getGlobalMemstoreLimit() {
return this.globalMemStoreLimit; return this.globalMemStoreLimit;
} }
public long getOnheapGlobalMemstoreLimit() { long getGlobalOnHeapMemstoreLimit() {
return this.globalOnHeapMemstoreLimit; return this.globalOnHeapMemstoreLimit;
} }
// Called by the tuners. // Called by the tuners.
public void setGlobalMemstoreLimits(long newGlobalMemstoreLimit) { void setGlobalMemstoreLimits(long newGlobalMemstoreLimit) {
if (this.memType == MemoryType.HEAP) { if (this.memType == MemoryType.HEAP) {
this.globalMemStoreLimit = newGlobalMemstoreLimit; this.globalMemStoreLimit = newGlobalMemstoreLimit;
this.globalMemStoreLimitLowMark = this.globalMemStoreLimitLowMark =
@ -100,15 +99,15 @@ public class RegionServerAccounting {
} }
} }
public boolean isOffheap() { boolean isOffheap() {
return this.memType == MemoryType.NON_HEAP; return this.memType == MemoryType.NON_HEAP;
} }
public long getGlobalMemstoreLimitLowMark() { long getGlobalMemstoreLimitLowMark() {
return this.globalMemStoreLimitLowMark; return this.globalMemStoreLimitLowMark;
} }
public float getGlobalMemstoreLimitLowMarkPercent() { float getGlobalMemstoreLimitLowMarkPercent() {
return this.globalMemStoreLimitLowMarkPercent; return this.globalMemStoreLimitLowMarkPercent;
} }
@ -118,24 +117,12 @@ public class RegionServerAccounting {
public long getGlobalMemstoreDataSize() { public long getGlobalMemstoreDataSize() {
return globalMemstoreDataSize.get(); return globalMemstoreDataSize.get();
} }
/**
* @return the global memstore heap overhead size in the RegionServer
*/
public long getGlobalMemstoreHeapOverhead() {
return this.globalMemstoreHeapOverhead.get();
}
/** /**
* @return the global memstore data size and heap overhead size for an onheap memstore * @return the global memstore heap size in the RegionServer
* whereas return the heap overhead size for an offheap memstore
*/ */
public long getGlobalMemstoreSize() { public long getGlobalMemstoreHeapSize() {
if (isOffheap()) { return this.globalMemstoreHeapSize.get();
// get only the heap overhead for offheap memstore
return getGlobalMemstoreHeapOverhead();
} else {
return getGlobalMemstoreDataSize() + getGlobalMemstoreHeapOverhead();
}
} }
/** /**
@ -144,12 +131,12 @@ public class RegionServerAccounting {
*/ */
public void incGlobalMemstoreSize(MemstoreSize memStoreSize) { public void incGlobalMemstoreSize(MemstoreSize memStoreSize) {
globalMemstoreDataSize.addAndGet(memStoreSize.getDataSize()); globalMemstoreDataSize.addAndGet(memStoreSize.getDataSize());
globalMemstoreHeapOverhead.addAndGet(memStoreSize.getHeapOverhead()); globalMemstoreHeapSize.addAndGet(memStoreSize.getHeapSize());
} }
public void decGlobalMemstoreSize(MemstoreSize memStoreSize) { public void decGlobalMemstoreSize(MemstoreSize memStoreSize) {
globalMemstoreDataSize.addAndGet(-memStoreSize.getDataSize()); globalMemstoreDataSize.addAndGet(-memStoreSize.getDataSize());
globalMemstoreHeapOverhead.addAndGet(-memStoreSize.getHeapOverhead()); globalMemstoreHeapSize.addAndGet(-memStoreSize.getHeapSize());
} }
/** /**
@ -160,7 +147,7 @@ public class RegionServerAccounting {
// for onheap memstore we check if the global memstore size and the // for onheap memstore we check if the global memstore size and the
// global heap overhead is greater than the global memstore limit // global heap overhead is greater than the global memstore limit
if (memType == MemoryType.HEAP) { if (memType == MemoryType.HEAP) {
if (getGlobalMemstoreDataSize() + getGlobalMemstoreHeapOverhead() >= globalMemStoreLimit) { if (getGlobalMemstoreHeapSize() >= globalMemStoreLimit) {
return FlushType.ABOVE_ONHEAP_HIGHER_MARK; return FlushType.ABOVE_ONHEAP_HIGHER_MARK;
} }
} else { } else {
@ -175,7 +162,7 @@ public class RegionServerAccounting {
// Indicates that global memstore size is above the configured // Indicates that global memstore size is above the configured
// 'hbase.regionserver.offheap.global.memstore.size' // 'hbase.regionserver.offheap.global.memstore.size'
return FlushType.ABOVE_OFFHEAP_HIGHER_MARK; return FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
} else if (getGlobalMemstoreHeapOverhead() >= this.globalOnHeapMemstoreLimit) { } else if (getGlobalMemstoreHeapSize() >= this.globalOnHeapMemstoreLimit) {
// Indicates that the offheap memstore's heap overhead is greater than the // Indicates that the offheap memstore's heap overhead is greater than the
// configured 'hbase.regionserver.global.memstore.size'. // configured 'hbase.regionserver.global.memstore.size'.
return FlushType.ABOVE_ONHEAP_HIGHER_MARK; return FlushType.ABOVE_ONHEAP_HIGHER_MARK;
@ -191,7 +178,7 @@ public class RegionServerAccounting {
// for onheap memstore we check if the global memstore size and the // for onheap memstore we check if the global memstore size and the
// global heap overhead is greater than the global memstore lower mark limit // global heap overhead is greater than the global memstore lower mark limit
if (memType == MemoryType.HEAP) { if (memType == MemoryType.HEAP) {
if (getGlobalMemstoreDataSize() + getGlobalMemstoreHeapOverhead() >= globalMemStoreLimitLowMark) { if (getGlobalMemstoreHeapSize() >= globalMemStoreLimitLowMark) {
return FlushType.ABOVE_ONHEAP_LOWER_MARK; return FlushType.ABOVE_ONHEAP_LOWER_MARK;
} }
} else { } else {
@ -199,7 +186,7 @@ public class RegionServerAccounting {
// Indicates that the offheap memstore's data size is greater than the global memstore // Indicates that the offheap memstore's data size is greater than the global memstore
// lower limit // lower limit
return FlushType.ABOVE_OFFHEAP_LOWER_MARK; return FlushType.ABOVE_OFFHEAP_LOWER_MARK;
} else if (getGlobalMemstoreHeapOverhead() >= globalOnHeapMemstoreLimitLowMark) { } else if (getGlobalMemstoreHeapSize() >= globalOnHeapMemstoreLimitLowMark) {
// Indicates that the offheap memstore's heap overhead is greater than the global memstore // Indicates that the offheap memstore's heap overhead is greater than the global memstore
// onheap lower limit // onheap lower limit
return FlushType.ABOVE_ONHEAP_LOWER_MARK; return FlushType.ABOVE_ONHEAP_LOWER_MARK;
@ -215,11 +202,10 @@ public class RegionServerAccounting {
*/ */
public double getFlushPressure() { public double getFlushPressure() {
if (memType == MemoryType.HEAP) { if (memType == MemoryType.HEAP) {
return (getGlobalMemstoreDataSize() + getGlobalMemstoreHeapOverhead()) * 1.0 return (getGlobalMemstoreHeapSize()) * 1.0 / globalMemStoreLimitLowMark;
/ globalMemStoreLimitLowMark;
} else { } else {
return Math.max(getGlobalMemstoreDataSize() * 1.0 / globalMemStoreLimitLowMark, return Math.max(getGlobalMemstoreDataSize() * 1.0 / globalMemStoreLimitLowMark,
getGlobalMemstoreHeapOverhead() * 1.0 / globalOnHeapMemstoreLimitLowMark); getGlobalMemstoreHeapSize() * 1.0 / globalOnHeapMemstoreLimitLowMark);
} }
} }

50
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
View File

@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceAudience;
@ -62,7 +62,7 @@ public abstract class Segment {
// Sum of sizes of all Cells added to this Segment. Cell's heapSize is considered. This is not // Sum of sizes of all Cells added to this Segment. Cell's heapSize is considered. This is not
// including the heap overhead of this class. // including the heap overhead of this class.
protected final AtomicLong dataSize; protected final AtomicLong dataSize;
protected final AtomicLong heapOverhead; protected final AtomicLong heapSize;
protected final TimeRangeTracker timeRangeTracker; protected final TimeRangeTracker timeRangeTracker;
protected volatile boolean tagsPresent; protected volatile boolean tagsPresent;
@ -71,7 +71,7 @@ public abstract class Segment {
protected Segment(CellComparator comparator) { protected Segment(CellComparator comparator) {
this.comparator = comparator; this.comparator = comparator;
this.dataSize = new AtomicLong(0); this.dataSize = new AtomicLong(0);
this.heapOverhead = new AtomicLong(0); this.heapSize = new AtomicLong(0);
this.timeRangeTracker = new TimeRangeTracker(); this.timeRangeTracker = new TimeRangeTracker();
} }
@ -82,7 +82,7 @@ public abstract class Segment {
this.minSequenceId = Long.MAX_VALUE; this.minSequenceId = Long.MAX_VALUE;
this.memStoreLAB = memStoreLAB; this.memStoreLAB = memStoreLAB;
this.dataSize = new AtomicLong(0); this.dataSize = new AtomicLong(0);
this.heapOverhead = new AtomicLong(0); this.heapSize = new AtomicLong(0);
this.tagsPresent = false; this.tagsPresent = false;
this.timeRangeTracker = new TimeRangeTracker(); this.timeRangeTracker = new TimeRangeTracker();
} }
@ -93,7 +93,7 @@ public abstract class Segment {
this.minSequenceId = segment.getMinSequenceId(); this.minSequenceId = segment.getMinSequenceId();
this.memStoreLAB = segment.getMemStoreLAB(); this.memStoreLAB = segment.getMemStoreLAB();
this.dataSize = new AtomicLong(segment.keySize()); this.dataSize = new AtomicLong(segment.keySize());
this.heapOverhead = new AtomicLong(segment.heapOverhead.get()); this.heapSize = new AtomicLong(segment.heapSize.get());
this.tagsPresent = segment.isTagsPresent(); this.tagsPresent = segment.isTagsPresent();
this.timeRangeTracker = segment.getTimeRangeTracker(); this.timeRangeTracker = segment.getTimeRangeTracker();
} }
@ -217,22 +217,19 @@ public abstract class Segment {
} }
/** /**
* @return The heap overhead of this segment. * @return The heap size of this segment.
*/ */
public long heapOverhead() { public long heapSize() {
return this.heapOverhead.get(); return this.heapSize.get();
} }
/** /**
* Updates the heap size counter of the segment by the given delta * Updates the size counters of the segment by the given delta
*/ */
//TODO
protected void incSize(long delta, long heapOverhead) { protected void incSize(long delta, long heapOverhead) {
this.dataSize.addAndGet(delta); this.dataSize.addAndGet(delta);
this.heapOverhead.addAndGet(heapOverhead); this.heapSize.addAndGet(heapOverhead);
}
protected void incHeapOverheadSize(long delta) {
this.heapOverhead.addAndGet(delta);
} }
public long getMinSequenceId() { public long getMinSequenceId() {
@ -293,10 +290,10 @@ public abstract class Segment {
if (succ || mslabUsed) { if (succ || mslabUsed) {
cellSize = getCellLength(cellToAdd); cellSize = getCellLength(cellToAdd);
} }
long overhead = heapOverheadChange(cellToAdd, succ); long heapSize = heapSizeChange(cellToAdd, succ);
incSize(cellSize, overhead); incSize(cellSize, heapSize);
if (memstoreSize != null) { if (memstoreSize != null) {
memstoreSize.incMemstoreSize(cellSize, overhead); memstoreSize.incMemstoreSize(cellSize, heapSize);
} }
getTimeRangeTracker().includeTimestamp(cellToAdd); getTimeRangeTracker().includeTimestamp(cellToAdd);
minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId()); minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId());
@ -309,15 +306,14 @@ public abstract class Segment {
} }
} }
protected long heapOverheadChange(Cell cell, boolean succ) { /**
* @return The increase in heap size because of this cell addition. This includes this cell POJO's
* heap size itself and additional overhead because of addition on to CSLM.
*/
protected long heapSizeChange(Cell cell, boolean succ) {
if (succ) { if (succ) {
if (cell instanceof ExtendedCell) { return ClassSize
return ClassSize .align(ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + CellUtil.estimatedHeapSizeOf(cell));
.align(ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + ((ExtendedCell) cell).heapOverhead());
}
// All cells in server side will be of type ExtendedCell. If not just go with estimation on
// the heap overhead considering it is KeyValue.
return ClassSize.align(ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + KeyValue.FIXED_OVERHEAD);
} }
return 0; return 0;
} }
@ -350,9 +346,9 @@ public abstract class Segment {
public String toString() { public String toString() {
String res = "Store segment of type "+this.getClass().getName()+"; "; String res = "Store segment of type "+this.getClass().getName()+"; ";
res += "isEmpty "+(isEmpty()?"yes":"no")+"; "; res += "isEmpty "+(isEmpty()?"yes":"no")+"; ";
res += "cellCount "+getCellsCount()+"; "; res += "cellsCount "+getCellsCount()+"; ";
res += "cellsSize "+keySize()+"; "; res += "cellsSize "+keySize()+"; ";
res += "heapOverhead "+heapOverhead()+"; "; res += "totalHeapSize "+heapSize()+"; ";
res += "Min ts "+getMinTimestamp()+"; "; res += "Min ts "+getMinTimestamp()+"; ";
return res; return res;
} }

9
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
View File

@ -291,6 +291,9 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
/** /**
* @return The size of this store's memstore, in bytes * @return The size of this store's memstore, in bytes
* @deprecated Since 2.0 and will be removed in 3.0. Use {@link #getSizeOfMemStore()} instead. * @deprecated Since 2.0 and will be removed in 3.0. Use {@link #getSizeOfMemStore()} instead.
* <p>
* Note: When using off heap MSLAB feature, this will not account the cell data bytes size which
* is in off heap MSLAB area.
*/ */
@Deprecated @Deprecated
long getMemStoreSize(); long getMemStoreSize();
@ -305,6 +308,9 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
* {@link #getMemStoreSize()} unless we are carrying snapshots and then it will be the size of * {@link #getMemStoreSize()} unless we are carrying snapshots and then it will be the size of
* outstanding snapshots. * outstanding snapshots.
* @deprecated Since 2.0 and will be removed in 3.0. Use {@link #getSizeToFlush()} instead. * @deprecated Since 2.0 and will be removed in 3.0. Use {@link #getSizeToFlush()} instead.
* <p>
* Note: When using off heap MSLAB feature, this will not account the cell data bytes size which
* is in off heap MSLAB area.
*/ */
@Deprecated @Deprecated
long getFlushableSize(); long getFlushableSize();
@ -320,6 +326,9 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
* Returns the memstore snapshot size * Returns the memstore snapshot size
* @return size of the memstore snapshot * @return size of the memstore snapshot
* @deprecated Since 2.0 and will be removed in 3.0. Use {@link #getSizeOfSnapshot()} instead. * @deprecated Since 2.0 and will be removed in 3.0. Use {@link #getSizeOfSnapshot()} instead.
* <p>
* Note: When using off heap MSLAB feature, this will not account the cell data bytes size which
* is in off heap MSLAB area.
*/ */
@Deprecated @Deprecated
long getSnapshotSize(); long getSnapshotSize();

67
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
View File

@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.EnvironmentEdge; import org.apache.hadoop.hbase.util.EnvironmentEdge;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.Threads;
@ -563,9 +562,11 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
// test 1 bucket // test 1 bucket
int totalCellsLen = addRowsByKeys(memstore, keys1); int totalCellsLen = addRowsByKeys(memstore, keys1);
long totalHeapOverhead = 4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); int oneCellOnCSLMHeapSize = 120;
int oneCellOnCAHeapSize = 88;
long totalHeapSize = 4 * oneCellOnCSLMHeapSize;
assertEquals(totalCellsLen, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
MemstoreSize size = memstore.getFlushableSize(); MemstoreSize size = memstore.getFlushableSize();
((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact
@ -573,9 +574,9 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
// One cell is duplicated and the compaction will remove it. All cells of same size so adjusting // One cell is duplicated and the compaction will remove it. All cells of same size so adjusting
// totalCellsLen // totalCellsLen
totalCellsLen = (totalCellsLen * 3) / 4; totalCellsLen = (totalCellsLen * 3) / 4;
totalHeapOverhead = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); totalHeapSize = 3 * oneCellOnCAHeapSize;
assertEquals(totalCellsLen, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
size = memstore.getFlushableSize(); size = memstore.getFlushableSize();
MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot
@ -599,10 +600,12 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
String[] keys2 = { "A", "B", "D" }; String[] keys2 = { "A", "B", "D" };
int totalCellsLen1 = addRowsByKeys(memstore, keys1); int totalCellsLen1 = addRowsByKeys(memstore, keys1);
long totalHeapOverhead = 4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); int oneCellOnCSLMHeapSize = 120;
int oneCellOnCAHeapSize = 88;
long totalHeapSize = 4 * oneCellOnCSLMHeapSize;
assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
MemstoreSize size = memstore.getFlushableSize(); MemstoreSize size = memstore.getFlushableSize();
((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact
@ -616,21 +619,21 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
// totalCellsLen // totalCellsLen
totalCellsLen1 = (totalCellsLen1 * 3) / 4; totalCellsLen1 = (totalCellsLen1 * 3) / 4;
assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize());
totalHeapOverhead = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); totalHeapSize = 3 * oneCellOnCAHeapSize;
assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
int totalCellsLen2 = addRowsByKeys(memstore, keys2); int totalCellsLen2 = addRowsByKeys(memstore, keys2);
totalHeapOverhead += 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); totalHeapSize += 3 * oneCellOnCSLMHeapSize;
assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
size = memstore.getFlushableSize(); size = memstore.getFlushableSize();
((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact
assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(0, memstore.getSnapshot().getCellsCount());
totalCellsLen2 = totalCellsLen2 / 3;// 2 cells duplicated in set 2 totalCellsLen2 = totalCellsLen2 / 3;// 2 cells duplicated in set 2
assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize());
totalHeapOverhead = 4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); totalHeapSize = 4 * oneCellOnCAHeapSize;
assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
size = memstore.getFlushableSize(); size = memstore.getFlushableSize();
MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot
@ -655,9 +658,11 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
String[] keys3 = { "D", "B", "B" }; String[] keys3 = { "D", "B", "B" };
int totalCellsLen1 = addRowsByKeys(memstore, keys1);// Adding 4 cells. int totalCellsLen1 = addRowsByKeys(memstore, keys1);// Adding 4 cells.
int oneCellOnCSLMHeapSize = 120;
int oneCellOnCAHeapSize = 88;
assertEquals(totalCellsLen1, region.getMemstoreSize()); assertEquals(totalCellsLen1, region.getMemstoreSize());
long totalHeapOverhead = 4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); long totalHeapSize = 4 * oneCellOnCSLMHeapSize;
assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
MemstoreSize size = memstore.getFlushableSize(); MemstoreSize size = memstore.getFlushableSize();
((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact
@ -668,16 +673,14 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
totalCellsLen1 = (totalCellsLen1 * 3) / 4; totalCellsLen1 = (totalCellsLen1 * 3) / 4;
assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize());
// In memory flush to make a CellArrayMap instead of CSLM. See the overhead diff. // In memory flush to make a CellArrayMap instead of CSLM. See the overhead diff.
totalHeapOverhead = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); totalHeapSize = 3 * oneCellOnCAHeapSize;
assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
int totalCellsLen2 = addRowsByKeys(memstore, keys2);// Adding 3 more cells. int totalCellsLen2 = addRowsByKeys(memstore, keys2);// Adding 3 more cells.
long totalHeapOverhead2 = 3 long totalHeapSize2 = 3 * oneCellOnCSLMHeapSize;
* (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY);
assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead + totalHeapOverhead2, assertEquals(totalHeapSize + totalHeapSize2, ((CompactingMemStore) memstore).heapSize());
((CompactingMemStore) memstore).heapOverhead());
((CompactingMemStore) memstore).disableCompaction(); ((CompactingMemStore) memstore).disableCompaction();
size = memstore.getFlushableSize(); size = memstore.getFlushableSize();
@ -685,23 +688,18 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(0, memstore.getSnapshot().getCellsCount());
// No change in the cells data size. ie. memstore size. as there is no compaction. // No change in the cells data size. ie. memstore size. as there is no compaction.
assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead + totalHeapOverhead2, assertEquals(totalHeapSize + totalHeapSize2, ((CompactingMemStore) memstore).heapSize());
((CompactingMemStore) memstore).heapOverhead());
int totalCellsLen3 = addRowsByKeys(memstore, keys3);// 3 more cells added int totalCellsLen3 = addRowsByKeys(memstore, keys3);// 3 more cells added
assertEquals(totalCellsLen1 + totalCellsLen2 + totalCellsLen3, assertEquals(totalCellsLen1 + totalCellsLen2 + totalCellsLen3,
regionServicesForStores.getMemstoreSize()); regionServicesForStores.getMemstoreSize());
long totalHeapOverhead3 = 3 long totalHeapSize3 = 3 * oneCellOnCSLMHeapSize;
* (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); assertEquals(totalHeapSize + totalHeapSize2 + totalHeapSize3,
assertEquals(totalHeapOverhead + totalHeapOverhead2 + totalHeapOverhead3, ((CompactingMemStore) memstore).heapSize());
((CompactingMemStore) memstore).heapOverhead());
((CompactingMemStore)memstore).enableCompaction(); ((CompactingMemStore)memstore).enableCompaction();
size = memstore.getFlushableSize(); size = memstore.getFlushableSize();
((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact
while (((CompactingMemStore)memstore).isMemStoreFlushingInMemory()) {
Threads.sleep(10);
}
assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(0, memstore.getSnapshot().getCellsCount());
// active flushed to pipeline and all 3 segments compacted. Will get rid of duplicated cells. // active flushed to pipeline and all 3 segments compacted. Will get rid of duplicated cells.
// Out of total 10, only 4 cells are unique // Out of total 10, only 4 cells are unique
@ -710,8 +708,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
assertEquals(totalCellsLen1 + totalCellsLen2 + totalCellsLen3, assertEquals(totalCellsLen1 + totalCellsLen2 + totalCellsLen3,
regionServicesForStores.getMemstoreSize()); regionServicesForStores.getMemstoreSize());
// Only 4 unique cells left // Only 4 unique cells left
assertEquals(4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY), assertEquals(4 * oneCellOnCAHeapSize, ((CompactingMemStore) memstore).heapSize());
((CompactingMemStore) memstore).heapOverhead());
size = memstore.getFlushableSize(); size = memstore.getFlushableSize();
MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot
@ -721,15 +718,13 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
assertEquals(0, regionServicesForStores.getMemstoreSize()); assertEquals(0, regionServicesForStores.getMemstoreSize());
memstore.clearSnapshot(snapshot.getId()); memstore.clearSnapshot(snapshot.getId());
//assertTrue(tstStr, false);
} }
private int addRowsByKeys(final AbstractMemStore hmc, String[] keys) { private int addRowsByKeys(final AbstractMemStore hmc, String[] keys) {
byte[] fam = Bytes.toBytes("testfamily"); byte[] fam = Bytes.toBytes("testfamily");
byte[] qf = Bytes.toBytes("testqualifier"); byte[] qf = Bytes.toBytes("testqualifier");
long size = hmc.getActive().keySize(); long size = hmc.getActive().keySize();
long heapOverhead = hmc.getActive().heapOverhead(); long heapOverhead = hmc.getActive().heapSize();
int totalLen = 0; int totalLen = 0;
for (int i = 0; i < keys.length; i++) { for (int i = 0; i < keys.length; i++) {
long timestamp = System.currentTimeMillis(); long timestamp = System.currentTimeMillis();
@ -742,7 +737,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp());
} }
regionServicesForStores.addMemstoreSize(new MemstoreSize(hmc.getActive().keySize() - size, regionServicesForStores.addMemstoreSize(new MemstoreSize(hmc.getActive().keySize() - size,
hmc.getActive().heapOverhead() - heapOverhead)); hmc.getActive().heapSize() - heapOverhead));
return totalLen; return totalLen;
} }

80
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java
View File

@ -25,7 +25,6 @@ import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.Threads;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -80,23 +79,22 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore
// test 1 bucket // test 1 bucket
long totalCellsLen = addRowsByKeys(memstore, keys1); long totalCellsLen = addRowsByKeys(memstore, keys1);
long totalHeapOverhead = 4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); int oneCellOnCSLMHeapSize = 120;
int oneCellOnCAHeapSize = 88;
long totalHeapSize = 4 * oneCellOnCSLMHeapSize;
assertEquals(totalCellsLen, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
assertEquals(4, memstore.getActive().getCellsCount()); assertEquals(4, memstore.getActive().getCellsCount());
MemstoreSize size = memstore.getFlushableSize(); MemstoreSize size = memstore.getFlushableSize();
((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact
while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) {
Threads.sleep(10);
}
assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(0, memstore.getSnapshot().getCellsCount());
// One cell is duplicated and the compaction will remove it. All cells of same size so adjusting // One cell is duplicated and the compaction will remove it. All cells of same size so adjusting
// totalCellsLen // totalCellsLen
totalCellsLen = (totalCellsLen * 3) / 4; totalCellsLen = (totalCellsLen * 3) / 4;
assertEquals(totalCellsLen, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen, regionServicesForStores.getMemstoreSize());
totalHeapOverhead = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); totalHeapSize = 3 * oneCellOnCAHeapSize;
assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
for ( Segment s : memstore.getSegments()) { for ( Segment s : memstore.getSegments()) {
counter += s.getCellsCount(); counter += s.getCellsCount();
} }
@ -117,16 +115,14 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore
String[] keys2 = { "A", "B", "D" }; String[] keys2 = { "A", "B", "D" };
long totalCellsLen1 = addRowsByKeys(memstore, keys1); long totalCellsLen1 = addRowsByKeys(memstore, keys1);
long totalHeapOverhead1 = 4 int oneCellOnCSLMHeapSize = 120;
* (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); int oneCellOnCAHeapSize = 88;
long totalHeapSize1 = 4 * oneCellOnCSLMHeapSize;
assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead1, ((CompactingMemStore) memstore).heapOverhead()); assertEquals(totalHeapSize1, ((CompactingMemStore) memstore).heapSize());
MemstoreSize size = memstore.getFlushableSize(); MemstoreSize size = memstore.getFlushableSize();
((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact
while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) {
Threads.sleep(1000);
}
int counter = 0; int counter = 0;
for ( Segment s : memstore.getSegments()) { for ( Segment s : memstore.getSegments()) {
counter += s.getCellsCount(); counter += s.getCellsCount();
@ -136,27 +132,17 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore
// One cell is duplicated and the compaction will remove it. All cells of same size so adjusting // One cell is duplicated and the compaction will remove it. All cells of same size so adjusting
// totalCellsLen // totalCellsLen
totalCellsLen1 = (totalCellsLen1 * 3) / 4; totalCellsLen1 = (totalCellsLen1 * 3) / 4;
totalHeapOverhead1 = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); totalHeapSize1 = 3 * oneCellOnCAHeapSize;
assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead1, ((CompactingMemStore) memstore).heapOverhead()); assertEquals(totalHeapSize1, ((CompactingMemStore) memstore).heapSize());
long totalCellsLen2 = addRowsByKeys(memstore, keys2); long totalCellsLen2 = addRowsByKeys(memstore, keys2);
long totalHeapOverhead2 = 3 long totalHeapSize2 = 3 * oneCellOnCSLMHeapSize;
* (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY);
assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead1 + totalHeapOverhead2, assertEquals(totalHeapSize1 + totalHeapSize2, ((CompactingMemStore) memstore).heapSize());
((CompactingMemStore) memstore).heapOverhead());
size = memstore.getFlushableSize(); size = memstore.getFlushableSize();
((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact
int i = 0;
while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) {
Threads.sleep(10);
if (i > 10000000) {
((CompactingMemStore) memstore).debug();
assertTrue("\n\n<<< Infinite loop! :( \n", false);
}
}
assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(0, memstore.getSnapshot().getCellsCount());
counter = 0; counter = 0;
for ( Segment s : memstore.getSegments()) { for ( Segment s : memstore.getSegments()) {
@ -165,9 +151,8 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore
assertEquals(4,counter); assertEquals(4,counter);
totalCellsLen2 = totalCellsLen2 / 3;// 2 cells duplicated in set 2 totalCellsLen2 = totalCellsLen2 / 3;// 2 cells duplicated in set 2
assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize());
totalHeapOverhead2 = 1 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); totalHeapSize2 = 1 * oneCellOnCAHeapSize;
assertEquals(totalHeapOverhead1 + totalHeapOverhead2, assertEquals(totalHeapSize1 + totalHeapSize2, ((CompactingMemStore) memstore).heapSize());
((CompactingMemStore) memstore).heapOverhead());
size = memstore.getFlushableSize(); size = memstore.getFlushableSize();
MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot
@ -186,48 +171,42 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore
String[] keys3 = { "D", "B", "B" }; String[] keys3 = { "D", "B", "B" };
long totalCellsLen1 = addRowsByKeys(memstore, keys1); long totalCellsLen1 = addRowsByKeys(memstore, keys1);
long totalHeapOverhead1 = 4 int oneCellOnCSLMHeapSize = 120;
* (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); int oneCellOnCAHeapSize = 88;
long totalHeapSize1 = 4 * oneCellOnCSLMHeapSize;
assertEquals(totalCellsLen1, region.getMemstoreSize()); assertEquals(totalCellsLen1, region.getMemstoreSize());
assertEquals(totalHeapOverhead1, ((CompactingMemStore) memstore).heapOverhead()); assertEquals(totalHeapSize1, ((CompactingMemStore) memstore).heapSize());
MemstoreSize size = memstore.getFlushableSize(); MemstoreSize size = memstore.getFlushableSize();
((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact
while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) {
Threads.sleep(10);
}
assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(0, memstore.getSnapshot().getCellsCount());
// One cell is duplicated and the compaction will remove it. All cells of same size so adjusting // One cell is duplicated and the compaction will remove it. All cells of same size so adjusting
// totalCellsLen // totalCellsLen
totalCellsLen1 = (totalCellsLen1 * 3) / 4; totalCellsLen1 = (totalCellsLen1 * 3) / 4;
totalHeapOverhead1 = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); totalHeapSize1 = 3 * oneCellOnCAHeapSize;
assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead1, ((CompactingMemStore) memstore).heapOverhead()); assertEquals(totalHeapSize1, ((CompactingMemStore) memstore).heapSize());
long totalCellsLen2 = addRowsByKeys(memstore, keys2); long totalCellsLen2 = addRowsByKeys(memstore, keys2);
long totalHeapOverhead2 = 3 long totalHeapSize2 = 3 * oneCellOnCSLMHeapSize;
* (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY);
assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead1 + totalHeapOverhead2, assertEquals(totalHeapSize1 + totalHeapSize2, ((CompactingMemStore) memstore).heapSize());
((CompactingMemStore) memstore).heapOverhead());
((CompactingMemStore) memstore).disableCompaction(); ((CompactingMemStore) memstore).disableCompaction();
size = memstore.getFlushableSize(); size = memstore.getFlushableSize();
((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline without compaction ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline without compaction
assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(0, memstore.getSnapshot().getCellsCount());
assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead1 + totalHeapOverhead2, assertEquals(totalHeapSize1 + totalHeapSize2, ((CompactingMemStore) memstore).heapSize());
((CompactingMemStore) memstore).heapOverhead());
long totalCellsLen3 = addRowsByKeys(memstore, keys3); long totalCellsLen3 = addRowsByKeys(memstore, keys3);
long totalHeapOverhead3 = 3 long totalHeapSize3 = 3 * oneCellOnCSLMHeapSize;
* (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY);
assertEquals(totalCellsLen1 + totalCellsLen2 + totalCellsLen3, assertEquals(totalCellsLen1 + totalCellsLen2 + totalCellsLen3,
regionServicesForStores.getMemstoreSize()); regionServicesForStores.getMemstoreSize());
assertEquals(totalHeapOverhead1 + totalHeapOverhead2 + totalHeapOverhead3, assertEquals(totalHeapSize1 + totalHeapSize2 + totalHeapSize3,
((CompactingMemStore) memstore).heapOverhead()); ((CompactingMemStore) memstore).heapSize());
((CompactingMemStore) memstore).enableCompaction(); ((CompactingMemStore) memstore).enableCompaction();
size = memstore.getFlushableSize(); size = memstore.getFlushableSize();
@ -243,8 +222,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore
assertEquals(totalCellsLen1 + totalCellsLen2 + totalCellsLen3, assertEquals(totalCellsLen1 + totalCellsLen2 + totalCellsLen3,
regionServicesForStores.getMemstoreSize()); regionServicesForStores.getMemstoreSize());
// Only 4 unique cells left // Only 4 unique cells left
assertEquals(4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY), assertEquals(4 * oneCellOnCAHeapSize, ((CompactingMemStore) memstore).heapSize());
((CompactingMemStore) memstore).heapOverhead());
size = memstore.getFlushableSize(); size = memstore.getFlushableSize();
MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot

4
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
View File

@ -122,8 +122,6 @@ public class TestDefaultMemStore {
this.memstore.add(kv, sizeChangeForSecondCell); this.memstore.add(kv, sizeChangeForSecondCell);
// make sure memstore size increase won't double-count MSLAB chunk size // make sure memstore size increase won't double-count MSLAB chunk size
assertEquals(Segment.getCellLength(kv), sizeChangeForFirstCell.getDataSize()); assertEquals(Segment.getCellLength(kv), sizeChangeForFirstCell.getDataSize());
assertEquals(this.memstore.active.heapOverheadChange(kv, true),
sizeChangeForFirstCell.getHeapOverhead());
Segment segment = this.memstore.getActive(); Segment segment = this.memstore.getActive();
MemStoreLAB msLab = segment.getMemStoreLAB(); MemStoreLAB msLab = segment.getMemStoreLAB();
if (msLab != null) { if (msLab != null) {
@ -137,7 +135,7 @@ public class TestDefaultMemStore {
} else { } else {
// make sure no memstore size change w/o MSLAB // make sure no memstore size change w/o MSLAB
assertEquals(0, sizeChangeForSecondCell.getDataSize()); assertEquals(0, sizeChangeForSecondCell.getDataSize());
assertEquals(0, sizeChangeForSecondCell.getHeapOverhead()); assertEquals(0, sizeChangeForSecondCell.getHeapSize());
} }
} }

3
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
View File

@ -24,7 +24,6 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
import java.lang.management.ManagementFactory; import java.lang.management.ManagementFactory;
import java.lang.management.MemoryType;
import java.util.Iterator; import java.util.Iterator;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@ -928,7 +927,7 @@ public class TestHeapMemoryManager {
} }
@Override @Override
public long getGlobalMemstoreHeapOverhead() { public long getGlobalMemstoreHeapSize() {
return testMemstoreSize; return testMemstoreSize;
} }

18
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
View File

@ -193,7 +193,8 @@ public class TestPerColumnFamilyFlush {
// We should have cleared out only CF1, since we chose the flush thresholds // We should have cleared out only CF1, since we chose the flush thresholds
// and number of puts accordingly. // and number of puts accordingly.
assertEquals(MemstoreSize.EMPTY_SIZE, cf1MemstoreSize); assertEquals(0, cf1MemstoreSize.getDataSize());
assertEquals(0, cf1MemstoreSize.getHeapSize());
// Nothing should have happened to CF2, ... // Nothing should have happened to CF2, ...
assertEquals(cf2MemstoreSize, oldCF2MemstoreSize); assertEquals(cf2MemstoreSize, oldCF2MemstoreSize);
// ... or CF3 // ... or CF3
@ -229,8 +230,10 @@ public class TestPerColumnFamilyFlush {
.getEarliestMemstoreSeqNum(region.getRegionInfo().getEncodedNameAsBytes()); .getEarliestMemstoreSeqNum(region.getRegionInfo().getEncodedNameAsBytes());
// CF1 and CF2, both should be absent. // CF1 and CF2, both should be absent.
assertEquals(MemstoreSize.EMPTY_SIZE, cf1MemstoreSize); assertEquals(0, cf1MemstoreSize.getDataSize());
assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSize); assertEquals(0, cf1MemstoreSize.getHeapSize());
assertEquals(0, cf2MemstoreSize.getDataSize());
assertEquals(0, cf2MemstoreSize.getHeapSize());
// CF3 shouldn't have been touched. // CF3 shouldn't have been touched.
assertEquals(cf3MemstoreSize, oldCF3MemstoreSize); assertEquals(cf3MemstoreSize, oldCF3MemstoreSize);
assertEquals(totalMemstoreSize, cf3MemstoreSize.getDataSize()); assertEquals(totalMemstoreSize, cf3MemstoreSize.getDataSize());
@ -310,9 +313,12 @@ public class TestPerColumnFamilyFlush {
region.getWAL().getEarliestMemstoreSeqNum(region.getRegionInfo().getEncodedNameAsBytes()); region.getWAL().getEarliestMemstoreSeqNum(region.getRegionInfo().getEncodedNameAsBytes());
// Everything should have been cleared // Everything should have been cleared
assertEquals(MemstoreSize.EMPTY_SIZE, cf1MemstoreSize); assertEquals(0, cf1MemstoreSize.getDataSize());
assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSize); assertEquals(0, cf1MemstoreSize.getHeapSize());
assertEquals(MemstoreSize.EMPTY_SIZE, cf3MemstoreSize); assertEquals(0, cf2MemstoreSize.getDataSize());
assertEquals(0, cf2MemstoreSize.getHeapSize());
assertEquals(0, cf3MemstoreSize.getDataSize());
assertEquals(0, cf3MemstoreSize.getHeapSize());
assertEquals(0, totalMemstoreSize); assertEquals(0, totalMemstoreSize);
assertEquals(HConstants.NO_SEQNUM, smallestSeqInRegionCurrentMemstore); assertEquals(HConstants.NO_SEQNUM, smallestSeqInRegionCurrentMemstore);
HBaseTestingUtility.closeRegionAndWAL(region); HBaseTestingUtility.closeRegionAndWAL(region);

3
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
View File

@ -245,7 +245,8 @@ public class TestStore {
Assert.assertEquals(kvSize2, size); Assert.assertEquals(kvSize2, size);
flushStore(store, id++); flushStore(store, id++);
size = store.memstore.getFlushableSize(); size = store.memstore.getFlushableSize();
Assert.assertEquals(MemstoreSize.EMPTY_SIZE, size); assertEquals(0, size.getDataSize());
assertEquals(0, size.getHeapSize());
return null; return null;
} }
}); });

45
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
View File

@ -232,15 +232,16 @@ public class TestWalAndCompactingMemStoreFlush {
// CF1 was flushed to memory, but there is nothing to compact, and CF1 was flattened // CF1 was flushed to memory, but there is nothing to compact, and CF1 was flattened
assertTrue(cf1MemstoreSizePhaseII.getDataSize() == cf1MemstoreSizePhaseI.getDataSize()); assertTrue(cf1MemstoreSizePhaseII.getDataSize() == cf1MemstoreSizePhaseI.getDataSize());
assertTrue(cf1MemstoreSizePhaseII.getHeapOverhead() < cf1MemstoreSizePhaseI.getHeapOverhead()); assertTrue(cf1MemstoreSizePhaseII.getHeapSize() < cf1MemstoreSizePhaseI.getHeapSize());
// CF2 should become empty // CF2 should become empty
assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseII); assertEquals(0, cf2MemstoreSizePhaseII.getDataSize());
assertEquals(0, cf2MemstoreSizePhaseII.getHeapSize());
// verify that CF3 was flushed to memory and was compacted (this is approximation check) // verify that CF3 was flushed to memory and was compacted (this is approximation check)
assertTrue(cf3MemstoreSizePhaseI.getDataSize() > cf3MemstoreSizePhaseII.getDataSize()); assertTrue(cf3MemstoreSizePhaseI.getDataSize() > cf3MemstoreSizePhaseII.getDataSize());
assertTrue( assertTrue(
cf3MemstoreSizePhaseI.getHeapOverhead() / 2 > cf3MemstoreSizePhaseII.getHeapOverhead()); cf3MemstoreSizePhaseI.getHeapSize() / 2 > cf3MemstoreSizePhaseII.getHeapSize());
// Now the smallest LSN in the region should be the same as the smallest // Now the smallest LSN in the region should be the same as the smallest
// LSN in the memstore of CF1. // LSN in the memstore of CF1.
@ -295,7 +296,8 @@ public class TestWalAndCompactingMemStoreFlush {
// CF1's pipeline component (inserted before first flush) should be flushed to disk // CF1's pipeline component (inserted before first flush) should be flushed to disk
// CF2 should be flushed to disk // CF2 should be flushed to disk
assertTrue(cf1MemstoreSizePhaseIII.getDataSize() > cf1MemstoreSizePhaseIV.getDataSize()); assertTrue(cf1MemstoreSizePhaseIII.getDataSize() > cf1MemstoreSizePhaseIV.getDataSize());
assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseIV); assertEquals(0, cf2MemstoreSizePhaseIV.getDataSize());
assertEquals(0, cf2MemstoreSizePhaseIV.getHeapSize());
// CF3 shouldn't have been touched. // CF3 shouldn't have been touched.
assertEquals(cf3MemstoreSizePhaseIV, cf3MemstoreSizePhaseII); assertEquals(cf3MemstoreSizePhaseIV, cf3MemstoreSizePhaseII);
@ -318,9 +320,12 @@ public class TestWalAndCompactingMemStoreFlush {
long smallestSeqInRegionCurrentMemstorePhaseV = getWAL(region) long smallestSeqInRegionCurrentMemstorePhaseV = getWAL(region)
.getEarliestMemstoreSeqNum(region.getRegionInfo().getEncodedNameAsBytes()); .getEarliestMemstoreSeqNum(region.getRegionInfo().getEncodedNameAsBytes());
assertEquals(MemstoreSize.EMPTY_SIZE , cf1MemstoreSizePhaseV); assertEquals(0, cf1MemstoreSizePhaseV.getDataSize());
assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseV); assertEquals(0, cf1MemstoreSizePhaseV.getHeapSize());
assertEquals(MemstoreSize.EMPTY_SIZE, cf3MemstoreSizePhaseV); assertEquals(0, cf2MemstoreSizePhaseV.getDataSize());
assertEquals(0, cf2MemstoreSizePhaseV.getHeapSize());
assertEquals(0, cf3MemstoreSizePhaseV.getDataSize());
assertEquals(0, cf3MemstoreSizePhaseV.getHeapSize());
// What happens when we hit the memstore limit, but we are not able to find // What happens when we hit the memstore limit, but we are not able to find
// any Column Family above the threshold? // any Column Family above the threshold?
@ -463,14 +468,14 @@ public class TestWalAndCompactingMemStoreFlush {
/* PHASE II - validation */ /* PHASE II - validation */
// CF1 was flushed to memory, should be flattened and take less space // CF1 was flushed to memory, should be flattened and take less space
assertEquals(cf1MemstoreSizePhaseII.getDataSize() , cf1MemstoreSizePhaseI.getDataSize()); assertEquals(cf1MemstoreSizePhaseII.getDataSize() , cf1MemstoreSizePhaseI.getDataSize());
assertTrue(cf1MemstoreSizePhaseII.getHeapOverhead() < cf1MemstoreSizePhaseI.getHeapOverhead()); assertTrue(cf1MemstoreSizePhaseII.getHeapSize() < cf1MemstoreSizePhaseI.getHeapSize());
// CF2 should become empty // CF2 should become empty
assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseII); assertEquals(0, cf2MemstoreSizePhaseII.getDataSize());
assertEquals(0, cf2MemstoreSizePhaseII.getHeapSize());
// verify that CF3 was flushed to memory and was not compacted (this is an approximation check) // verify that CF3 was flushed to memory and was not compacted (this is an approximation check)
// if compacted CF# should be at least twice less because its every key was duplicated // if compacted CF# should be at least twice less because its every key was duplicated
assertEquals(cf3MemstoreSizePhaseII.getDataSize() , cf3MemstoreSizePhaseI.getDataSize()); assertEquals(cf3MemstoreSizePhaseII.getDataSize() , cf3MemstoreSizePhaseI.getDataSize());
assertTrue( assertTrue(cf3MemstoreSizePhaseI.getHeapSize() / 2 < cf3MemstoreSizePhaseII.getHeapSize());
cf3MemstoreSizePhaseI.getHeapOverhead() / 2 < cf3MemstoreSizePhaseII.getHeapOverhead());
// Now the smallest LSN in the region should be the same as the smallest // Now the smallest LSN in the region should be the same as the smallest
// LSN in the memstore of CF1. // LSN in the memstore of CF1.
@ -533,7 +538,8 @@ public class TestWalAndCompactingMemStoreFlush {
// CF1's biggest pipeline component (inserted before first flush) should be flushed to disk // CF1's biggest pipeline component (inserted before first flush) should be flushed to disk
// CF2 should remain empty // CF2 should remain empty
assertTrue(cf1MemstoreSizePhaseIII.getDataSize() > cf1MemstoreSizePhaseIV.getDataSize()); assertTrue(cf1MemstoreSizePhaseIII.getDataSize() > cf1MemstoreSizePhaseIV.getDataSize());
assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseIV); assertEquals(0, cf2MemstoreSizePhaseIV.getDataSize());
assertEquals(0, cf2MemstoreSizePhaseIV.getHeapSize());
// CF3 shouldn't have been touched. // CF3 shouldn't have been touched.
assertEquals(cf3MemstoreSizePhaseIV, cf3MemstoreSizePhaseII); assertEquals(cf3MemstoreSizePhaseIV, cf3MemstoreSizePhaseII);
// the smallest LSN of CF3 shouldn't change // the smallest LSN of CF3 shouldn't change
@ -561,9 +567,12 @@ public class TestWalAndCompactingMemStoreFlush {
/*------------------------------------------------------------------------------*/ /*------------------------------------------------------------------------------*/
/* PHASE V - validation */ /* PHASE V - validation */
assertEquals(MemstoreSize.EMPTY_SIZE, cf1MemstoreSizePhaseV); assertEquals(0, cf1MemstoreSizePhaseV.getDataSize());
assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseV); assertEquals(0, cf1MemstoreSizePhaseV.getHeapSize());
assertEquals(MemstoreSize.EMPTY_SIZE, cf3MemstoreSizePhaseV); assertEquals(0, cf2MemstoreSizePhaseV.getDataSize());
assertEquals(0, cf2MemstoreSizePhaseV.getHeapSize());
assertEquals(0, cf3MemstoreSizePhaseV.getDataSize());
assertEquals(0, cf3MemstoreSizePhaseV.getHeapSize());
// The total memstores size should be empty // The total memstores size should be empty
assertEquals(0, totalMemstoreSizePhaseV); assertEquals(0, totalMemstoreSizePhaseV);
// Because there is nothing in any memstore the WAL's LSN should be -1 // Because there is nothing in any memstore the WAL's LSN should be -1
@ -683,7 +692,8 @@ public class TestWalAndCompactingMemStoreFlush {
long smallestSeqCF3PhaseII = region.getOldestSeqIdOfStore(FAMILY3); long smallestSeqCF3PhaseII = region.getOldestSeqIdOfStore(FAMILY3);
// CF2 should have been cleared // CF2 should have been cleared
assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseII); assertEquals(0, cf2MemstoreSizePhaseII.getDataSize());
assertEquals(0, cf2MemstoreSizePhaseII.getHeapSize());
String s = "\n\n----------------------------------\n" String s = "\n\n----------------------------------\n"
+ "Upon initial insert and flush, LSN of CF1 is:" + "Upon initial insert and flush, LSN of CF1 is:"
@ -816,7 +826,8 @@ public class TestWalAndCompactingMemStoreFlush {
long smallestSeqCF3PhaseII = region.getOldestSeqIdOfStore(FAMILY3); long smallestSeqCF3PhaseII = region.getOldestSeqIdOfStore(FAMILY3);
// CF2 should have been cleared // CF2 should have been cleared
assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseII); assertEquals(0, cf2MemstoreSizePhaseII.getDataSize());
assertEquals(0, cf2MemstoreSizePhaseII.getHeapSize());
// Add same entries to compact them later // Add same entries to compact them later
for (int i = 1; i <= 1200; i++) { for (int i = 1; i <= 1200; i++) {