Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-16502 Reduce garbage in BufferedDataBlockEncoder - addendum adopts Hiroshi's suggestion (binlijin)

This commit is contained in:
tedyu 2016-08-30 18:04:34 -07:00
parent af33f94513
commit 7b95ac117d
4 changed files with 15 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
View File

@ -121,8 +121,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected boolean uncompressTags = true; protected boolean uncompressTags = true;
/** We need to store a copy of the key. */ /** We need to store a copy of the key. */
protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE]; protected byte[] keyBuffer = HConstants.EMPTY_BYTE_ARRAY;
protected byte[] tagsBuffer = null; protected byte[] tagsBuffer = HConstants.EMPTY_BYTE_ARRAY;
protected long memstoreTS; protected long memstoreTS;
protected int nextKvOffset; protected int nextKvOffset;
@ -132,15 +132,9 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
private final ObjectIntPair<ByteBuffer> tmpPair; private final ObjectIntPair<ByteBuffer> tmpPair;
private final boolean includeTags; private final boolean includeTags;
public SeekerState(ObjectIntPair<ByteBuffer> tmpPair, boolean includeTags, public SeekerState(ObjectIntPair<ByteBuffer> tmpPair, boolean includeTags) {
boolean tagsCompressed) {
this.tmpPair = tmpPair; this.tmpPair = tmpPair;
this.includeTags = includeTags; this.includeTags = includeTags;
if (tagsCompressed) {
tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
} else {
tagsBuffer = HConstants.EMPTY_BYTE_ARRAY;
}
} }
protected boolean isValid() { protected boolean isValid() {
@ -157,11 +151,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected void ensureSpaceForKey() { protected void ensureSpaceForKey() {
if (keyLength > keyBuffer.length) { if (keyLength > keyBuffer.length) {
// rare case, but we need to handle arbitrary length of key int newKeyBufferLength = Integer.highestOneBit(Math.max(
int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2; INITIAL_KEY_BUFFER_SIZE, keyLength) - 1) << 1;
while (keyLength > newKeyBufferLength) {
newKeyBufferLength *= 2;
}
byte[] newKeyBuffer = new byte[newKeyBufferLength]; byte[] newKeyBuffer = new byte[newKeyBufferLength];
System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length); System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length);
keyBuffer = newKeyBuffer; keyBuffer = newKeyBuffer;
@ -170,11 +161,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected void ensureSpaceForTags() { protected void ensureSpaceForTags() {
if (tagsLength > tagsBuffer.length) { if (tagsLength > tagsBuffer.length) {
// rare case, but we need to handle arbitrary length of tags int newTagsBufferLength = Integer.highestOneBit(Math.max(
int newTagsBufferLength = Math.max(tagsBuffer.length, 1) * 2; INITIAL_KEY_BUFFER_SIZE, tagsLength) - 1) << 1;
while (tagsLength > newTagsBufferLength) {
newTagsBufferLength *= 2;
}
byte[] newTagsBuffer = new byte[newTagsBufferLength]; byte[] newTagsBuffer = new byte[newTagsBufferLength];
System.arraycopy(tagsBuffer, 0, newTagsBuffer, 0, tagsBuffer.length); System.arraycopy(tagsBuffer, 0, newTagsBuffer, 0, tagsBuffer.length);
tagsBuffer = newTagsBuffer; tagsBuffer = newTagsBuffer;
@ -730,10 +718,6 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
return this.decodingCtx.getHFileContext().isIncludesTags(); return this.decodingCtx.getHFileContext().isIncludesTags();
} }
protected boolean tagsCompressed() {
return this.decodingCtx.getHFileContext().isCompressTags();
}
@Override @Override
public int compareKey(CellComparator comparator, Cell key) { public int compareKey(CellComparator comparator, Cell key) {
keyOnlyKV.setKey(current.keyBuffer, 0, current.keyLength); keyOnlyKV.setKey(current.keyBuffer, 0, current.keyLength);
@ -988,8 +972,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected STATE createSeekerState() { protected STATE createSeekerState() {
// This will fail for non-default seeker state if the subclass does not // This will fail for non-default seeker state if the subclass does not
// override this method. // override this method.
return (STATE) new SeekerState(this.tmpPair, this.includesTags(), return (STATE) new SeekerState(this.tmpPair, this.includesTags());
this.tagsCompressed());
} }
abstract protected void decodeFirst(); abstract protected void decodeFirst();

7
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
View File

@ -368,8 +368,8 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
private long timestamp; private long timestamp;
public DiffSeekerState(ObjectIntPair<ByteBuffer> tmpPair, public DiffSeekerState(ObjectIntPair<ByteBuffer> tmpPair,
boolean includeTags, boolean tagsCompressed) { boolean includeTags) {
super(tmpPair, includeTags, tagsCompressed); super(tmpPair, includeTags);
} }
@Override @Override
@ -504,8 +504,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
@Override @Override
protected DiffSeekerState createSeekerState() { protected DiffSeekerState createSeekerState() {
return new DiffSeekerState(this.tmpPair, this.includesTags(), return new DiffSeekerState(this.tmpPair, this.includesTags());
this.tagsCompressed());
} }
}; };
} }

7
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
View File

@ -379,8 +379,8 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
private int familyLengthWithSize; private int familyLengthWithSize;
public FastDiffSeekerState(ObjectIntPair<ByteBuffer> tmpPair, public FastDiffSeekerState(ObjectIntPair<ByteBuffer> tmpPair,
boolean includeTags, boolean tagsCompressed) { boolean includeTags) {
super(tmpPair, includeTags, tagsCompressed); super(tmpPair, includeTags);
} }
@Override @Override
@ -520,8 +520,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
@Override @Override
protected FastDiffSeekerState createSeekerState() { protected FastDiffSeekerState createSeekerState() {
return new FastDiffSeekerState(this.tmpPair, this.includesTags(), return new FastDiffSeekerState(this.tmpPair, this.includesTags());
this.tagsCompressed());
} }
}; };
} }

2
hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
View File

@ -49,7 +49,7 @@ public class TestBufferedDataBlockEncoder {
@Test @Test
public void testEnsureSpaceForKey() { public void testEnsureSpaceForKey() {
BufferedDataBlockEncoder.SeekerState state = new BufferedDataBlockEncoder.SeekerState( BufferedDataBlockEncoder.SeekerState state = new BufferedDataBlockEncoder.SeekerState(
new ObjectIntPair<ByteBuffer>(), false, false); new ObjectIntPair<ByteBuffer>(), false);
for (int i = 1; i <= 65536; ++i) { for (int i = 1; i <= 65536; ++i) {
state.keyLength = i; state.keyLength = i;
state.ensureSpaceForKey(); state.ensureSpaceForKey();