Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-16502 Reduce garbage in BufferedDataBlockEncoder (binlijin)

This commit is contained in:
tedyu 2016-08-25 20:35:30 -07:00
parent 3fd59b0ba9
commit b1ee8a88c3
4 changed files with 28 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
View File

@ -122,7 +122,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
/** We need to store a copy of the key. */ /** We need to store a copy of the key. */
protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE]; protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
protected byte[] tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE]; protected byte[] tagsBuffer = null;
protected long memstoreTS; protected long memstoreTS;
protected int nextKvOffset; protected int nextKvOffset;
@ -132,9 +132,15 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
private final ObjectIntPair<ByteBuffer> tmpPair; private final ObjectIntPair<ByteBuffer> tmpPair;
private final boolean includeTags; private final boolean includeTags;
public SeekerState(ObjectIntPair<ByteBuffer> tmpPair, boolean includeTags) { public SeekerState(ObjectIntPair<ByteBuffer> tmpPair, boolean includeTags,
boolean tagsCompressed) {
this.tmpPair = tmpPair; this.tmpPair = tmpPair;
this.includeTags = includeTags; this.includeTags = includeTags;
if (tagsCompressed) {
tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
} else {
tagsBuffer = HConstants.EMPTY_BYTE_ARRAY;
}
} }
protected boolean isValid() { protected boolean isValid() {
@ -724,6 +730,10 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
return this.decodingCtx.getHFileContext().isIncludesTags(); return this.decodingCtx.getHFileContext().isIncludesTags();
} }
protected boolean tagsCompressed() {
return this.decodingCtx.getHFileContext().isCompressTags();
}
@Override @Override
public int compareKey(CellComparator comparator, Cell key) { public int compareKey(CellComparator comparator, Cell key) {
keyOnlyKV.setKey(current.keyBuffer, 0, current.keyLength); keyOnlyKV.setKey(current.keyBuffer, 0, current.keyLength);
@ -978,7 +988,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected STATE createSeekerState() { protected STATE createSeekerState() {
// This will fail for non-default seeker state if the subclass does not // This will fail for non-default seeker state if the subclass does not
// override this method. // override this method.
return (STATE) new SeekerState(this.tmpPair, this.includesTags()); return (STATE) new SeekerState(this.tmpPair, this.includesTags(),
this.tagsCompressed());
} }
abstract protected void decodeFirst(); abstract protected void decodeFirst();

8
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
View File

@ -367,8 +367,9 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
private int rowLengthWithSize; private int rowLengthWithSize;
private long timestamp; private long timestamp;
public DiffSeekerState(ObjectIntPair<ByteBuffer> tmpPair, boolean includeTags) { public DiffSeekerState(ObjectIntPair<ByteBuffer> tmpPair,
super(tmpPair, includeTags); boolean includeTags, boolean tagsCompressed) {
super(tmpPair, includeTags, tagsCompressed);
} }
@Override @Override
@ -503,7 +504,8 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
@Override @Override
protected DiffSeekerState createSeekerState() { protected DiffSeekerState createSeekerState() {
return new DiffSeekerState(this.tmpPair, this.includesTags()); return new DiffSeekerState(this.tmpPair, this.includesTags(),
this.tagsCompressed());
} }
}; };
} }

8
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
View File

@ -378,8 +378,9 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
private int rowLengthWithSize; private int rowLengthWithSize;
private int familyLengthWithSize; private int familyLengthWithSize;
public FastDiffSeekerState(ObjectIntPair<ByteBuffer> tmpPair, boolean includeTags) { public FastDiffSeekerState(ObjectIntPair<ByteBuffer> tmpPair,
super(tmpPair, includeTags); boolean includeTags, boolean tagsCompressed) {
super(tmpPair, includeTags, tagsCompressed);
} }
@Override @Override
@ -519,7 +520,8 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
@Override @Override
protected FastDiffSeekerState createSeekerState() { protected FastDiffSeekerState createSeekerState() {
return new FastDiffSeekerState(this.tmpPair, this.includesTags()); return new FastDiffSeekerState(this.tmpPair, this.includesTags(),
this.tagsCompressed());
} }
}; };
} }

9
hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
View File

@ -21,16 +21,15 @@ import static org.junit.Assert.assertTrue;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ObjectIntPair; import org.apache.hadoop.hbase.util.ObjectIntPair;
import org.junit.Test; import org.junit.Test;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
import org.apache.hadoop.hbase.KeyValue;
@Category({IOTests.class, MediumTests.class}) @Category({IOTests.class, MediumTests.class})
public class TestBufferedDataBlockEncoder { public class TestBufferedDataBlockEncoder {
@ -49,8 +48,8 @@ public class TestBufferedDataBlockEncoder {
@Test @Test
public void testEnsureSpaceForKey() { public void testEnsureSpaceForKey() {
BufferedDataBlockEncoder.SeekerState state = BufferedDataBlockEncoder.SeekerState state = new BufferedDataBlockEncoder.SeekerState(
new BufferedDataBlockEncoder.SeekerState(new ObjectIntPair<ByteBuffer>(), false); new ObjectIntPair<ByteBuffer>(), false, false);
for (int i = 1; i <= 65536; ++i) { for (int i = 1; i <= 65536; ++i) {
state.keyLength = i; state.keyLength = i;
state.ensureSpaceForKey(); state.ensureSpaceForKey();