From b59f18c2af22f3e6a67c30d585b7a8a9ae5d43ef Mon Sep 17 00:00:00 2001 From: tedyu Date: Thu, 25 Aug 2016 20:39:42 -0700 Subject: [PATCH] HBASE-16502 Reduce garbage in BufferedDataBlockEncoder (binlijin) --- .../io/encoding/BufferedDataBlockEncoder.java | 21 +++++++++++++++---- .../io/encoding/DiffKeyDeltaEncoder.java | 10 ++++++--- .../io/encoding/FastDiffDeltaEncoder.java | 10 ++++++--- .../TestBufferedDataBlockEncoder.java | 4 ++-- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index d808f88c6b5..6e06bfc18fe 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -95,12 +95,20 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { /** We need to store a copy of the key. */ protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE]; - protected byte[] tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE]; + protected byte[] tagsBuffer = null; protected long memstoreTS; protected int nextKvOffset; protected KeyValue.KeyOnlyKeyValue currentKey = new KeyValue.KeyOnlyKeyValue(); + public SeekerState(boolean tagsCompressed) { + if (tagsCompressed) { + tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE]; + } else { + tagsBuffer = HConstants.EMPTY_BYTE_ARRAY; + } + } + protected boolean isValid() { return valueOffset != -1; } @@ -527,8 +535,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { protected final KVComparator comparator; protected final SamePrefixComparator samePrefixComparator; protected ByteBuffer currentBuffer; - protected STATE current = createSeekerState(); // always valid - protected STATE previous = createSeekerState(); // may not be valid + protected STATE current, previous; protected TagCompressionContext tagCompressionContext = null; public BufferedEncodedSeeker(KVComparator comparator, @@ -543,6 +550,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { throw new RuntimeException("Failed to initialize TagCompressionContext", e); } } + current = createSeekerState(); // always valid + previous = createSeekerState(); // may not be valid } protected boolean includesMvcc() { @@ -553,6 +562,10 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { return this.decodingCtx.getHFileContext().isIncludesTags(); } + protected boolean tagsCompressed() { + return this.decodingCtx.getHFileContext().isCompressTags(); + } + @Override public int compareKey(KVComparator comparator, byte[] key, int offset, int length) { return comparator.compareFlatKey(key, offset, length, @@ -805,7 +818,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { protected STATE createSeekerState() { // This will fail for non-default seeker state if the subclass does not // override this method. - return (STATE) new SeekerState(); + return (STATE) new SeekerState(this.tagsCompressed()); } abstract protected void decodeFirst(); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java index 14048e41e68..e8540b31356 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java @@ -21,12 +21,12 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.KeyValue.KVComparator; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; @@ -362,6 +362,10 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder { private int rowLengthWithSize; private long timestamp; + public DiffSeekerState(boolean tagsCompressed) { + super(tagsCompressed); + } + @Override protected void copyFromNext(SeekerState that) { super.copyFromNext(that); @@ -497,7 +501,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder { @Override protected DiffSeekerState createSeekerState() { - return new DiffSeekerState(); + return new DiffSeekerState(this.tagsCompressed()); } }; } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java index 5e28479b0fb..258e6995c20 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java @@ -21,12 +21,12 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.KeyValue.KVComparator; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; @@ -379,6 +379,10 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder { private int rowLengthWithSize; private int familyLengthWithSize; + public FastDiffSeekerState(boolean tagsCompressed) { + super(tagsCompressed); + } + @Override protected void copyFromNext(SeekerState that) { super.copyFromNext(that); @@ -518,7 +522,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder { @Override protected FastDiffSeekerState createSeekerState() { - return new FastDiffSeekerState(); + return new FastDiffSeekerState(this.tagsCompressed()); } }; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java index 5d9e56abd89..44ae89dd555 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java @@ -27,8 +27,8 @@ public class TestBufferedDataBlockEncoder { @Test public void testEnsureSpaceForKey() { - BufferedDataBlockEncoder.SeekerState state = - new BufferedDataBlockEncoder.SeekerState(); + BufferedDataBlockEncoder.SeekerState state = new BufferedDataBlockEncoder.SeekerState( + false); for (int i = 1; i <= 65536; ++i) { state.keyLength = i; state.ensureSpaceForKey();