Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-13939 - Make HFileReaderImpl.getFirstKeyInBlock() to return a Cell 

(Ram)
This commit is contained in:
ramkrishna 2015-06-26 20:47:30 +05:30
parent 4167a3f69a
commit 130817b559
10 changed files with 41 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
View File

@ -66,13 +66,13 @@ public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder {
}
@Override
public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
public Cell getFirstKeyCellInBlock(ByteBuffer block) {
int keyLength = block.getInt(Bytes.SIZEOF_INT);
ByteBuffer dup = block.duplicate();
int pos = 3 * Bytes.SIZEOF_INT;
dup.position(pos);
dup.limit(pos + keyLength);
return dup.slice();
return new KeyValue.KeyOnlyKeyValue(dup.array(), dup.arrayOffset() + pos, keyLength);
}
@Override

6
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
View File

@ -83,14 +83,14 @@ public interface DataBlockEncoder {
throws IOException;
/**
* Return first key in block. Useful for indexing. Typically does not make
* Return first key in block as a cell. Useful for indexing. Typically does not make
* a deep copy but returns a buffer wrapping a segment of the actual block's
* byte array. This is because the first key in block is usually stored
* unencoded.
* @param block encoded block we want index, the position will not change
* @return First key in block.
* @return First key in block as a cell.
*/
ByteBuffer getFirstKeyInBlock(ByteBuffer block);
Cell getFirstKeyCellInBlock(ByteBuffer block);
/**
* Create a HFileBlock seeker which find KeyValues within a block.

4
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
View File

@ -305,7 +305,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
}
@Override
public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
public Cell getFirstKeyCellInBlock(ByteBuffer block) {
block.mark();
block.position(Bytes.SIZEOF_INT);
byte familyLength = block.get();
@ -350,7 +350,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
block.get(result.array(), pos, Bytes.SIZEOF_BYTE);
block.reset();
return result;
return new KeyValue.KeyOnlyKeyValue(result.array(), 0, keyLength);
}
@Override

4
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
View File

@ -354,7 +354,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
}
@Override
public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
public Cell getFirstKeyCellInBlock(ByteBuffer block) {
block.mark();
block.position(Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE);
int keyLength = ByteBufferUtils.readCompressedInt(block);
@ -365,7 +365,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
ByteBuffer dup = block.duplicate();
dup.position(pos);
dup.limit(pos + keyLength);
return dup.slice();
return new KeyValue.KeyOnlyKeyValue(dup.array(), dup.arrayOffset() + pos, keyLength);
}
@Override

4
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
View File

@ -172,7 +172,7 @@ public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder {
}
@Override
public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
public Cell getFirstKeyCellInBlock(ByteBuffer block) {
block.mark();
block.position(Bytes.SIZEOF_INT);
int keyLength = ByteBufferUtils.readCompressedInt(block);
@ -187,7 +187,7 @@ public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder {
ByteBuffer dup = block.duplicate();
dup.position(pos);
dup.limit(pos + keyLength);
return dup.slice();
return new KeyValue.KeyOnlyKeyValue(dup.array(), dup.arrayOffset() + pos, keyLength);
}
@Override

2
hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
View File

@ -2056,8 +2056,6 @@ public class Bytes implements Comparable<Bytes> {
int low = 0;
int high = arr.length - 1;
KeyValue.KeyOnlyKeyValue r = new KeyValue.KeyOnlyKeyValue();
r.setKey(key, offset, length);
while (low <= high) {
int mid = (low + high) >>> 1;
// we have to compare in this order, because the comparator order

4
hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java
View File

@ -114,7 +114,7 @@ public class PrefixTreeCodec implements DataBlockEncoder {
@Override
public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
public Cell getFirstKeyCellInBlock(ByteBuffer block) {
block.rewind();
PrefixTreeArraySearcher searcher = null;
try {
@ -123,7 +123,7 @@ public class PrefixTreeCodec implements DataBlockEncoder {
if (!searcher.positionAtFirstCell()) {
return null;
}
return KeyValueUtil.copyKeyToNewByteBuffer(searcher.current());
return searcher.current();
} finally {
DecoderFactory.checkIn(searcher);
}

5
hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
View File

@ -23,7 +23,6 @@ import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.SettableSequenceId;
@ -204,10 +203,8 @@ public class PrefixTreeSeeker implements EncodedSeeker {
@Override
public int compareKey(CellComparator comparator, Cell key) {
// can't optimize this, make a copy of the key
ByteBuffer bb = getKeyDeepCopy();
return comparator.compare(key,
new KeyValue.KeyOnlyKeyValue(bb.array(), bb.arrayOffset(), bb.limit()));
ptSearcher.current());
}
/**
* Cloned version of the PrefixTreeCell where except the value part, the rest

21
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
View File

@ -758,12 +758,9 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
if (seekToBlock == null) {
return false;
}
ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
// Will be changed as part of HBASE-13939
Cell firstKey = getFirstKeyCellInBlock(seekToBlock);
if (reader.getComparator()
.compareKeyIgnoresMvcc(
new KeyValue.KeyOnlyKeyValue(firstKey.array(), firstKey.arrayOffset(),
firstKey.limit()), key) >= 0) {
.compareKeyIgnoresMvcc(firstKey, key) >= 0) {
long previousBlockOffset = seekToBlock.getPrevBlockOffset();
// The key we are interested in
if (previousBlockOffset == -1) {
@ -780,8 +777,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
// TODO shortcut: seek forward in this block to the last key of the
// block.
}
Cell firstKeyInCurrentBlock = new KeyValue.KeyOnlyKeyValue(Bytes.getBytes(firstKey));
loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, true);
loadBlockAndSeekToKey(seekToBlock, firstKey, true, key, true);
return true;
}
@ -1033,7 +1029,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
this.nextIndexedKey = null;
}
protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) {
ByteBuffer buffer = curBlock.getBufferWithoutHeader();
// It is safe to manipulate this buffer because we own the buffer object.
buffer.rewind();
@ -1042,7 +1038,10 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
ByteBuffer keyBuff = buffer.slice();
keyBuff.limit(klen);
keyBuff.rewind();
return keyBuff;
// Create a KeyOnlyKv now.
// TODO : Will change when Buffer backed cells come
return new KeyValue.KeyOnlyKeyValue(keyBuff.array(), keyBuff.arrayOffset()
+ keyBuff.position(), klen);
}
@Override
@ -1556,8 +1555,8 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
}
}
protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) {
return dataBlockEncoder.getFirstKeyCellInBlock(getEncodedBuffer(curBlock));
}
protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,

24
hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
View File

@ -30,7 +30,9 @@ import java.util.Collection;
import java.util.List;
import java.util.Random;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
@ -300,23 +302,31 @@ public class TestDataBlockEncoders {
DataBlockEncoder encoder = encoding.getEncoder();
ByteBuffer encodedBuffer = encodeKeyValues(encoding, sampleKv,
getEncodingContext(Compression.Algorithm.NONE, encoding));
ByteBuffer keyBuffer = encoder.getFirstKeyInBlock(encodedBuffer);
Cell key = encoder.getFirstKeyCellInBlock(encodedBuffer);
KeyValue keyBuffer = null;
if(encoding == DataBlockEncoding.PREFIX_TREE) {
// This is not an actual case. So the Prefix tree block is not loaded in case of Prefix_tree
// Just copy only the key part to form a keyBuffer
byte[] serializedKey = CellUtil.getCellKeySerializedAsKeyValueKey(key);
keyBuffer = KeyValueUtil.createKeyValueFromKey(serializedKey);
} else {
keyBuffer = KeyValueUtil.ensureKeyValue(key);
}
KeyValue firstKv = sampleKv.get(0);
if (0 != Bytes.compareTo(keyBuffer.array(), keyBuffer.arrayOffset(), keyBuffer.limit(),
firstKv.getBuffer(), firstKv.getKeyOffset(), firstKv.getKeyLength())) {
if (0 != CellComparator.COMPARATOR.compareKeyIgnoresMvcc(keyBuffer, firstKv)) {
int commonPrefix = 0;
int length = Math.min(keyBuffer.limit(), firstKv.getKeyLength());
int length = Math.min(keyBuffer.getKeyLength(), firstKv.getKeyLength());
while (commonPrefix < length
&& keyBuffer.array()[keyBuffer.arrayOffset() + commonPrefix] == firstKv.getBuffer()[firstKv
.getKeyOffset() + commonPrefix]) {
&& keyBuffer.getBuffer()[keyBuffer.getKeyOffset() + commonPrefix] == firstKv
.getBuffer()[firstKv.getKeyOffset() + commonPrefix]) {
commonPrefix++;
}
fail(String.format("Bug in '%s' commonPrefix %d", encoder.toString(), commonPrefix));
}
}
}
private void checkSeekingConsistency(List<DataBlockEncoder.EncodedSeeker> encodedSeekers,
boolean seekBefore, KeyValue keyValue) {
ByteBuffer expectedKeyValue = null;