Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-12210 Avoid KeyValue in Prefix Tree (Ram)

This commit is contained in:
Ramkrishna 2014-10-10 22:45:26 +05:30
parent 6ddb2f1965
commit 3f2e599a99
4 changed files with 228 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java
View File

@ -21,9 +21,9 @@ package org.apache.hadoop.hbase;
import java.io.Serializable;
import java.util.Comparator;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.util.Bytes;
import com.google.common.primitives.Longs;
@ -255,6 +255,29 @@ public class CellComparator implements Comparator<Cell>, Serializable{
return 0;
}
int hash = calculateHashForKeyValue(cell);
hash = 31 * hash + (int)cell.getMvccVersion();
return hash;
}
/**
* Returns a hash code that is always the same for two Cells having a matching
* equals(..) result. Currently does not guard against nulls, but it could if
* necessary. Note : Ignore mvcc while calculating the hashcode
*
* @param cell
* @return hashCode
*/
public static int hashCodeIgnoreMvcc(Cell cell) {
if (cell == null) {// return 0 for empty Cell
return 0;
}
int hash = calculateHashForKeyValue(cell);
return hash;
}
private static int calculateHashForKeyValue(Cell cell) {
//pre-calculate the 3 hashes made of byte ranges
int rowHash = Bytes.hashCode(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
int familyHash =
@ -267,7 +290,6 @@ public class CellComparator implements Comparator<Cell>, Serializable{
hash = 31 * hash + qualifierHash;
hash = 31 * hash + (int)cell.getTimestamp();
hash = 31 * hash + cell.getTypeByte();
hash = 31 * hash + (int)cell.getMvccVersion();
return hash;
}

192
hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
View File

@ -20,12 +20,13 @@ package org.apache.hadoop.hbase.codec.prefixtree;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.SettableSequenceId;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory;
import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher;
import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellScannerPosition;
@ -91,11 +92,17 @@ public class PrefixTreeSeeker implements EncodedSeeker {
* currently must do deep copy into new array
*/
@Override
public KeyValue getKeyValue() {
if (ptSearcher.current() == null) {
public Cell getKeyValue() {
Cell cell = ptSearcher.current();
if (cell == null) {
return null;
}
return KeyValueUtil.copyToNewKeyValue(ptSearcher.current());
return new ClonedPrefixTreeCell(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(), cell.getTagsArray(),
cell.getTagsOffset(), cell.getTagsLength(), cell.getTimestamp(), cell.getTypeByte(),
cell.getSequenceId());
}
/**
@ -248,7 +255,7 @@ public class PrefixTreeSeeker implements EncodedSeeker {
public int seekToKeyInBlock(Cell key, boolean forceBeforeOnExactMatch) {
if (USE_POSITION_BEFORE) {
return seekToOrBeforeUsingPositionAtOrBefore(key, forceBeforeOnExactMatch);
}else{
} else {
return seekToOrBeforeUsingPositionAtOrAfter(key, forceBeforeOnExactMatch);
}
}
@ -259,4 +266,179 @@ public class PrefixTreeSeeker implements EncodedSeeker {
return comparator.compare(key,
new KeyValue.KeyOnlyKeyValue(bb.array(), bb.arrayOffset(), bb.limit()));
}
/**
* Cloned version of the PrefixTreeCell where except the value part, the rest
* of the key part is deep copied
*
*/
private static class ClonedPrefixTreeCell implements Cell, SettableSequenceId {
private byte[] row;
private short rowLength;
private byte[] fam;
private byte famLength;
private byte[] qual;
private int qualLength;
private byte[] val;
private int valOffset;
private int valLength;
private byte[] tag;
private int tagLength;
private long ts;
private long seqId;
private byte type;
public ClonedPrefixTreeCell(byte[] row, int rowOffset, short rowLength, byte[] fam,
int famOffset, byte famLength, byte[] qual, int qualOffset, int qualLength, byte[] val,
int valOffset, int valLength, byte[] tag, int tagOffset, int tagLength, long ts, byte type,
long seqId) {
this.row = new byte[rowLength];
System.arraycopy(row, rowOffset, this.row, 0, rowLength);
this.rowLength = rowLength;
this.fam = new byte[famLength];
System.arraycopy(fam, famOffset, this.fam, 0, famLength);
this.famLength = famLength;
this.qual = new byte[qualLength];
System.arraycopy(qual, qualOffset, this.qual, 0, qualLength);
this.qualLength = qualLength;
this.tag = new byte[tagLength];
System.arraycopy(tag, tagOffset, this.tag, 0, tagLength);
this.tagLength = tagLength;
this.val = val;
this.valLength = valLength;
this.valOffset = valOffset;
this.ts = ts;
this.seqId = seqId;
this.type = type;
}
@Override
public void setSequenceId(long seqId) {
this.seqId = seqId;
}
@Override
public byte[] getRowArray() {
return this.row;
}
@Override
public int getRowOffset() {
return 0;
}
@Override
public short getRowLength() {
return this.rowLength;
}
@Override
public byte[] getFamilyArray() {
return this.fam;
}
@Override
public int getFamilyOffset() {
return 0;
}
@Override
public byte getFamilyLength() {
return this.famLength;
}
@Override
public byte[] getQualifierArray() {
return this.qual;
}
@Override
public int getQualifierOffset() {
return 0;
}
@Override
public int getQualifierLength() {
return this.qualLength;
}
@Override
public long getTimestamp() {
return ts;
}
@Override
public byte getTypeByte() {
return type;
}
@Override
@Deprecated
public long getMvccVersion() {
return getSequenceId();
}
@Override
public long getSequenceId() {
return seqId;
}
@Override
public byte[] getValueArray() {
return val;
}
@Override
public int getValueOffset() {
return this.valOffset;
}
@Override
public int getValueLength() {
return this.valLength;
}
@Override
public byte[] getTagsArray() {
return this.tag;
}
@Override
public int getTagsOffset() {
return 0;
}
@Override
public int getTagsLength() {
return this.tagLength;
}
@Override
@Deprecated
public byte[] getValue() {
return this.val;
}
@Override
@Deprecated
public byte[] getFamily() {
return this.fam;
}
@Override
@Deprecated
public byte[] getQualifier() {
return this.qual;
}
@Override
@Deprecated
public byte[] getRow() {
return this.row;
}
@Override
public String toString() {
return KeyValueUtil.copyToNewKeyValue(this).toString();
}
}
}

18
hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java
View File

@ -18,12 +18,13 @@
package org.apache.hadoop.hbase.codec.prefixtree.decode;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.SettableSequenceId;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
/**
* As the PrefixTreeArrayScanner moves through the tree bytes, it changes the values in the fields
@ -31,7 +32,7 @@ import org.apache.hadoop.hbase.KeyValueUtil;
* iterated through.
*/
@InterfaceAudience.Private
public class PrefixTreeCell implements Cell, Comparable<Cell> {
public class PrefixTreeCell implements Cell, SettableSequenceId, Comparable<Cell> {
/********************** static **********************/
@ -96,12 +97,8 @@ public class PrefixTreeCell implements Cell, Comparable<Cell> {
}
@Override
public int hashCode(){
//Temporary hack to maintain backwards compatibility with KeyValue.hashCode
//I don't think this is used in any hot code paths
return KeyValueUtil.copyToNewKeyValue(this).hashCode();
//TODO return CellComparator.hashCode(this);//see HBASE-6907
public int hashCode() {
return CellComparator.hashCodeIgnoreMvcc(this);
}
@Override
@ -237,4 +234,9 @@ public class PrefixTreeCell implements Cell, Comparable<Cell> {
public byte[] getTagsArray() {
return this.tagsBuffer;
}
@Override
public void setSequenceId(long seqId) {
mvccVersion = seqId;
}
}

7
hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
View File

@ -116,6 +116,10 @@ public class TestPrefixTree {
rows[0] = row1;
rows[1] = row2;
rows[2] = row3;
byte[][] val = new byte[3][];
val[0] = Bytes.toBytes("c1-value");
val[1] = Bytes.toBytes("c2-value");
val[2] = Bytes.toBytes("c2-value-2");
Scan scan = new Scan();
scan.setStartRow(row1_bytes);
scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
@ -128,6 +132,9 @@ public class TestPrefixTree {
while (cellScanner.advance()) {
assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
.current().getRowOffset(), cellScanner.current().getRowLength()));
assertEquals(Bytes.toString(val[i]), Bytes.toString(
cellScanner.current().getValueArray(), cellScanner.current().getValueOffset(),
cellScanner.current().getValueLength()));
}
i++;
}