Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-5625 Avoid byte buffer allocations when reading a value from a Result object (Tudor Scurtu) 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1333159 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Zhihong Yu 2012-05-02 19:19:01 +00:00
parent 00cfc8f7f7
commit d032ea818b
4 changed files with 791 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

396
src/main/java/org/apache/hadoop/hbase/KeyValue.java
View File

@ -22,6 +22,7 @@ package org.apache.hadoop.hbase;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.Comparator;
import java.util.HashMap;
@ -166,6 +167,37 @@ public class KeyValue implements Writable, HeapSize {
// Size of the length ints in a KeyValue datastructure.
public static final int KEYVALUE_INFRASTRUCTURE_SIZE = ROW_OFFSET;
/**
* Computes the number of bytes that a <code>KeyValue</code> instance with the provided
* characteristics would take up for its underlying data structure.
*
* @param rlength row length
* @param flength family length
* @param qlength qualifier length
* @param vlength value length
*
* @return the <code>KeyValue</code> data structure length
*/
public static long getKeyValueDataStructureSize(int rlength,
int flength, int qlength, int vlength) {
return KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE +
getKeyDataStructureSize(rlength, flength, qlength) + vlength;
}
/**
* Computes the number of bytes that a <code>KeyValue</code> instance with the provided
* characteristics would take up in its underlying data structure for the key.
*
* @param rlength row length
* @param flength family length
* @param qlength qualifier length
*
* @return the key data structure length
*/
public static long getKeyDataStructureSize(int rlength, int flength, int qlength) {
return KeyValue.KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength;
}
/**
* Key type.
* Has space for other key types to be added later. Cannot rely on
@ -478,7 +510,7 @@ public class KeyValue implements Writable, HeapSize {
throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
}
// Key length
long longkeylength = KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength;
long longkeylength = getKeyDataStructureSize(rlength, flength, qlength);
if (longkeylength > Integer.MAX_VALUE) {
throw new IllegalArgumentException("keylength " + longkeylength + " > " +
Integer.MAX_VALUE);
@ -491,7 +523,8 @@ public class KeyValue implements Writable, HeapSize {
}
// Allocate right-sized byte array.
byte [] bytes = new byte[KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength];
byte [] bytes =
new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength)];
// Write the correct size markers
int pos = 0;
pos = Bytes.putInt(bytes, pos, keylength);
@ -505,6 +538,198 @@ public class KeyValue implements Writable, HeapSize {
return bytes;
}
/**
* Constructs KeyValue structure filled with specified values. Uses the provided buffer as its
* backing data buffer.
* <p>
* Column is split into two fields, family and qualifier.
*
* @param buffer the bytes buffer to use
* @param row row key
* @param roffset row offset
* @param rlength row length
* @param family family name
* @param foffset family offset
* @param flength family length
* @param qualifier column qualifier
* @param qoffset qualifier offset
* @param qlength qualifier length
* @param timestamp version timestamp
* @param type key type
* @param value column value
* @param voffset value offset
* @param vlength value length
* @throws IllegalArgumentException an illegal value was passed or there is insufficient space
* remaining in the buffer
*/
public KeyValue(byte [] buffer,
final byte [] row, final int roffset, final int rlength,
final byte [] family, final int foffset, final int flength,
final byte [] qualifier, final int qoffset, final int qlength,
final long timestamp, final Type type,
final byte [] value, final int voffset, final int vlength) {
this(buffer, 0,
row, roffset, rlength,
family, foffset, flength,
qualifier, qoffset, qlength,
timestamp, type,
value, voffset, vlength);
}
/**
* Constructs KeyValue structure filled with specified values. Uses the provided buffer as the
* data buffer.
* <p>
* Column is split into two fields, family and qualifier.
*
* @param buffer the bytes buffer to use
* @param boffset buffer offset
* @param row row key
* @param roffset row offset
* @param rlength row length
* @param family family name
* @param foffset family offset
* @param flength family length
* @param qualifier column qualifier
* @param qoffset qualifier offset
* @param qlength qualifier length
* @param timestamp version timestamp
* @param type key type
* @param value column value
* @param voffset value offset
* @param vlength value length
* @throws IllegalArgumentException an illegal value was passed or there is insufficient space
* remaining in the buffer
*/
public KeyValue(byte [] buffer, final int boffset,
final byte [] row, final int roffset, final int rlength,
final byte [] family, final int foffset, final int flength,
final byte [] qualifier, final int qoffset, final int qlength,
final long timestamp, final Type type,
final byte [] value, final int voffset, final int vlength) {
this.bytes = buffer;
this.length = writeByteArray(buffer, boffset,
row, roffset, rlength,
family, foffset, flength, qualifier, qoffset, qlength,
timestamp, type, value, voffset, vlength);
this.offset = boffset;
}
/**
* Checks the parameters passed to a constructor.
*
* @param row row key
* @param rlength row length
* @param family family name
* @param flength family length
* @param qualifier column qualifier
* @param qlength qualifier length
* @param value column value
* @param vlength value length
*
* @throws IllegalArgumentException an illegal value was passed
*/
private static void checkParameters(final byte [] row, final int rlength,
final byte [] family, int flength,
final byte [] qualifier, int qlength,
final byte [] value, int vlength)
throws IllegalArgumentException {
if (rlength > Short.MAX_VALUE) {
throw new IllegalArgumentException("Row > " + Short.MAX_VALUE);
}
if (row == null) {
throw new IllegalArgumentException("Row is null");
}
// Family length
flength = family == null ? 0 : flength;
if (flength > Byte.MAX_VALUE) {
throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE);
}
// Qualifier length
qlength = qualifier == null ? 0 : qlength;
if (qlength > Integer.MAX_VALUE - rlength - flength) {
throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
}
// Key length
long longKeyLength = getKeyDataStructureSize(rlength, flength, qlength);
if (longKeyLength > Integer.MAX_VALUE) {
throw new IllegalArgumentException("keylength " + longKeyLength + " > " +
Integer.MAX_VALUE);
}
// Value length
vlength = value == null? 0 : vlength;
if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON
throw new IllegalArgumentException("Value length " + vlength + " > " +
HConstants.MAXIMUM_VALUE_LENGTH);
}
}
/**
* Write KeyValue format into the provided byte array.
*
* @param buffer the bytes buffer to use
* @param boffset buffer offset
* @param row row key
* @param roffset row offset
* @param rlength row length
* @param family family name
* @param foffset family offset
* @param flength family length
* @param qualifier column qualifier
* @param qoffset qualifier offset
* @param qlength qualifier length
* @param timestamp version timestamp
* @param type key type
* @param value column value
* @param voffset value offset
* @param vlength value length
*
* @return The number of useful bytes in the buffer.
*
* @throws IllegalArgumentException an illegal value was passed or there is insufficient space
* remaining in the buffer
*/
static int writeByteArray(byte [] buffer, final int boffset,
final byte [] row, final int roffset, final int rlength,
final byte [] family, final int foffset, int flength,
final byte [] qualifier, final int qoffset, int qlength,
final long timestamp, final Type type,
final byte [] value, final int voffset, int vlength) {
checkParameters(row, rlength, family, flength, qualifier, qlength, value, vlength);
int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
int keyValueLength = (int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength);
if (keyValueLength > buffer.length - boffset) {
throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < " +
keyValueLength);
}
// Write key, value and key row length.
int pos = boffset;
pos = Bytes.putInt(buffer, pos, keyLength);
pos = Bytes.putInt(buffer, pos, vlength);
pos = Bytes.putShort(buffer, pos, (short)(rlength & 0x0000ffff));
pos = Bytes.putBytes(buffer, pos, row, roffset, rlength);
pos = Bytes.putByte(buffer, pos, (byte) (flength & 0x0000ff));
if (flength != 0) {
pos = Bytes.putBytes(buffer, pos, family, foffset, flength);
}
if (qlength != 0) {
pos = Bytes.putBytes(buffer, pos, qualifier, qoffset, qlength);
}
pos = Bytes.putLong(buffer, pos, timestamp);
pos = Bytes.putByte(buffer, pos, type.getCode());
if (value != null && value.length > 0) {
pos = Bytes.putBytes(buffer, pos, value, voffset, vlength);
}
return keyValueLength;
}
/**
* Write KeyValue format into a byte array.
*
@ -529,41 +754,16 @@ public class KeyValue implements Writable, HeapSize {
final byte [] qualifier, final int qoffset, int qlength,
final long timestamp, final Type type,
final byte [] value, final int voffset, int vlength) {
if (rlength > Short.MAX_VALUE) {
throw new IllegalArgumentException("Row > " + Short.MAX_VALUE);
}
if (row == null) {
throw new IllegalArgumentException("Row is null");
}
// Family length
flength = family == null ? 0 : flength;
if (flength > Byte.MAX_VALUE) {
throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE);
}
// Qualifier length
qlength = qualifier == null ? 0 : qlength;
if (qlength > Integer.MAX_VALUE - rlength - flength) {
throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
}
// Key length
long longkeylength = KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength;
if (longkeylength > Integer.MAX_VALUE) {
throw new IllegalArgumentException("keylength " + longkeylength + " > " +
Integer.MAX_VALUE);
}
int keylength = (int)longkeylength;
// Value length
vlength = value == null? 0 : vlength;
if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON
throw new IllegalArgumentException("Valuer > " +
HConstants.MAXIMUM_VALUE_LENGTH);
}
checkParameters(row, rlength, family, flength, qualifier, qlength, value, vlength);
// Allocate right-sized byte array.
byte [] bytes = new byte[KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength];
int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
byte [] bytes =
new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength)];
// Write key, value and key row length.
int pos = 0;
pos = Bytes.putInt(bytes, pos, keylength);
pos = Bytes.putInt(bytes, pos, keyLength);
pos = Bytes.putInt(bytes, pos, vlength);
pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
pos = Bytes.putBytes(bytes, pos, row, roffset, rlength);
@ -913,8 +1113,7 @@ public class KeyValue implements Writable, HeapSize {
* @return Qualifier length
*/
public int getQualifierLength(int rlength, int flength) {
return getKeyLength() -
(KEY_INFRASTRUCTURE_SIZE + rlength + flength);
return getKeyLength() - (int) getKeyDataStructureSize(rlength, flength, 0);
}
/**
@ -1007,6 +1206,28 @@ public class KeyValue implements Writable, HeapSize {
return result;
}
/**
* Returns the value wrapped in a new <code>ByteBuffer</code>.
*
* @return the value
*/
public ByteBuffer getValueAsByteBuffer() {
return ByteBuffer.wrap(getBuffer(), getValueOffset(), getValueLength());
}
/**
* Loads this object's value into the provided <code>ByteBuffer</code>.
* <p>
* Does not clear or flip the buffer.
*
* @param dst the buffer where to write the value
*
* @throws BufferOverflowException if there is insufficient space remaining in the buffer
*/
public void loadValue(ByteBuffer dst) throws BufferOverflowException {
dst.put(getBuffer(), getValueOffset(), getValueLength());
}
/**
* Primarily for use client-side. Returns the row of this KeyValue in a new
* byte array.<p>
@ -1278,21 +1499,36 @@ public class KeyValue implements Writable, HeapSize {
* @return True if column matches
*/
public boolean matchingColumn(final byte[] family, final byte[] qualifier) {
return matchingColumn(family, 0, family == null ? 0 : family.length,
qualifier, 0, qualifier == null ? 0 : qualifier.length);
}
/**
* Checks if column matches.
*
* @param family family name
* @param foffset family offset
* @param flength family length
* @param qualifier column qualifier
* @param qoffset qualifier offset
* @param qlength qualifier length
*
* @return True if column matches
*/
public boolean matchingColumn(final byte [] family, final int foffset, final int flength,
final byte [] qualifier, final int qoffset, final int qlength) {
int rl = getRowLength();
int o = getFamilyOffset(rl);
int fl = getFamilyLength(o);
int ql = getQualifierLength(rl,fl);
if (!Bytes.equals(family, 0, family.length, this.bytes, o, fl)) {
if (!Bytes.equals(family, foffset, flength, this.bytes, o, fl)) {
return false;
}
if (qualifier == null || qualifier.length == 0) {
if (ql == 0) {
return true;
}
return false;
int ql = getQualifierLength(rl, fl);
if (qualifier == null || qlength == 0) {
return (ql == 0);
}
return Bytes.equals(qualifier, 0, qualifier.length,
this.bytes, o + fl, ql);
return Bytes.equals(qualifier, qoffset, qlength, this.bytes, o + fl, ql);
}
/**
@ -1820,6 +2056,78 @@ public class KeyValue implements Writable, HeapSize {
HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
}
/**
* Create a KeyValue for the specified row, family and qualifier that would be
* smaller than all other possible KeyValues that have the same row,
* family, qualifier.
* Used for seeking.
*
* @param buffer the buffer to use for the new <code>KeyValue</code> object
* @param row the value key
* @param family family name
* @param qualifier column qualifier
*
* @return First possible key on passed Row, Family, Qualifier.
*
* @throws IllegalArgumentException The resulting <code>KeyValue</code> object would be larger
* than the provided buffer or than <code>Integer.MAX_VALUE</code>
*/
public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row,
final byte [] family, final byte [] qualifier)
throws IllegalArgumentException {
return createFirstOnRow(buffer, 0, row, 0, row.length,
family, 0, family.length,
qualifier, 0, qualifier.length);
}
/**
* Create a KeyValue for the specified row, family and qualifier that would be
* smaller than all other possible KeyValues that have the same row,
* family, qualifier.
* Used for seeking.
*
* @param buffer the buffer to use for the new <code>KeyValue</code> object
* @param boffset buffer offset
* @param row the value key
* @param roffset row offset
* @param rlength row length
* @param family family name
* @param foffset family offset
* @param flength family length
* @param qualifier column qualifier
* @param qoffset qualifier offset
* @param qlength qualifier length
*
* @return First possible key on passed Row, Family, Qualifier.
*
* @throws IllegalArgumentException The resulting <code>KeyValue</code> object would be larger
* than the provided buffer or than <code>Integer.MAX_VALUE</code>
*/
public static KeyValue createFirstOnRow(byte [] buffer, final int boffset,
final byte [] row, final int roffset, final int rlength,
final byte [] family, final int foffset, final int flength,
final byte [] qualifier, final int qoffset, final int qlength)
throws IllegalArgumentException {
long lLength = getKeyValueDataStructureSize(rlength, flength, qlength, 0);
if (lLength > Integer.MAX_VALUE) {
throw new IllegalArgumentException("KeyValue length " + lLength + " > " + Integer.MAX_VALUE);
}
int iLength = (int) lLength;
if (buffer.length - boffset < iLength) {
throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < " +
iLength);
}
return new KeyValue(buffer, boffset,
row, roffset, rlength,
family, foffset, flength,
qualifier, qoffset, qlength,
HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum,
null, 0, 0);
}
/**
* Create a KeyValue for the specified row, family and qualifier that would be
* larger than or equal to all other possible KeyValues that have the same

264
src/main/java/org/apache/hadoop/hbase/client/Result.java
View File

@ -23,6 +23,8 @@ package org.apache.hadoop.hbase.client;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@ -71,6 +73,7 @@ import org.apache.hadoop.io.Writable;
@InterfaceStability.Stable
public class Result implements Writable, WritableWithSize {
private static final byte RESULT_VERSION = (byte)1;
private static final int DEFAULT_BUFFER_SIZE = 1024;
private KeyValue [] kvs = null;
private NavigableMap<byte[],
@ -80,6 +83,10 @@ public class Result implements Writable, WritableWithSize {
private transient byte [] row = null;
private ImmutableBytesWritable bytes = null;
// never use directly
private static byte [] buffer = null;
private static final int PAD_WIDTH = 128;
/**
* Constructor used for Writable.
*/
@ -228,13 +235,56 @@ public class Result implements Writable, WritableWithSize {
}
/**
* The KeyValue for the most recent for a given column. If the column does
* not exist in the result set - if it wasn't selected in the query (Get/Scan)
* or just does not exist in the row the return value is null.
* Searches for the latest value for the specified column.
*
* @param kvs the array to search
* @param family family name
* @param foffset family offset
* @param flength family length
* @param qualifier column qualifier
* @param qoffset qualifier offset
* @param qlength qualifier length
*
* @return the index where the value was found, or -1 otherwise
*/
protected int binarySearch(final KeyValue [] kvs,
final byte [] family, final int foffset, final int flength,
final byte [] qualifier, final int qoffset, final int qlength) {
double keyValueSize = (double)
KeyValue.getKeyValueDataStructureSize(kvs[0].getRowLength(), flength, qlength, 0);
if (buffer == null || keyValueSize > buffer.length) {
// pad to the smallest multiple of the pad width
buffer = new byte[(int) Math.ceil(keyValueSize / PAD_WIDTH) * PAD_WIDTH];
}
KeyValue searchTerm = KeyValue.createFirstOnRow(buffer, 0,
kvs[0].getBuffer(), kvs[0].getRowOffset(), kvs[0].getRowLength(),
family, foffset, flength,
qualifier, qoffset, qlength);
// pos === ( -(insertion point) - 1)
int pos = Arrays.binarySearch(kvs, searchTerm, KeyValue.COMPARATOR);
// never will exact match
if (pos < 0) {
pos = (pos+1) * -1;
// pos is now insertion point
}
if (pos == kvs.length) {
return -1; // doesn't exist
}
return pos;
}
/**
* The KeyValue for the most recent timestamp for a given column.
*
* @param family
* @param qualifier
* @return KeyValue for the column or null
*
* @return the KeyValue for the column, or null if no value exists in the row or none have been
* selected in the query (Get/Scan)
*/
public KeyValue getColumnLatest(byte [] family, byte [] qualifier) {
KeyValue [] kvs = raw(); // side effect possibly.
@ -252,6 +302,37 @@ public class Result implements Writable, WritableWithSize {
return null;
}
/**
* The KeyValue for the most recent timestamp for a given column.
*
* @param family family name
* @param foffset family offset
* @param flength family length
* @param qualifier column qualifier
* @param qoffset qualifier offset
* @param qlength qualifier length
*
* @return the KeyValue for the column, or null if no value exists in the row or none have been
* selected in the query (Get/Scan)
*/
public KeyValue getColumnLatest(byte [] family, int foffset, int flength,
byte [] qualifier, int qoffset, int qlength) {
KeyValue [] kvs = raw(); // side effect possibly.
if (kvs == null || kvs.length == 0) {
return null;
}
int pos = binarySearch(kvs, family, foffset, flength, qualifier, qoffset, qlength);
if (pos == -1) {
return null;
}
KeyValue kv = kvs[pos];
if (kv.matchingColumn(family, foffset, flength, qualifier, qoffset, qlength)) {
return kv;
}
return null;
}
/**
* Get the latest version of the specified column.
* @param family family name
@ -267,9 +348,164 @@ public class Result implements Writable, WritableWithSize {
}
/**
* Checks for existence of the specified column.
* Returns the value wrapped in a new <code>ByteBuffer</code>.
*
* @param family family name
* @param qualifier column qualifier
*
* @return the latest version of the column, or <code>null</code> if none found
*/
public ByteBuffer getValueAsByteBuffer(byte [] family, byte [] qualifier) {
KeyValue kv = getColumnLatest(family, 0, family.length, qualifier, 0, qualifier.length);
if (kv == null) {
return null;
}
return kv.getValueAsByteBuffer();
}
/**
* Returns the value wrapped in a new <code>ByteBuffer</code>.
*
* @param family family name
* @param foffset family offset
* @param flength family length
* @param qualifier column qualifier
* @param qoffset qualifier offset
* @param qlength qualifier length
*
* @return the latest version of the column, or <code>null</code> if none found
*/
public ByteBuffer getValueAsByteBuffer(byte [] family, int foffset, int flength,
byte [] qualifier, int qoffset, int qlength) {
KeyValue kv = getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength);
if (kv == null) {
return null;
}
return kv.getValueAsByteBuffer();
}
/**
* Loads the latest version of the specified column into the provided <code>ByteBuffer</code>.
* <p>
* Does not clear or flip the buffer.
*
* @param family family name
* @param qualifier column qualifier
* @param dst the buffer where to write the value
*
* @return <code>true</code> if a value was found, <code>false</code> otherwise
*
* @throws BufferOverflowException there is insufficient space remaining in the buffer
*/
public boolean loadValue(byte [] family, byte [] qualifier, ByteBuffer dst)
throws BufferOverflowException {
return loadValue(family, 0, family.length, qualifier, 0, qualifier.length, dst);
}
/**
* Loads the latest version of the specified column into the provided <code>ByteBuffer</code>.
* <p>
* Does not clear or flip the buffer.
*
* @param family family name
* @param foffset family offset
* @param flength family length
* @param qualifier column qualifier
* @param qoffset qualifier offset
* @param qlength qualifier length
* @param dst the buffer where to write the value
*
* @return <code>true</code> if a value was found, <code>false</code> otherwise
*
* @throws BufferOverflowException there is insufficient space remaining in the buffer
*/
public boolean loadValue(byte [] family, int foffset, int flength,
byte [] qualifier, int qoffset, int qlength, ByteBuffer dst)
throws BufferOverflowException {
KeyValue kv = getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength);
if (kv == null) {
return false;
}
kv.loadValue(dst);
return true;
}
/**
* Checks if the specified column contains a non-empty value (not a zero-length byte array).
*
* @param family family name
* @param qualifier column qualifier
*
* @return whether or not a latest value exists and is not empty
*/
public boolean containsNonEmptyColumn(byte [] family, byte [] qualifier) {
return containsNonEmptyColumn(family, 0, family.length, qualifier, 0, qualifier.length);
}
/**
* Checks if the specified column contains a non-empty value (not a zero-length byte array).
*
* @param family family name
* @param foffset family offset
* @param flength family length
* @param qualifier column qualifier
* @param qoffset qualifier offset
* @param qlength qualifier length
*
* @return whether or not a latest value exists and is not empty
*/
public boolean containsNonEmptyColumn(byte [] family, int foffset, int flength,
byte [] qualifier, int qoffset, int qlength) {
KeyValue kv = getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength);
return (kv != null) && (kv.getValueLength() > 0);
}
/**
* Checks if the specified column contains an empty value (a zero-length byte array).
*
* @param family family name
* @param qualifier column qualifier
*
* @return whether or not a latest value exists and is empty
*/
public boolean containsEmptyColumn(byte [] family, byte [] qualifier) {
return containsEmptyColumn(family, 0, family.length, qualifier, 0, qualifier.length);
}
/**
* Checks if the specified column contains an empty value (a zero-length byte array).
*
* @param family family name
* @param foffset family offset
* @param flength family length
* @param qualifier column qualifier
* @param qoffset qualifier offset
* @param qlength qualifier length
*
* @return whether or not a latest value exists and is empty
*/
public boolean containsEmptyColumn(byte [] family, int foffset, int flength,
byte [] qualifier, int qoffset, int qlength) {
KeyValue kv = getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength);
return (kv != null) && (kv.getValueLength() == 0);
}
/**
* Checks for existence of a value for the specified column (empty or not).
*
* @param family family name
* @param qualifier column qualifier
*
* @return true if at least one value exists in the result, false if not
*/
public boolean containsColumn(byte [] family, byte [] qualifier) {
@ -277,6 +513,24 @@ public class Result implements Writable, WritableWithSize {
return kv != null;
}
/**
* Checks for existence of a value for the specified column (empty or not).
*
* @param family family name
* @param foffset family offset
* @param flength family length
* @param qualifier column qualifier
* @param qoffset qualifier offset
* @param qlength qualifier length
*
* @return true if at least one value exists in the result, false if not
*/
public boolean containsColumn(byte [] family, int foffset, int flength,
byte [] qualifier, int qoffset, int qlength) {
return getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength) != null;
}
/**
* Map of families to all versions of its qualifiers and values.
* <p>

18
src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
View File

@ -344,31 +344,47 @@ public class TestKeyValue extends TestCase {
public void testFirstLastOnRow() {
final KVComparator c = KeyValue.COMPARATOR;
long ts = 1;
byte[] bufferA = new byte[128];
int offsetA = 0;
byte[] bufferB = new byte[128];
int offsetB = 7;
// These are listed in sort order (ie: every one should be less
// than the one on the next line).
final KeyValue firstOnRowA = KeyValue.createFirstOnRow(rowA);
final KeyValue firstOnRowABufferFamQual = KeyValue.createFirstOnRow(bufferA, offsetA,
rowA, 0, rowA.length, family, 0, family.length, qualA, 0, qualA.length);
final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, Type.Put);
final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, Type.Put);
final KeyValue lastOnRowA = KeyValue.createLastOnRow(rowA);
final KeyValue firstOnRowB = KeyValue.createFirstOnRow(rowB);
final KeyValue firstOnRowBBufferFam = KeyValue.createFirstOnRow(bufferB, offsetB,
rowB, 0, rowB.length, family, 0, family.length, null, 0, 0);
final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, Type.Put);
assertKVLess(c, firstOnRowA, firstOnRowB);
assertKVLess(c, firstOnRowA, firstOnRowBBufferFam);
assertKVLess(c, firstOnRowABufferFamQual, firstOnRowB);
assertKVLess(c, firstOnRowA, kvA_1);
assertKVLess(c, firstOnRowA, kvA_2);
assertKVLess(c, firstOnRowABufferFamQual, kvA_2);
assertKVLess(c, kvA_1, kvA_2);
assertKVLess(c, kvA_2, firstOnRowB);
assertKVLess(c, kvA_1, firstOnRowB);
assertKVLess(c, kvA_2, firstOnRowBBufferFam);
assertKVLess(c, kvA_1, firstOnRowBBufferFam);
assertKVLess(c, lastOnRowA, firstOnRowB);
assertKVLess(c, lastOnRowA, firstOnRowBBufferFam);
assertKVLess(c, firstOnRowB, kvB);
assertKVLess(c, firstOnRowBBufferFam, kvB);
assertKVLess(c, lastOnRowA, kvB);
assertKVLess(c, kvA_2, lastOnRowA);
assertKVLess(c, kvA_1, lastOnRowA);
assertKVLess(c, firstOnRowA, lastOnRowA);
assertKVLess(c, firstOnRowABufferFamQual, lastOnRowA);
}
public void testCreateKeyOnly() throws Exception {

170
src/test/java/org/apache/hadoop/hbase/client/TestResult.java
View File

@ -21,6 +21,8 @@
package org.apache.hadoop.hbase.client;
import junit.framework.TestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
@ -28,6 +30,7 @@ import org.junit.experimental.categories.Category;
import static org.apache.hadoop.hbase.HBaseTestCase.assertByteEquals;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@ -36,6 +39,8 @@ import java.util.NavigableMap;
@Category(SmallTests.class)
public class TestResult extends TestCase {
private static final Log LOG = LogFactory.getLog(TestResult.class.getName());
static KeyValue[] genKVs(final byte[] row, final byte[] family,
final byte[] value,
final long timestamp,
@ -55,7 +60,7 @@ public class TestResult extends TestCase {
static final byte [] family = Bytes.toBytes("family");
static final byte [] value = Bytes.toBytes("value");
public void testBasic() throws Exception {
public void testBasicGetColumn() throws Exception {
KeyValue [] kvs = genKVs(row, family, value, 1, 100);
Arrays.sort(kvs, KeyValue.COMPARATOR);
@ -68,13 +73,11 @@ public class TestResult extends TestCase {
List<KeyValue> ks = r.getColumn(family, qf);
assertEquals(1, ks.size());
assertByteEquals(qf, ks.get(0).getQualifier());
assertEquals(ks.get(0), r.getColumnLatest(family, qf));
assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf));
assertTrue(r.containsColumn(family, qf));
}
}
public void testMultiVersion() throws Exception {
public void testMultiVersionGetColumn() throws Exception {
KeyValue [] kvs1 = genKVs(row, family, value, 1, 100);
KeyValue [] kvs2 = genKVs(row, family, value, 200, 100);
@ -92,13 +95,89 @@ public class TestResult extends TestCase {
assertEquals(2, ks.size());
assertByteEquals(qf, ks.get(0).getQualifier());
assertEquals(200, ks.get(0).getTimestamp());
assertEquals(ks.get(0), r.getColumnLatest(family, qf));
}
}
public void testBasicGetValue() throws Exception {
KeyValue [] kvs = genKVs(row, family, value, 1, 100);
Arrays.sort(kvs, KeyValue.COMPARATOR);
Result r = new Result(kvs);
for (int i = 0; i < 100; ++i) {
final byte[] qf = Bytes.toBytes(i);
assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf));
assertTrue(r.containsColumn(family, qf));
}
}
public void testMultiVersionGetValue() throws Exception {
KeyValue [] kvs1 = genKVs(row, family, value, 1, 100);
KeyValue [] kvs2 = genKVs(row, family, value, 200, 100);
KeyValue [] kvs = new KeyValue[kvs1.length+kvs2.length];
System.arraycopy(kvs1, 0, kvs, 0, kvs1.length);
System.arraycopy(kvs2, 0, kvs, kvs1.length, kvs2.length);
Arrays.sort(kvs, KeyValue.COMPARATOR);
Result r = new Result(kvs);
for (int i = 0; i < 100; ++i) {
final byte[] qf = Bytes.toBytes(i);
assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf));
assertTrue(r.containsColumn(family, qf));
}
}
public void testBasicLoadValue() throws Exception {
KeyValue [] kvs = genKVs(row, family, value, 1, 100);
Arrays.sort(kvs, KeyValue.COMPARATOR);
Result r = new Result(kvs);
ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024);
for (int i = 0; i < 100; ++i) {
final byte[] qf = Bytes.toBytes(i);
loadValueBuffer.clear();
r.loadValue(family, qf, loadValueBuffer);
loadValueBuffer.flip();
assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))), loadValueBuffer);
assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))),
r.getValueAsByteBuffer(family, qf));
}
}
public void testMultiVersionLoadValue() throws Exception {
KeyValue [] kvs1 = genKVs(row, family, value, 1, 100);
KeyValue [] kvs2 = genKVs(row, family, value, 200, 100);
KeyValue [] kvs = new KeyValue[kvs1.length+kvs2.length];
System.arraycopy(kvs1, 0, kvs, 0, kvs1.length);
System.arraycopy(kvs2, 0, kvs, kvs1.length, kvs2.length);
Arrays.sort(kvs, KeyValue.COMPARATOR);
ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024);
Result r = new Result(kvs);
for (int i = 0; i < 100; ++i) {
final byte[] qf = Bytes.toBytes(i);
loadValueBuffer.clear();
r.loadValue(family, qf, loadValueBuffer);
loadValueBuffer.flip();
assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))), loadValueBuffer);
assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))),
r.getValueAsByteBuffer(family, qf));
}
}
/**
* Verify that Result.compareResults(...) behaves correctly.
*/
@ -125,5 +204,82 @@ public class TestResult extends TestCase {
@org.junit.Rule
public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
}
/**
* Microbenchmark that compares {@link Result#getValue} and {@link Result#loadValue} performance.
*
* @throws Exception
*/
public void doReadBenchmark() throws Exception {
final int n = 5;
final int m = 100000000;
StringBuilder valueSB = new StringBuilder();
for (int i = 0; i < 100; i++) {
valueSB.append((byte)(Math.random() * 10));
}
StringBuilder rowSB = new StringBuilder();
for (int i = 0; i < 50; i++) {
rowSB.append((byte)(Math.random() * 10));
}
KeyValue [] kvs = genKVs(Bytes.toBytes(rowSB.toString()), family,
Bytes.toBytes(valueSB.toString()), 1, n);
Arrays.sort(kvs, KeyValue.COMPARATOR);
ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024);
Result r = new Result(kvs);
byte[][] qfs = new byte[n][Bytes.SIZEOF_INT];
for (int i = 0; i < n; ++i) {
System.arraycopy(qfs[i], 0, Bytes.toBytes(i), 0, Bytes.SIZEOF_INT);
}
// warm up
for (int k = 0; k < 100000; k++) {
for (int i = 0; i < n; ++i) {
r.getValue(family, qfs[i]);
loadValueBuffer.clear();
r.loadValue(family, qfs[i], loadValueBuffer);
loadValueBuffer.flip();
}
}
System.gc();
long start = System.nanoTime();
for (int k = 0; k < m; k++) {
for (int i = 0; i < n; ++i) {
loadValueBuffer.clear();
r.loadValue(family, qfs[i], loadValueBuffer);
loadValueBuffer.flip();
}
}
long stop = System.nanoTime();
System.out.println("loadValue(): " + (stop - start));
System.gc();
start = System.nanoTime();
for (int k = 0; k < m; k++) {
for (int i = 0; i < n; i++) {
r.getValue(family, qfs[i]);
}
}
stop = System.nanoTime();
System.out.println("getValue(): " + (stop - start));
}
/**
* Calls non-functional test methods.
*
* @param args
*/
public static void main(String[] args) {
TestResult testResult = new TestResult();
try {
testResult.doReadBenchmark();
} catch (Exception e) {
LOG.error("Unexpected exception", e);
}
}
}