diff --git a/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/src/main/java/org/apache/hadoop/hbase/KeyValue.java index 9ae9e02d608..254965d5eb2 100644 --- a/src/main/java/org/apache/hadoop/hbase/KeyValue.java +++ b/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -22,6 +22,7 @@ package org.apache.hadoop.hbase; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.util.Comparator; import java.util.HashMap; @@ -166,6 +167,37 @@ public class KeyValue implements Writable, HeapSize { // Size of the length ints in a KeyValue datastructure. public static final int KEYVALUE_INFRASTRUCTURE_SIZE = ROW_OFFSET; + /** + * Computes the number of bytes that a KeyValue instance with the provided + * characteristics would take up for its underlying data structure. + * + * @param rlength row length + * @param flength family length + * @param qlength qualifier length + * @param vlength value length + * + * @return the KeyValue data structure length + */ + public static long getKeyValueDataStructureSize(int rlength, + int flength, int qlength, int vlength) { + return KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + + getKeyDataStructureSize(rlength, flength, qlength) + vlength; + } + + /** + * Computes the number of bytes that a KeyValue instance with the provided + * characteristics would take up in its underlying data structure for the key. + * + * @param rlength row length + * @param flength family length + * @param qlength qualifier length + * + * @return the key data structure length + */ + public static long getKeyDataStructureSize(int rlength, int flength, int qlength) { + return KeyValue.KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength; + } + /** * Key type. * Has space for other key types to be added later. Cannot rely on @@ -478,7 +510,7 @@ public class KeyValue implements Writable, HeapSize { throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE); } // Key length - long longkeylength = KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength; + long longkeylength = getKeyDataStructureSize(rlength, flength, qlength); if (longkeylength > Integer.MAX_VALUE) { throw new IllegalArgumentException("keylength " + longkeylength + " > " + Integer.MAX_VALUE); @@ -491,7 +523,8 @@ public class KeyValue implements Writable, HeapSize { } // Allocate right-sized byte array. - byte [] bytes = new byte[KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength]; + byte [] bytes = + new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength)]; // Write the correct size markers int pos = 0; pos = Bytes.putInt(bytes, pos, keylength); @@ -505,6 +538,198 @@ public class KeyValue implements Writable, HeapSize { return bytes; } + /** + * Constructs KeyValue structure filled with specified values. Uses the provided buffer as its + * backing data buffer. + *

+ * Column is split into two fields, family and qualifier. + * + * @param buffer the bytes buffer to use + * @param row row key + * @param roffset row offset + * @param rlength row length + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * @param timestamp version timestamp + * @param type key type + * @param value column value + * @param voffset value offset + * @param vlength value length + * @throws IllegalArgumentException an illegal value was passed or there is insufficient space + * remaining in the buffer + */ + public KeyValue(byte [] buffer, + final byte [] row, final int roffset, final int rlength, + final byte [] family, final int foffset, final int flength, + final byte [] qualifier, final int qoffset, final int qlength, + final long timestamp, final Type type, + final byte [] value, final int voffset, final int vlength) { + + this(buffer, 0, + row, roffset, rlength, + family, foffset, flength, + qualifier, qoffset, qlength, + timestamp, type, + value, voffset, vlength); + } + + /** + * Constructs KeyValue structure filled with specified values. Uses the provided buffer as the + * data buffer. + *

+ * Column is split into two fields, family and qualifier. + * + * @param buffer the bytes buffer to use + * @param boffset buffer offset + * @param row row key + * @param roffset row offset + * @param rlength row length + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * @param timestamp version timestamp + * @param type key type + * @param value column value + * @param voffset value offset + * @param vlength value length + * @throws IllegalArgumentException an illegal value was passed or there is insufficient space + * remaining in the buffer + */ + public KeyValue(byte [] buffer, final int boffset, + final byte [] row, final int roffset, final int rlength, + final byte [] family, final int foffset, final int flength, + final byte [] qualifier, final int qoffset, final int qlength, + final long timestamp, final Type type, + final byte [] value, final int voffset, final int vlength) { + + this.bytes = buffer; + this.length = writeByteArray(buffer, boffset, + row, roffset, rlength, + family, foffset, flength, qualifier, qoffset, qlength, + timestamp, type, value, voffset, vlength); + this.offset = boffset; + } + + /** + * Checks the parameters passed to a constructor. + * + * @param row row key + * @param rlength row length + * @param family family name + * @param flength family length + * @param qualifier column qualifier + * @param qlength qualifier length + * @param value column value + * @param vlength value length + * + * @throws IllegalArgumentException an illegal value was passed + */ + private static void checkParameters(final byte [] row, final int rlength, + final byte [] family, int flength, + final byte [] qualifier, int qlength, + final byte [] value, int vlength) + throws IllegalArgumentException { + + if (rlength > Short.MAX_VALUE) { + throw new IllegalArgumentException("Row > " + Short.MAX_VALUE); + } + if (row == null) { + throw new IllegalArgumentException("Row is null"); + } + // Family length + flength = family == null ? 0 : flength; + if (flength > Byte.MAX_VALUE) { + throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE); + } + // Qualifier length + qlength = qualifier == null ? 0 : qlength; + if (qlength > Integer.MAX_VALUE - rlength - flength) { + throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE); + } + // Key length + long longKeyLength = getKeyDataStructureSize(rlength, flength, qlength); + if (longKeyLength > Integer.MAX_VALUE) { + throw new IllegalArgumentException("keylength " + longKeyLength + " > " + + Integer.MAX_VALUE); + } + // Value length + vlength = value == null? 0 : vlength; + if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON + throw new IllegalArgumentException("Value length " + vlength + " > " + + HConstants.MAXIMUM_VALUE_LENGTH); + } + } + + /** + * Write KeyValue format into the provided byte array. + * + * @param buffer the bytes buffer to use + * @param boffset buffer offset + * @param row row key + * @param roffset row offset + * @param rlength row length + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * @param timestamp version timestamp + * @param type key type + * @param value column value + * @param voffset value offset + * @param vlength value length + * + * @return The number of useful bytes in the buffer. + * + * @throws IllegalArgumentException an illegal value was passed or there is insufficient space + * remaining in the buffer + */ + static int writeByteArray(byte [] buffer, final int boffset, + final byte [] row, final int roffset, final int rlength, + final byte [] family, final int foffset, int flength, + final byte [] qualifier, final int qoffset, int qlength, + final long timestamp, final Type type, + final byte [] value, final int voffset, int vlength) { + + checkParameters(row, rlength, family, flength, qualifier, qlength, value, vlength); + + int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength); + int keyValueLength = (int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength); + if (keyValueLength > buffer.length - boffset) { + throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < " + + keyValueLength); + } + + // Write key, value and key row length. + int pos = boffset; + pos = Bytes.putInt(buffer, pos, keyLength); + pos = Bytes.putInt(buffer, pos, vlength); + pos = Bytes.putShort(buffer, pos, (short)(rlength & 0x0000ffff)); + pos = Bytes.putBytes(buffer, pos, row, roffset, rlength); + pos = Bytes.putByte(buffer, pos, (byte) (flength & 0x0000ff)); + if (flength != 0) { + pos = Bytes.putBytes(buffer, pos, family, foffset, flength); + } + if (qlength != 0) { + pos = Bytes.putBytes(buffer, pos, qualifier, qoffset, qlength); + } + pos = Bytes.putLong(buffer, pos, timestamp); + pos = Bytes.putByte(buffer, pos, type.getCode()); + if (value != null && value.length > 0) { + pos = Bytes.putBytes(buffer, pos, value, voffset, vlength); + } + + return keyValueLength; + } + /** * Write KeyValue format into a byte array. * @@ -529,41 +754,16 @@ public class KeyValue implements Writable, HeapSize { final byte [] qualifier, final int qoffset, int qlength, final long timestamp, final Type type, final byte [] value, final int voffset, int vlength) { - if (rlength > Short.MAX_VALUE) { - throw new IllegalArgumentException("Row > " + Short.MAX_VALUE); - } - if (row == null) { - throw new IllegalArgumentException("Row is null"); - } - // Family length - flength = family == null ? 0 : flength; - if (flength > Byte.MAX_VALUE) { - throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE); - } - // Qualifier length - qlength = qualifier == null ? 0 : qlength; - if (qlength > Integer.MAX_VALUE - rlength - flength) { - throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE); - } - // Key length - long longkeylength = KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength; - if (longkeylength > Integer.MAX_VALUE) { - throw new IllegalArgumentException("keylength " + longkeylength + " > " + - Integer.MAX_VALUE); - } - int keylength = (int)longkeylength; - // Value length - vlength = value == null? 0 : vlength; - if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON - throw new IllegalArgumentException("Valuer > " + - HConstants.MAXIMUM_VALUE_LENGTH); - } + + checkParameters(row, rlength, family, flength, qualifier, qlength, value, vlength); // Allocate right-sized byte array. - byte [] bytes = new byte[KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength]; + int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength); + byte [] bytes = + new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength)]; // Write key, value and key row length. int pos = 0; - pos = Bytes.putInt(bytes, pos, keylength); + pos = Bytes.putInt(bytes, pos, keyLength); pos = Bytes.putInt(bytes, pos, vlength); pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff)); pos = Bytes.putBytes(bytes, pos, row, roffset, rlength); @@ -913,8 +1113,7 @@ public class KeyValue implements Writable, HeapSize { * @return Qualifier length */ public int getQualifierLength(int rlength, int flength) { - return getKeyLength() - - (KEY_INFRASTRUCTURE_SIZE + rlength + flength); + return getKeyLength() - (int) getKeyDataStructureSize(rlength, flength, 0); } /** @@ -1007,6 +1206,28 @@ public class KeyValue implements Writable, HeapSize { return result; } + /** + * Returns the value wrapped in a new ByteBuffer. + * + * @return the value + */ + public ByteBuffer getValueAsByteBuffer() { + return ByteBuffer.wrap(getBuffer(), getValueOffset(), getValueLength()); + } + + /** + * Loads this object's value into the provided ByteBuffer. + *

+ * Does not clear or flip the buffer. + * + * @param dst the buffer where to write the value + * + * @throws BufferOverflowException if there is insufficient space remaining in the buffer + */ + public void loadValue(ByteBuffer dst) throws BufferOverflowException { + dst.put(getBuffer(), getValueOffset(), getValueLength()); + } + /** * Primarily for use client-side. Returns the row of this KeyValue in a new * byte array.

@@ -1278,21 +1499,36 @@ public class KeyValue implements Writable, HeapSize { * @return True if column matches */ public boolean matchingColumn(final byte[] family, final byte[] qualifier) { + return matchingColumn(family, 0, family == null ? 0 : family.length, + qualifier, 0, qualifier == null ? 0 : qualifier.length); + } + + /** + * Checks if column matches. + * + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * + * @return True if column matches + */ + public boolean matchingColumn(final byte [] family, final int foffset, final int flength, + final byte [] qualifier, final int qoffset, final int qlength) { int rl = getRowLength(); int o = getFamilyOffset(rl); int fl = getFamilyLength(o); - int ql = getQualifierLength(rl,fl); - if (!Bytes.equals(family, 0, family.length, this.bytes, o, fl)) { + if (!Bytes.equals(family, foffset, flength, this.bytes, o, fl)) { return false; } - if (qualifier == null || qualifier.length == 0) { - if (ql == 0) { - return true; - } - return false; + + int ql = getQualifierLength(rl, fl); + if (qualifier == null || qlength == 0) { + return (ql == 0); } - return Bytes.equals(qualifier, 0, qualifier.length, - this.bytes, o + fl, ql); + return Bytes.equals(qualifier, qoffset, qlength, this.bytes, o + fl, ql); } /** @@ -1820,6 +2056,78 @@ public class KeyValue implements Writable, HeapSize { HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0); } + /** + * Create a KeyValue for the specified row, family and qualifier that would be + * smaller than all other possible KeyValues that have the same row, + * family, qualifier. + * Used for seeking. + * + * @param buffer the buffer to use for the new KeyValue object + * @param row the value key + * @param family family name + * @param qualifier column qualifier + * + * @return First possible key on passed Row, Family, Qualifier. + * + * @throws IllegalArgumentException The resulting KeyValue object would be larger + * than the provided buffer or than Integer.MAX_VALUE + */ + public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row, + final byte [] family, final byte [] qualifier) + throws IllegalArgumentException { + + return createFirstOnRow(buffer, 0, row, 0, row.length, + family, 0, family.length, + qualifier, 0, qualifier.length); + } + + /** + * Create a KeyValue for the specified row, family and qualifier that would be + * smaller than all other possible KeyValues that have the same row, + * family, qualifier. + * Used for seeking. + * + * @param buffer the buffer to use for the new KeyValue object + * @param boffset buffer offset + * @param row the value key + * @param roffset row offset + * @param rlength row length + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * + * @return First possible key on passed Row, Family, Qualifier. + * + * @throws IllegalArgumentException The resulting KeyValue object would be larger + * than the provided buffer or than Integer.MAX_VALUE + */ + public static KeyValue createFirstOnRow(byte [] buffer, final int boffset, + final byte [] row, final int roffset, final int rlength, + final byte [] family, final int foffset, final int flength, + final byte [] qualifier, final int qoffset, final int qlength) + throws IllegalArgumentException { + + long lLength = getKeyValueDataStructureSize(rlength, flength, qlength, 0); + + if (lLength > Integer.MAX_VALUE) { + throw new IllegalArgumentException("KeyValue length " + lLength + " > " + Integer.MAX_VALUE); + } + int iLength = (int) lLength; + if (buffer.length - boffset < iLength) { + throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < " + + iLength); + } + return new KeyValue(buffer, boffset, + row, roffset, rlength, + family, foffset, flength, + qualifier, qoffset, qlength, + HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum, + null, 0, 0); + } + /** * Create a KeyValue for the specified row, family and qualifier that would be * larger than or equal to all other possible KeyValues that have the same diff --git a/src/main/java/org/apache/hadoop/hbase/client/Result.java b/src/main/java/org/apache/hadoop/hbase/client/Result.java index df0b3ef7967..25bcddd43b8 100644 --- a/src/main/java/org/apache/hadoop/hbase/client/Result.java +++ b/src/main/java/org/apache/hadoop/hbase/client/Result.java @@ -23,6 +23,8 @@ package org.apache.hadoop.hbase.client; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -71,6 +73,7 @@ import org.apache.hadoop.io.Writable; @InterfaceStability.Stable public class Result implements Writable, WritableWithSize { private static final byte RESULT_VERSION = (byte)1; + private static final int DEFAULT_BUFFER_SIZE = 1024; private KeyValue [] kvs = null; private NavigableMap buffer.length) { + // pad to the smallest multiple of the pad width + buffer = new byte[(int) Math.ceil(keyValueSize / PAD_WIDTH) * PAD_WIDTH]; + } + + KeyValue searchTerm = KeyValue.createFirstOnRow(buffer, 0, + kvs[0].getBuffer(), kvs[0].getRowOffset(), kvs[0].getRowLength(), + family, foffset, flength, + qualifier, qoffset, qlength); + + // pos === ( -(insertion point) - 1) + int pos = Arrays.binarySearch(kvs, searchTerm, KeyValue.COMPARATOR); + // never will exact match + if (pos < 0) { + pos = (pos+1) * -1; + // pos is now insertion point + } + if (pos == kvs.length) { + return -1; // doesn't exist + } + return pos; + } + + /** + * The KeyValue for the most recent timestamp for a given column. * * @param family * @param qualifier - * @return KeyValue for the column or null + * + * @return the KeyValue for the column, or null if no value exists in the row or none have been + * selected in the query (Get/Scan) */ public KeyValue getColumnLatest(byte [] family, byte [] qualifier) { KeyValue [] kvs = raw(); // side effect possibly. @@ -252,6 +302,37 @@ public class Result implements Writable, WritableWithSize { return null; } + /** + * The KeyValue for the most recent timestamp for a given column. + * + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * + * @return the KeyValue for the column, or null if no value exists in the row or none have been + * selected in the query (Get/Scan) + */ + public KeyValue getColumnLatest(byte [] family, int foffset, int flength, + byte [] qualifier, int qoffset, int qlength) { + + KeyValue [] kvs = raw(); // side effect possibly. + if (kvs == null || kvs.length == 0) { + return null; + } + int pos = binarySearch(kvs, family, foffset, flength, qualifier, qoffset, qlength); + if (pos == -1) { + return null; + } + KeyValue kv = kvs[pos]; + if (kv.matchingColumn(family, foffset, flength, qualifier, qoffset, qlength)) { + return kv; + } + return null; + } + /** * Get the latest version of the specified column. * @param family family name @@ -267,9 +348,164 @@ public class Result implements Writable, WritableWithSize { } /** - * Checks for existence of the specified column. + * Returns the value wrapped in a new ByteBuffer. + * * @param family family name * @param qualifier column qualifier + * + * @return the latest version of the column, or null if none found + */ + public ByteBuffer getValueAsByteBuffer(byte [] family, byte [] qualifier) { + + KeyValue kv = getColumnLatest(family, 0, family.length, qualifier, 0, qualifier.length); + + if (kv == null) { + return null; + } + return kv.getValueAsByteBuffer(); + } + + /** + * Returns the value wrapped in a new ByteBuffer. + * + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * + * @return the latest version of the column, or null if none found + */ + public ByteBuffer getValueAsByteBuffer(byte [] family, int foffset, int flength, + byte [] qualifier, int qoffset, int qlength) { + + KeyValue kv = getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength); + + if (kv == null) { + return null; + } + return kv.getValueAsByteBuffer(); + } + + /** + * Loads the latest version of the specified column into the provided ByteBuffer. + *

+ * Does not clear or flip the buffer. + * + * @param family family name + * @param qualifier column qualifier + * @param dst the buffer where to write the value + * + * @return true if a value was found, false otherwise + * + * @throws BufferOverflowException there is insufficient space remaining in the buffer + */ + public boolean loadValue(byte [] family, byte [] qualifier, ByteBuffer dst) + throws BufferOverflowException { + return loadValue(family, 0, family.length, qualifier, 0, qualifier.length, dst); + } + + /** + * Loads the latest version of the specified column into the provided ByteBuffer. + *

+ * Does not clear or flip the buffer. + * + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * @param dst the buffer where to write the value + * + * @return true if a value was found, false otherwise + * + * @throws BufferOverflowException there is insufficient space remaining in the buffer + */ + public boolean loadValue(byte [] family, int foffset, int flength, + byte [] qualifier, int qoffset, int qlength, ByteBuffer dst) + throws BufferOverflowException { + KeyValue kv = getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength); + + if (kv == null) { + return false; + } + kv.loadValue(dst); + return true; + } + + /** + * Checks if the specified column contains a non-empty value (not a zero-length byte array). + * + * @param family family name + * @param qualifier column qualifier + * + * @return whether or not a latest value exists and is not empty + */ + public boolean containsNonEmptyColumn(byte [] family, byte [] qualifier) { + + return containsNonEmptyColumn(family, 0, family.length, qualifier, 0, qualifier.length); + } + + /** + * Checks if the specified column contains a non-empty value (not a zero-length byte array). + * + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * + * @return whether or not a latest value exists and is not empty + */ + public boolean containsNonEmptyColumn(byte [] family, int foffset, int flength, + byte [] qualifier, int qoffset, int qlength) { + + KeyValue kv = getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength); + + return (kv != null) && (kv.getValueLength() > 0); + } + + /** + * Checks if the specified column contains an empty value (a zero-length byte array). + * + * @param family family name + * @param qualifier column qualifier + * + * @return whether or not a latest value exists and is empty + */ + public boolean containsEmptyColumn(byte [] family, byte [] qualifier) { + + return containsEmptyColumn(family, 0, family.length, qualifier, 0, qualifier.length); + } + + /** + * Checks if the specified column contains an empty value (a zero-length byte array). + * + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * + * @return whether or not a latest value exists and is empty + */ + public boolean containsEmptyColumn(byte [] family, int foffset, int flength, + byte [] qualifier, int qoffset, int qlength) { + KeyValue kv = getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength); + + return (kv != null) && (kv.getValueLength() == 0); + } + + /** + * Checks for existence of a value for the specified column (empty or not). + * + * @param family family name + * @param qualifier column qualifier + * * @return true if at least one value exists in the result, false if not */ public boolean containsColumn(byte [] family, byte [] qualifier) { @@ -277,6 +513,24 @@ public class Result implements Writable, WritableWithSize { return kv != null; } + /** + * Checks for existence of a value for the specified column (empty or not). + * + * @param family family name + * @param foffset family offset + * @param flength family length + * @param qualifier column qualifier + * @param qoffset qualifier offset + * @param qlength qualifier length + * + * @return true if at least one value exists in the result, false if not + */ + public boolean containsColumn(byte [] family, int foffset, int flength, + byte [] qualifier, int qoffset, int qlength) { + + return getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength) != null; + } + /** * Map of families to all versions of its qualifiers and values. *

diff --git a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java index 786d2df5223..55fd65283fb 100644 --- a/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java +++ b/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java @@ -344,31 +344,47 @@ public class TestKeyValue extends TestCase { public void testFirstLastOnRow() { final KVComparator c = KeyValue.COMPARATOR; long ts = 1; + byte[] bufferA = new byte[128]; + int offsetA = 0; + byte[] bufferB = new byte[128]; + int offsetB = 7; // These are listed in sort order (ie: every one should be less // than the one on the next line). final KeyValue firstOnRowA = KeyValue.createFirstOnRow(rowA); + final KeyValue firstOnRowABufferFamQual = KeyValue.createFirstOnRow(bufferA, offsetA, + rowA, 0, rowA.length, family, 0, family.length, qualA, 0, qualA.length); final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, Type.Put); final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, Type.Put); - + final KeyValue lastOnRowA = KeyValue.createLastOnRow(rowA); final KeyValue firstOnRowB = KeyValue.createFirstOnRow(rowB); + final KeyValue firstOnRowBBufferFam = KeyValue.createFirstOnRow(bufferB, offsetB, + rowB, 0, rowB.length, family, 0, family.length, null, 0, 0); final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, Type.Put); assertKVLess(c, firstOnRowA, firstOnRowB); + assertKVLess(c, firstOnRowA, firstOnRowBBufferFam); + assertKVLess(c, firstOnRowABufferFamQual, firstOnRowB); assertKVLess(c, firstOnRowA, kvA_1); assertKVLess(c, firstOnRowA, kvA_2); + assertKVLess(c, firstOnRowABufferFamQual, kvA_2); assertKVLess(c, kvA_1, kvA_2); assertKVLess(c, kvA_2, firstOnRowB); assertKVLess(c, kvA_1, firstOnRowB); + assertKVLess(c, kvA_2, firstOnRowBBufferFam); + assertKVLess(c, kvA_1, firstOnRowBBufferFam); assertKVLess(c, lastOnRowA, firstOnRowB); + assertKVLess(c, lastOnRowA, firstOnRowBBufferFam); assertKVLess(c, firstOnRowB, kvB); + assertKVLess(c, firstOnRowBBufferFam, kvB); assertKVLess(c, lastOnRowA, kvB); assertKVLess(c, kvA_2, lastOnRowA); assertKVLess(c, kvA_1, lastOnRowA); assertKVLess(c, firstOnRowA, lastOnRowA); + assertKVLess(c, firstOnRowABufferFamQual, lastOnRowA); } public void testCreateKeyOnly() throws Exception { diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestResult.java b/src/test/java/org/apache/hadoop/hbase/client/TestResult.java index f9e29c267e3..d30e776282a 100644 --- a/src/test/java/org/apache/hadoop/hbase/client/TestResult.java +++ b/src/test/java/org/apache/hadoop/hbase/client/TestResult.java @@ -21,6 +21,8 @@ package org.apache.hadoop.hbase.client; import junit.framework.TestCase; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -28,6 +30,7 @@ import org.junit.experimental.categories.Category; import static org.apache.hadoop.hbase.HBaseTestCase.assertByteEquals; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -36,6 +39,8 @@ import java.util.NavigableMap; @Category(SmallTests.class) public class TestResult extends TestCase { + private static final Log LOG = LogFactory.getLog(TestResult.class.getName()); + static KeyValue[] genKVs(final byte[] row, final byte[] family, final byte[] value, final long timestamp, @@ -55,7 +60,7 @@ public class TestResult extends TestCase { static final byte [] family = Bytes.toBytes("family"); static final byte [] value = Bytes.toBytes("value"); - public void testBasic() throws Exception { + public void testBasicGetColumn() throws Exception { KeyValue [] kvs = genKVs(row, family, value, 1, 100); Arrays.sort(kvs, KeyValue.COMPARATOR); @@ -68,13 +73,11 @@ public class TestResult extends TestCase { List ks = r.getColumn(family, qf); assertEquals(1, ks.size()); assertByteEquals(qf, ks.get(0).getQualifier()); - assertEquals(ks.get(0), r.getColumnLatest(family, qf)); - assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf)); - assertTrue(r.containsColumn(family, qf)); } } - public void testMultiVersion() throws Exception { + + public void testMultiVersionGetColumn() throws Exception { KeyValue [] kvs1 = genKVs(row, family, value, 1, 100); KeyValue [] kvs2 = genKVs(row, family, value, 200, 100); @@ -92,13 +95,89 @@ public class TestResult extends TestCase { assertEquals(2, ks.size()); assertByteEquals(qf, ks.get(0).getQualifier()); assertEquals(200, ks.get(0).getTimestamp()); - assertEquals(ks.get(0), r.getColumnLatest(family, qf)); + } + } + + public void testBasicGetValue() throws Exception { + KeyValue [] kvs = genKVs(row, family, value, 1, 100); + + Arrays.sort(kvs, KeyValue.COMPARATOR); + + Result r = new Result(kvs); + + for (int i = 0; i < 100; ++i) { + final byte[] qf = Bytes.toBytes(i); + assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf)); assertTrue(r.containsColumn(family, qf)); } } + public void testMultiVersionGetValue() throws Exception { + KeyValue [] kvs1 = genKVs(row, family, value, 1, 100); + KeyValue [] kvs2 = genKVs(row, family, value, 200, 100); + + KeyValue [] kvs = new KeyValue[kvs1.length+kvs2.length]; + System.arraycopy(kvs1, 0, kvs, 0, kvs1.length); + System.arraycopy(kvs2, 0, kvs, kvs1.length, kvs2.length); + + Arrays.sort(kvs, KeyValue.COMPARATOR); + + Result r = new Result(kvs); + for (int i = 0; i < 100; ++i) { + final byte[] qf = Bytes.toBytes(i); + + assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf)); + assertTrue(r.containsColumn(family, qf)); + } + } + + public void testBasicLoadValue() throws Exception { + KeyValue [] kvs = genKVs(row, family, value, 1, 100); + + Arrays.sort(kvs, KeyValue.COMPARATOR); + + Result r = new Result(kvs); + ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024); + + for (int i = 0; i < 100; ++i) { + final byte[] qf = Bytes.toBytes(i); + + loadValueBuffer.clear(); + r.loadValue(family, qf, loadValueBuffer); + loadValueBuffer.flip(); + assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))), loadValueBuffer); + assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))), + r.getValueAsByteBuffer(family, qf)); + } + } + + public void testMultiVersionLoadValue() throws Exception { + KeyValue [] kvs1 = genKVs(row, family, value, 1, 100); + KeyValue [] kvs2 = genKVs(row, family, value, 200, 100); + + KeyValue [] kvs = new KeyValue[kvs1.length+kvs2.length]; + System.arraycopy(kvs1, 0, kvs, 0, kvs1.length); + System.arraycopy(kvs2, 0, kvs, kvs1.length, kvs2.length); + + Arrays.sort(kvs, KeyValue.COMPARATOR); + + ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024); + + Result r = new Result(kvs); + for (int i = 0; i < 100; ++i) { + final byte[] qf = Bytes.toBytes(i); + + loadValueBuffer.clear(); + r.loadValue(family, qf, loadValueBuffer); + loadValueBuffer.flip(); + assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))), loadValueBuffer); + assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))), + r.getValueAsByteBuffer(family, qf)); + } + } + /** * Verify that Result.compareResults(...) behaves correctly. */ @@ -125,5 +204,82 @@ public class TestResult extends TestCase { @org.junit.Rule public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = new org.apache.hadoop.hbase.ResourceCheckerJUnitRule(); -} + /** + * Microbenchmark that compares {@link Result#getValue} and {@link Result#loadValue} performance. + * + * @throws Exception + */ + public void doReadBenchmark() throws Exception { + + final int n = 5; + final int m = 100000000; + + StringBuilder valueSB = new StringBuilder(); + for (int i = 0; i < 100; i++) { + valueSB.append((byte)(Math.random() * 10)); + } + + StringBuilder rowSB = new StringBuilder(); + for (int i = 0; i < 50; i++) { + rowSB.append((byte)(Math.random() * 10)); + } + + KeyValue [] kvs = genKVs(Bytes.toBytes(rowSB.toString()), family, + Bytes.toBytes(valueSB.toString()), 1, n); + Arrays.sort(kvs, KeyValue.COMPARATOR); + ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024); + Result r = new Result(kvs); + + byte[][] qfs = new byte[n][Bytes.SIZEOF_INT]; + for (int i = 0; i < n; ++i) { + System.arraycopy(qfs[i], 0, Bytes.toBytes(i), 0, Bytes.SIZEOF_INT); + } + + // warm up + for (int k = 0; k < 100000; k++) { + for (int i = 0; i < n; ++i) { + r.getValue(family, qfs[i]); + loadValueBuffer.clear(); + r.loadValue(family, qfs[i], loadValueBuffer); + loadValueBuffer.flip(); + } + } + + System.gc(); + long start = System.nanoTime(); + for (int k = 0; k < m; k++) { + for (int i = 0; i < n; ++i) { + loadValueBuffer.clear(); + r.loadValue(family, qfs[i], loadValueBuffer); + loadValueBuffer.flip(); + } + } + long stop = System.nanoTime(); + System.out.println("loadValue(): " + (stop - start)); + + System.gc(); + start = System.nanoTime(); + for (int k = 0; k < m; k++) { + for (int i = 0; i < n; i++) { + r.getValue(family, qfs[i]); + } + } + stop = System.nanoTime(); + System.out.println("getValue(): " + (stop - start)); + } + + /** + * Calls non-functional test methods. + * + * @param args + */ + public static void main(String[] args) { + TestResult testResult = new TestResult(); + try { + testResult.doReadBenchmark(); + } catch (Exception e) { + LOG.error("Unexpected exception", e); + } + } +}