From 739f4381769e5d1220323e56fd772d7b310f07e5 Mon Sep 17 00:00:00 2001 From: Jonathan Hsieh Date: Thu, 29 Aug 2013 20:45:04 +0000 Subject: [PATCH] HBASE-9247 Cleanup Key/KV/Meta/MetaKey Comparators git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1518817 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/hbase/client/Result.java | 29 +- .../org/apache/hadoop/hbase/KeyValue.java | 1157 +++++++---------- .../org/apache/hadoop/hbase/TableName.java | 8 +- .../io/encoding/BufferedDataBlockEncoder.java | 8 +- .../io/encoding/CopyKeyDataBlockEncoder.java | 4 +- .../hbase/io/encoding/DataBlockEncoder.java | 3 +- .../io/encoding/DiffKeyDeltaEncoder.java | 3 +- .../io/encoding/FastDiffDeltaEncoder.java | 3 +- .../io/encoding/PrefixKeyDeltaEncoder.java | 3 +- .../org/apache/hadoop/hbase/util/Bytes.java | 2 +- .../org/apache/hadoop/hbase/TestKeyValue.java | 54 +- .../codec/prefixtree/PrefixTreeCodec.java | 12 +- .../hadoop/hbase/io/HalfStoreFileReader.java | 16 +- .../hbase/io/hfile/AbstractHFileReader.java | 5 +- .../hbase/io/hfile/AbstractHFileWriter.java | 11 +- .../hbase/io/hfile/FixedFileTrailer.java | 53 +- .../apache/hadoop/hbase/io/hfile/HFile.java | 10 +- .../hbase/io/hfile/HFileBlockIndex.java | 14 +- .../hadoop/hbase/io/hfile/HFileReaderV2.java | 10 +- .../hadoop/hbase/io/hfile/HFileWriterV2.java | 6 +- .../GetClosestRowBeforeTracker.java | 2 +- .../hbase/regionserver/HRegionFileSystem.java | 4 +- .../hbase/regionserver/ScanQueryMatcher.java | 4 +- .../hadoop/hbase/regionserver/StoreFile.java | 16 +- .../hadoop/hbase/util/BloomFilterBase.java | 4 +- .../hadoop/hbase/util/BloomFilterFactory.java | 4 +- .../hadoop/hbase/util/ByteBloomFilter.java | 7 +- .../hbase/util/CompoundBloomFilter.java | 4 +- .../hbase/util/CompoundBloomFilterBase.java | 5 +- .../hbase/util/CompoundBloomFilterWriter.java | 4 +- .../io/encoding/TestDataBlockEncoders.java | 4 +- .../io/encoding/TestPrefixTreeEncoding.java | 8 +- .../hbase/io/hfile/TestFixedFileTrailer.java | 4 +- .../hadoop/hbase/io/hfile/TestHFile.java | 34 +- .../hbase/io/hfile/TestHFileBlockIndex.java | 16 +- .../hbase/io/hfile/TestHFilePerformance.java | 2 +- .../hadoop/hbase/io/hfile/TestHFileSeek.java | 2 +- .../hbase/io/hfile/TestHFileWriterV2.java | 6 +- .../hadoop/hbase/io/hfile/TestReseekTo.java | 2 +- .../hadoop/hbase/io/hfile/TestSeekTo.java | 3 +- .../hbase/regionserver/TestMemStore.java | 5 +- .../hbase/regionserver/TestStoreFile.java | 4 +- 42 files changed, 633 insertions(+), 922 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java index 22170c4cd54..7a933da6678 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java @@ -19,16 +19,6 @@ package org.apache.hadoop.hbase.client; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellScannable; -import org.apache.hadoop.hbase.CellScanner; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.SplitKeyValue; -import org.apache.hadoop.hbase.util.Bytes; - import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -39,6 +29,15 @@ import java.util.Map; import java.util.NavigableMap; import java.util.TreeMap; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellScannable; +import org.apache.hadoop.hbase.CellScanner; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + /** * Single row result of a {@link Get} or {@link Scan} query.

* @@ -537,8 +536,7 @@ public class Result implements CellScannable { } this.familyMap = new TreeMap>>(Bytes.BYTES_COMPARATOR); for(KeyValue kv : this.kvs) { - SplitKeyValue splitKV = kv.split(); - byte [] family = splitKV.getFamily(); + byte [] family = kv.getFamily(); NavigableMap> columnMap = familyMap.get(family); if(columnMap == null) { @@ -546,7 +544,7 @@ public class Result implements CellScannable { (Bytes.BYTES_COMPARATOR); familyMap.put(family, columnMap); } - byte [] qualifier = splitKV.getQualifier(); + byte [] qualifier = kv.getQualifier(); NavigableMap versionMap = columnMap.get(qualifier); if(versionMap == null) { versionMap = new TreeMap(new Comparator() { @@ -556,8 +554,9 @@ public class Result implements CellScannable { }); columnMap.put(qualifier, versionMap); } - Long timestamp = Bytes.toLong(splitKV.getTimestamp()); - byte [] value = splitKV.getValue(); + Long timestamp = kv.getTimestamp(); + byte [] value = kv.getValue(); + versionMap.put(timestamp, value); } return this.familyMap; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java index b55d0222bc3..f85687c8613 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -54,8 +54,8 @@ import com.google.common.primitives.Longs; *

* Instances of this class are immutable. They do not implement Comparable but Comparators are * provided. Comparators change with context, whether user table or a catalog table comparison. Its - * critical you use the appropriate comparator. There are Comparators for KeyValue instances and - * then for just the Key portion of a KeyValue used mostly by HFile. + * critical you use the appropriate comparator. There are Comparators for normal HFiles, Meta's + * Hfiles, and bloom filter keys. *

* KeyValue wraps a byte array and takes offsets and lengths into passed array at where to start * interpreting the content as KeyValue. The KeyValue format inside a byte array is: @@ -71,9 +71,6 @@ import com.google.common.primitives.Longs; public class KeyValue implements Cell, HeapSize, Cloneable { static final Log LOG = LogFactory.getLog(KeyValue.class); - // TODO: Group Key-only comparators and operations into a Key class, just - // for neatness sake, if can figure what to call it. - /** * Colon character in UTF-8 */ @@ -83,22 +80,21 @@ public class KeyValue implements Cell, HeapSize, Cloneable { new byte[]{COLUMN_FAMILY_DELIMITER}; /** - * Comparator for plain key/values; i.e. non-catalog table key/values. - */ - public static final KVComparator COMPARATOR = new KVComparator(); - - /** - * Comparator for plain key; i.e. non-catalog table key. Works on Key portion + * Comparator for plain key/values; i.e. non-catalog table key/values. Works on Key portion * of KeyValue only. */ - public static final KeyComparator KEY_COMPARATOR = new KeyComparator(); - + public static final KVComparator COMPARATOR = new KVComparator(); /** * A {@link KVComparator} for .META. catalog table * {@link KeyValue}s. */ public static final KVComparator META_COMPARATOR = new MetaComparator(); + /** + * Needed for Bloom Filters. + */ + public static final KVComparator RAW_COMPARATOR = new RawBytesComparator(); + /** Size of the key length field in bytes*/ public static final int KEY_LENGTH_SIZE = Bytes.SIZEOF_INT; @@ -212,9 +208,11 @@ public class KeyValue implements Cell, HeapSize, Cloneable { public static final KeyValue LOWESTKEY = new KeyValue(HConstants.EMPTY_BYTE_ARRAY, HConstants.LATEST_TIMESTAMP); - private byte [] bytes = null; - private int offset = 0; - private int length = 0; + //// + // KeyValue core instance fields. + private byte [] bytes = null; // an immutable byte array that contains the KV + private int offset = 0; // offset into bytes buffer KV starts at + private int length = 0; // length of the KV starting from offset. /** * @return True if a delete type, a {@link KeyValue.Type#Delete} or @@ -230,15 +228,15 @@ public class KeyValue implements Cell, HeapSize, Cloneable { // used to achieve atomic operations in the memstore. @Override public long getMvccVersion() { - return memstoreTS; + return mvcc; } public void setMvccVersion(long mvccVersion){ - this.memstoreTS = mvccVersion; + this.mvcc = mvccVersion; } - // default value is 0, aka DNC - private long memstoreTS = 0; + // multi-version concurrency control version. default value is 0, aka do not care. + private long mvcc = 0; // this value is not part of a serialized KeyValue (not in HFiles) /** Dragon time over, return to normal business */ @@ -451,7 +449,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable { * @param vlength * @return The newly created byte array. */ - static byte[] createEmptyByteArray(final int rlength, int flength, + private static byte[] createEmptyByteArray(final int rlength, int flength, int qlength, final long timestamp, final Type type, int vlength) { if (rlength > Short.MAX_VALUE) { throw new IllegalArgumentException("Row > " + Short.MAX_VALUE); @@ -624,7 +622,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable { * @param vlength value length * @return The newly created byte array. */ - static byte [] createByteArray(final byte [] row, final int roffset, + private static byte [] createByteArray(final byte [] row, final int roffset, final int rlength, final byte [] family, final int foffset, int flength, final byte [] qualifier, final int qoffset, int qlength, final long timestamp, final Type type, @@ -657,46 +655,6 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return bytes; } - /** - * Write KeyValue format into a byte array. - *

- * Takes column in the form family:qualifier - * @param row - row key (arbitrary byte array) - * @param roffset - * @param rlength - * @param column - * @param coffset - * @param clength - * @param timestamp - * @param type - * @param value - * @param voffset - * @param vlength - * @return The newly created byte array. - */ - static byte [] createByteArray(final byte [] row, final int roffset, - final int rlength, - final byte [] column, final int coffset, int clength, - final long timestamp, final Type type, - final byte [] value, final int voffset, int vlength) { - // If column is non-null, figure where the delimiter is at. - int delimiteroffset = 0; - if (column != null && column.length > 0) { - delimiteroffset = getFamilyDelimiterIndex(column, coffset, clength); - if (delimiteroffset > Byte.MAX_VALUE) { - throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE); - } - } else { - return createByteArray(row,roffset,rlength,null,0,0,null,0,0,timestamp, - type,value,voffset,vlength); - } - int flength = delimiteroffset-coffset; - int qlength = clength - flength - 1; - return createByteArray(row, roffset, rlength, column, coffset, - flength, column, delimiteroffset+1, qlength, timestamp, type, - value, voffset, vlength); - } - /** * Needed doing 'contains' on List. Only compares the key portion, not the value. */ @@ -739,7 +697,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable { // Important to clone the memstoreTS as well - otherwise memstore's // update-in-place methods (eg increment) will end up creating // new entries - ret.setMvccVersion(memstoreTS); + ret.setMvccVersion(mvcc); return ret; } @@ -750,7 +708,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable { */ public KeyValue shallowCopy() { KeyValue shallowCopy = new KeyValue(this.bytes, this.offset, this.length); - shallowCopy.setMvccVersion(this.memstoreTS); + shallowCopy.setMvccVersion(this.mvcc); return shallowCopy; } @@ -765,7 +723,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return "empty"; } return keyToString(this.bytes, this.offset + ROW_OFFSET, getKeyLength()) + - "/vlen=" + getValueLength() + "/mvcc=" + memstoreTS; + "/vlen=" + getValueLength() + "/mvcc=" + mvcc; } /** @@ -1024,15 +982,6 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return getKeyLength() - (int) getKeyDataStructureSize(rlength, flength, 0); } - /** - * @return Column (family + qualifier) length - */ - public int getTotalColumnLength() { - int rlength = getRowLength(); - int foffset = getFamilyOffset(rlength); - return getTotalColumnLength(rlength,foffset); - } - /** * @return Column (family + qualifier) length */ @@ -1065,14 +1014,6 @@ public class KeyValue implements Cell, HeapSize, Cloneable { HConstants.LATEST_TIMESTAMP_BYTES, 0, Bytes.SIZEOF_LONG); } - /** - * @return True if this is a "fake" KV created for internal seeking purposes, - * which should not be seen by user code - */ - public boolean isInternal() { - byte type = getType(); - return type == Type.Minimum.code || type == Type.Maximum.code; - } /** * @param now Time to set into this IFF timestamp == * {@link HConstants#LATEST_TIMESTAMP} (else, its a noop). @@ -1268,76 +1209,6 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return result; } - //--------------------------------------------------------------------------- - // - // KeyValue splitter - // - //--------------------------------------------------------------------------- - - /** - * Utility class that splits a KeyValue buffer into separate byte arrays. - *

- * Should get rid of this if we can, but is very useful for debugging. - */ - public static class SplitKeyValue { - private byte [][] split; - SplitKeyValue() { - this.split = new byte[6][]; - } - public void setRow(byte [] value) { this.split[0] = value; } - public void setFamily(byte [] value) { this.split[1] = value; } - public void setQualifier(byte [] value) { this.split[2] = value; } - public void setTimestamp(byte [] value) { this.split[3] = value; } - public void setType(byte [] value) { this.split[4] = value; } - public void setValue(byte [] value) { this.split[5] = value; } - public byte [] getRow() { return this.split[0]; } - public byte [] getFamily() { return this.split[1]; } - public byte [] getQualifier() { return this.split[2]; } - public byte [] getTimestamp() { return this.split[3]; } - public byte [] getType() { return this.split[4]; } - public byte [] getValue() { return this.split[5]; } - } - - public SplitKeyValue split() { - SplitKeyValue split = new SplitKeyValue(); - int splitOffset = this.offset; - int keyLen = Bytes.toInt(bytes, splitOffset); - splitOffset += Bytes.SIZEOF_INT; - int valLen = Bytes.toInt(bytes, splitOffset); - splitOffset += Bytes.SIZEOF_INT; - short rowLen = Bytes.toShort(bytes, splitOffset); - splitOffset += Bytes.SIZEOF_SHORT; - byte [] row = new byte[rowLen]; - System.arraycopy(bytes, splitOffset, row, 0, rowLen); - splitOffset += rowLen; - split.setRow(row); - byte famLen = bytes[splitOffset]; - splitOffset += Bytes.SIZEOF_BYTE; - byte [] family = new byte[famLen]; - System.arraycopy(bytes, splitOffset, family, 0, famLen); - splitOffset += famLen; - split.setFamily(family); - int colLen = keyLen - - (rowLen + famLen + Bytes.SIZEOF_SHORT + Bytes.SIZEOF_BYTE + - Bytes.SIZEOF_LONG + Bytes.SIZEOF_BYTE); - byte [] qualifier = new byte[colLen]; - System.arraycopy(bytes, splitOffset, qualifier, 0, colLen); - splitOffset += colLen; - split.setQualifier(qualifier); - byte [] timestamp = new byte[Bytes.SIZEOF_LONG]; - System.arraycopy(bytes, splitOffset, timestamp, 0, Bytes.SIZEOF_LONG); - splitOffset += Bytes.SIZEOF_LONG; - split.setTimestamp(timestamp); - byte [] type = new byte[1]; - type[0] = bytes[splitOffset]; - splitOffset += Bytes.SIZEOF_BYTE; - split.setType(type); - byte [] value = new byte[valLen]; - System.arraycopy(bytes, splitOffset, value, 0, valLen); - split.setValue(value); - return split; - } - //--------------------------------------------------------------------------- // // Compare specified fields against those contained in this KeyValue @@ -1349,22 +1220,13 @@ public class KeyValue implements Cell, HeapSize, Cloneable { * @return True if matching families. */ public boolean matchingFamily(final byte [] family) { - return matchingFamily(family, 0, family.length); - } - - public boolean matchingFamily(final byte[] family, int offset, int length) { if (this.length == 0 || this.bytes.length == 0) { return false; } - return Bytes.equals(family, offset, length, + return Bytes.equals(family, 0, family.length, this.bytes, getFamilyOffset(), getFamilyLength()); } - public boolean matchingFamily(final KeyValue other) { - return matchingFamily(other.getBuffer(), other.getFamilyOffset(), - other.getFamilyLength()); - } - /** * @param qualifier * @return True if matching qualifiers. @@ -1397,18 +1259,6 @@ public class KeyValue implements Cell, HeapSize, Cloneable { other.getRowLength()); } - /** - * @param column Column minus its delimiter - * @return True if column matches. - */ - public boolean matchingColumnNoDelimiter(final byte [] column) { - int rl = getRowLength(); - int o = getFamilyOffset(rl); - int fl = getFamilyLength(o); - int l = fl + getQualifierLength(rl,fl); - return Bytes.equals(column, 0, column.length, this.bytes, o, l); - } - /** * * @param family column family @@ -1447,47 +1297,6 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return Bytes.equals(qualifier, qoffset, qlength, this.bytes, o + fl, ql); } - /** - * @param left - * @param loffset - * @param llength - * @param lfamilylength Offset of family delimiter in left column. - * @param right - * @param roffset - * @param rlength - * @param rfamilylength Offset of family delimiter in right column. - * @return The result of the comparison. - */ - static int compareColumns(final byte [] left, final int loffset, - final int llength, final int lfamilylength, - final byte [] right, final int roffset, final int rlength, - final int rfamilylength) { - // Compare family portion first. - int diff = Bytes.compareTo(left, loffset, lfamilylength, - right, roffset, rfamilylength); - if (diff != 0) { - return diff; - } - // Compare qualifier portion - return Bytes.compareTo(left, loffset + lfamilylength, - llength - lfamilylength, - right, roffset + rfamilylength, rlength - rfamilylength); - } - - /** - * @return True if non-null row and column. - */ - public boolean nonNullRowAndColumn() { - return getRowLength() > 0 && !isEmptyColumn(); - } - - /** - * @return True if column is empty. - */ - public boolean isEmptyColumn() { - return getQualifierLength() == 0; - } - /** * Creates a new KeyValue that only contains the key portion (the value is * set to be null). @@ -1548,26 +1357,6 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return Bytes.add(family, COLUMN_FAMILY_DELIM_ARRAY, qualifier); } - /** - * @param b - * @return Index of the family-qualifier colon delimiter character in passed - * buffer. - */ - public static int getFamilyDelimiterIndex(final byte [] b, final int offset, - final int length) { - return getRequiredDelimiter(b, offset, length, COLUMN_FAMILY_DELIMITER); - } - - private static int getRequiredDelimiter(final byte [] b, - final int offset, final int length, final int delimiter) { - int index = getDelimiter(b, offset, length, delimiter); - if (index < 0) { - throw new IllegalArgumentException("No " + (char)delimiter + " in <" + - Bytes.toString(b) + ">" + ", length=" + length + ", offset=" + offset); - } - return index; - } - /** * This function is only used in Meta key comparisons so its error message * is specific for meta key errors. @@ -1629,29 +1418,13 @@ public class KeyValue implements Cell, HeapSize, Cloneable { * {@link KeyValue}s. */ public static class MetaComparator extends KVComparator { - private final KeyComparator rawcomparator = new MetaKeyComparator(); - - public KeyComparator getRawComparator() { - return this.rawcomparator; - } - - @Override - protected Object clone() throws CloneNotSupportedException { - return new MetaComparator(); - } - /** - * Override the row key comparision to parse and compare the meta row key parts. + * Compare key portion of a {@link KeyValue} for keys in .META. + * table. */ @Override - protected int compareRowKey(final Cell l, final Cell r) { - byte[] left = l.getRowArray(); - int loffset = l.getRowOffset(); - int llength = l.getRowLength(); - byte[] right = r.getRowArray(); - int roffset = r.getRowOffset(); - int rlength = r.getRowLength(); - + public int compareRows(byte [] left, int loffset, int llength, + byte [] right, int roffset, int rlength) { int leftDelimiter = getDelimiter(left, loffset, llength, HConstants.DELIMITER); int rightDelimiter = getDelimiter(right, roffset, rlength, @@ -1680,45 +1453,127 @@ public class KeyValue implements Cell, HeapSize, Cloneable { rightDelimiter, rlength - (rightDelimiter - roffset), HConstants.DELIMITER); // Now compare middlesection of row. - result = Bytes.compareTo( - left, leftDelimiter, leftFarDelimiter - leftDelimiter, - right, rightDelimiter, rightFarDelimiter - rightDelimiter); + result = super.compareRows(left, leftDelimiter, + leftFarDelimiter - leftDelimiter, right, rightDelimiter, + rightFarDelimiter - rightDelimiter); if (result != 0) { return result; } // Compare last part of row, the rowid. leftFarDelimiter++; rightFarDelimiter++; - result = Bytes.compareTo( - left, leftFarDelimiter, llength - (leftFarDelimiter - loffset), + result = Bytes.compareTo(left, leftFarDelimiter, llength - (leftFarDelimiter - loffset), right, rightFarDelimiter, rlength - (rightFarDelimiter - roffset)); return result; } + + /** + * Don't do any fancy Block Index splitting tricks. + */ + @Override + public byte[] getShortMidpointKey(final byte[] leftKey, final byte[] rightKey) { + return Arrays.copyOf(rightKey, rightKey.length); + } + + /** + * The HFileV2 file format's trailer contains this class name. We reinterpret this and + * instantiate the appropriate comparator. + * TODO: With V3 consider removing this. + * @return legacy class name for FileFileTrailer#comparatorClassName + */ + @Override + public String getLegacyKeyComparatorName() { + return "org.apache.hadoop.hbase.KeyValue$MetaKeyComparator"; + } + + @Override + protected Object clone() throws CloneNotSupportedException { + return new MetaComparator(); + } + + /** + * Override the row key comparison to parse and compare the meta row key parts. + */ + @Override + protected int compareRowKey(final Cell l, final Cell r) { + byte[] left = l.getRowArray(); + int loffset = l.getRowOffset(); + int llength = l.getRowLength(); + byte[] right = r.getRowArray(); + int roffset = r.getRowOffset(); + int rlength = r.getRowLength(); + return compareRows(left, loffset, llength, right, roffset, rlength); + } } /** * Compare KeyValues. When we compare KeyValues, we only compare the Key * portion. This means two KeyValues with same Key but different Values are * considered the same as far as this Comparator is concerned. - * Hosts a {@link KeyComparator}. */ - public static class KVComparator implements java.util.Comparator { - private final KeyComparator rawcomparator = new KeyComparator(); + public static class KVComparator implements RawComparator, SamePrefixComparator { /** - * @return RawComparator that can compare the Key portion of a KeyValue. - * Used in hfile where indices are the Key portion of a KeyValue. + * The HFileV2 file format's trailer contains this class name. We reinterpret this and + * instantiate the appropriate comparator. + * TODO: With V3 consider removing this. + * @return legacy class name for FileFileTrailer#comparatorClassName */ - public KeyComparator getRawComparator() { - return this.rawcomparator; + public String getLegacyKeyComparatorName() { + return "org.apache.hadoop.hbase.KeyValue$KeyComparator"; } + @Override // RawComparator + public int compare(byte[] l, int loff, int llen, byte[] r, int roff, int rlen) { + return compareFlatKey(l,loff,llen, r,roff,rlen); + } + + + /** + * Compares the only the user specified portion of a Key. This is overridden by MetaComparator. + * @param left + * @param right + * @return 0 if equal, <0 if left smaller, >0 if right smaller + */ protected int compareRowKey(final Cell left, final Cell right) { return Bytes.compareTo( left.getRowArray(), left.getRowOffset(), left.getRowLength(), right.getRowArray(), right.getRowOffset(), right.getRowLength()); } - + + /** + * Compares left to right assuming that left,loffset,llength and right,roffset,rlength are + * full KVs laid out in a flat byte[]s. + * @param left + * @param loffset + * @param llength + * @param right + * @param roffset + * @param rlength + * @return 0 if equal, <0 if left smaller, >0 if right smaller + */ + public int compareFlatKey(byte[] left, int loffset, int llength, + byte[] right, int roffset, int rlength) { + // Compare row + short lrowlength = Bytes.toShort(left, loffset); + short rrowlength = Bytes.toShort(right, roffset); + int compare = compareRows(left, loffset + Bytes.SIZEOF_SHORT, + lrowlength, right, roffset + Bytes.SIZEOF_SHORT, rrowlength); + if (compare != 0) { + return compare; + } + + // Compare the rest of the two KVs without making any assumptions about + // the common prefix. This function will not compare rows anyway, so we + // don't need to tell it that the common prefix includes the row. + return compareWithoutRow(0, left, loffset, llength, right, roffset, + rlength, rrowlength); + } + + public int compareFlatKey(byte[] left, byte[] right) { + return compareFlatKey(left, 0, left.length, right, 0, right.length); + } + /** * Compares the Key of a cell -- with fields being more significant in this order: * rowkey, colfam/qual, timestamp, type, mvcc @@ -1730,7 +1585,6 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return compare; } - // compare vs minimum byte ltype = left.getTypeByte(); byte rtype = right.getTypeByte(); @@ -1759,7 +1613,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable { if (compare != 0) { return compare; } - + // Compare qualifier compare = Bytes.compareTo( left.getQualifierArray(), left.getQualifierOffset(), left.getQualifierLength(), @@ -1768,11 +1622,10 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return compare; } - // compare timestamp long ltimestamp = left.getTimestamp(); long rtimestamp = right.getTimestamp(); - compare = KeyComparator.compareTimestamps(ltimestamp, rtimestamp); + compare = compareTimestamps(ltimestamp, rtimestamp); if (compare != 0) { return compare; } @@ -1785,24 +1638,17 @@ public class KeyValue implements Cell, HeapSize, Cloneable { if (compare != 0) { return compare; } - - // compare Mvcc Version + // compare Mvcc Version // Negate this comparison so later edits show up first return -Longs.compare(left.getMvccVersion(), right.getMvccVersion()); } public int compareTimestamps(final KeyValue left, final KeyValue right) { - return compareTimestamps(left, left.getKeyLength(), right, - right.getKeyLength()); - } - - int compareTimestamps(final KeyValue left, final int lkeylength, - final KeyValue right, final int rkeylength) { // Compare timestamps - long ltimestamp = left.getTimestamp(lkeylength); - long rtimestamp = right.getTimestamp(rkeylength); - return KeyComparator.compareTimestamps(ltimestamp, rtimestamp); + long ltimestamp = left.getTimestamp(left.getKeyLength()); + long rtimestamp = right.getTimestamp(right.getKeyLength()); + return compareTimestamps(ltimestamp, rtimestamp); } /** @@ -1811,47 +1657,23 @@ public class KeyValue implements Cell, HeapSize, Cloneable { * @return Result comparing rows. */ public int compareRows(final KeyValue left, final KeyValue right) { - return compareRows(left, left.getRowLength(), right, - right.getRowLength()); + return compareRows(left.getBuffer(),left.getRowOffset(), left.getRowLength(), + right.getBuffer(), right.getRowOffset(), right.getRowLength()); } /** + * Get the b[],o,l for left and right rowkey portions and compare. * @param left - * @param lrowlength Length of left row. + * @param loffset + * @param llength * @param right - * @param rrowlength Length of right row. - * @return Result comparing rows. + * @param roffset + * @param rlength + * @return 0 if equal, <0 if left smaller, >0 if right smaller */ - public int compareRows(final KeyValue left, final short lrowlength, - final KeyValue right, final short rrowlength) { - return getRawComparator().compareRows(left.getBuffer(), - left.getRowOffset(), lrowlength, - right.getBuffer(), right.getRowOffset(), rrowlength); - } - - /** - * @param left - * @param row - row key (arbitrary byte array) - * @return RawComparator - */ - public int compareRows(final KeyValue left, final byte [] row) { - return getRawComparator().compareRows(left.getBuffer(), - left.getRowOffset(), left.getRowLength(), row, 0, row.length); - } - public int compareRows(byte [] left, int loffset, int llength, byte [] right, int roffset, int rlength) { - return getRawComparator().compareRows(left, loffset, llength, - right, roffset, rlength); - } - - public int compareColumns(final KeyValue left, final byte [] right, - final int roffset, final int rlength, final int rfamilyoffset) { - int offset = left.getFamilyOffset(); - int length = left.getFamilyLength() + left.getQualifierLength(); - return getRawComparator().compareColumns(left.getBuffer(), offset, length, - left.getFamilyLength(offset), - right, roffset, rlength, rfamilyoffset); + return Bytes.compareTo(left, loffset, llength, right, roffset, rlength); } int compareColumns(final KeyValue left, final short lrowlength, @@ -1862,11 +1684,175 @@ public class KeyValue implements Cell, HeapSize, Cloneable { int rclength = right.getTotalColumnLength(rrowlength, rfoffset); int lfamilylength = left.getFamilyLength(lfoffset); int rfamilylength = right.getFamilyLength(rfoffset); - return getRawComparator().compareColumns(left.getBuffer(), lfoffset, + return compareColumns(left.getBuffer(), lfoffset, lclength, lfamilylength, right.getBuffer(), rfoffset, rclength, rfamilylength); } + protected int compareColumns( + byte [] left, int loffset, int llength, final int lfamilylength, + byte [] right, int roffset, int rlength, final int rfamilylength) { + // Compare family portion first. + int diff = Bytes.compareTo(left, loffset, lfamilylength, + right, roffset, rfamilylength); + if (diff != 0) { + return diff; + } + // Compare qualifier portion + return Bytes.compareTo(left, loffset + lfamilylength, + llength - lfamilylength, + right, roffset + rfamilylength, rlength - rfamilylength); + } + + static int compareTimestamps(final long ltimestamp, final long rtimestamp) { + // The below older timestamps sorting ahead of newer timestamps looks + // wrong but it is intentional. This way, newer timestamps are first + // found when we iterate over a memstore and newer versions are the + // first we trip over when reading from a store file. + if (ltimestamp < rtimestamp) { + return 1; + } else if (ltimestamp > rtimestamp) { + return -1; + } + return 0; + } + + /** + * Overridden + * @param commonPrefix + * @param left + * @param loffset + * @param llength + * @param right + * @param roffset + * @param rlength + * @return 0 if equal, <0 if left smaller, >0 if right smaller + */ + @Override // SamePrefixComparator + public int compareIgnoringPrefix(int commonPrefix, byte[] left, + int loffset, int llength, byte[] right, int roffset, int rlength) { + // Compare row + short lrowlength = Bytes.toShort(left, loffset); + short rrowlength; + + int comparisonResult = 0; + if (commonPrefix < ROW_LENGTH_SIZE) { + // almost nothing in common + rrowlength = Bytes.toShort(right, roffset); + comparisonResult = compareRows(left, loffset + ROW_LENGTH_SIZE, + lrowlength, right, roffset + ROW_LENGTH_SIZE, rrowlength); + } else { // the row length is the same + rrowlength = lrowlength; + if (commonPrefix < ROW_LENGTH_SIZE + rrowlength) { + // The rows are not the same. Exclude the common prefix and compare + // the rest of the two rows. + int common = commonPrefix - ROW_LENGTH_SIZE; + comparisonResult = compareRows( + left, loffset + common + ROW_LENGTH_SIZE, lrowlength - common, + right, roffset + common + ROW_LENGTH_SIZE, rrowlength - common); + } + } + if (comparisonResult != 0) { + return comparisonResult; + } + + assert lrowlength == rrowlength; + return compareWithoutRow(commonPrefix, left, loffset, llength, right, + roffset, rlength, lrowlength); + } + + /** + * Compare columnFamily, qualifier, timestamp, and key type (everything + * except the row). This method is used both in the normal comparator and + * the "same-prefix" comparator. Note that we are assuming that row portions + * of both KVs have already been parsed and found identical, and we don't + * validate that assumption here. + * @param commonPrefix + * the length of the common prefix of the two key-values being + * compared, including row length and row + */ + private int compareWithoutRow(int commonPrefix, byte[] left, int loffset, + int llength, byte[] right, int roffset, int rlength, short rowlength) { + /*** + * KeyValue Format and commonLength: + * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|.... + * ------------------|-------commonLength--------|-------------- + */ + int commonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + rowlength; + + // commonLength + TIMESTAMP_TYPE_SIZE + int commonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + commonLength; + // ColumnFamily + Qualifier length. + int lcolumnlength = llength - commonLengthWithTSAndType; + int rcolumnlength = rlength - commonLengthWithTSAndType; + + byte ltype = left[loffset + (llength - 1)]; + byte rtype = right[roffset + (rlength - 1)]; + + // If the column is not specified, the "minimum" key type appears the + // latest in the sorted order, regardless of the timestamp. This is used + // for specifying the last key/value in a given row, because there is no + // "lexicographically last column" (it would be infinitely long). The + // "maximum" key type does not need this behavior. + if (lcolumnlength == 0 && ltype == Type.Minimum.getCode()) { + // left is "bigger", i.e. it appears later in the sorted order + return 1; + } + if (rcolumnlength == 0 && rtype == Type.Minimum.getCode()) { + return -1; + } + + int lfamilyoffset = commonLength + loffset; + int rfamilyoffset = commonLength + roffset; + + // Column family length. + int lfamilylength = left[lfamilyoffset - 1]; + int rfamilylength = right[rfamilyoffset - 1]; + // If left family size is not equal to right family size, we need not + // compare the qualifiers. + boolean sameFamilySize = (lfamilylength == rfamilylength); + int common = 0; + if (commonPrefix > 0) { + common = Math.max(0, commonPrefix - commonLength); + if (!sameFamilySize) { + // Common should not be larger than Math.min(lfamilylength, + // rfamilylength). + common = Math.min(common, Math.min(lfamilylength, rfamilylength)); + } else { + common = Math.min(common, Math.min(lcolumnlength, rcolumnlength)); + } + } + if (!sameFamilySize) { + // comparing column family is enough. + return Bytes.compareTo(left, lfamilyoffset + common, lfamilylength + - common, right, rfamilyoffset + common, rfamilylength - common); + } + // Compare family & qualifier together. + final int comparison = Bytes.compareTo(left, lfamilyoffset + common, + lcolumnlength - common, right, rfamilyoffset + common, + rcolumnlength - common); + if (comparison != 0) { + return comparison; + } + + //// + // Next compare timestamps. + long ltimestamp = Bytes.toLong(left, + loffset + (llength - TIMESTAMP_TYPE_SIZE)); + long rtimestamp = Bytes.toLong(right, + roffset + (rlength - TIMESTAMP_TYPE_SIZE)); + int compare = compareTimestamps(ltimestamp, rtimestamp); + if (compare != 0) { + return compare; + } + + // Compare types. Let the delete types sort ahead of puts; i.e. types + // of higher numbers sort before those of lesser numbers. Maximum (255) + // appears ahead of everything, and minimum (0) appears after + // everything. + return (0xff & rtype) - (0xff & ltype); + } + /** * Compares the row and column of two keyvalues for equality * @param left @@ -1877,21 +1863,26 @@ public class KeyValue implements Cell, HeapSize, Cloneable { final KeyValue right) { short lrowlength = left.getRowLength(); short rrowlength = right.getRowLength(); - // TsOffset = end of column data. just comparing Row+CF length of each - return ((left.getTimestampOffset() - left.getOffset()) == - (right.getTimestampOffset() - right.getOffset())) && - matchingRows(left, lrowlength, right, rrowlength) && - compareColumns(left, lrowlength, right, rrowlength) == 0; - } - /** - * @param left - * @param right - * @return True if rows match. - */ - public boolean matchingRows(final KeyValue left, final byte [] right) { - return Bytes.equals(left.getBuffer(), left.getRowOffset(), left.getRowLength(), - right, 0, right.length); + // TsOffset = end of column data. just comparing Row+CF length of each + if ((left.getTimestampOffset() - left.getOffset()) != + (right.getTimestampOffset() - right.getOffset())) { + return false; + } + + if (!matchingRows(left, lrowlength, right, rrowlength)) { + return false; + } + + int lfoffset = left.getFamilyOffset(lrowlength); + int rfoffset = right.getFamilyOffset(rrowlength); + int lclength = left.getTotalColumnLength(lrowlength,lfoffset); + int rclength = right.getTotalColumnLength(rrowlength, rfoffset); + int lfamilylength = left.getFamilyLength(lfoffset); + int rfamilylength = right.getFamilyLength(rfoffset); + int ccRes = compareColumns(left.getBuffer(), lfoffset, lclength, lfamilylength, + right.getBuffer(), rfoffset, rclength, rfamilylength); + return ccRes == 0; } /** @@ -1913,41 +1904,114 @@ public class KeyValue implements Cell, HeapSize, Cloneable { * @param rrowlength * @return True if rows match. */ - public boolean matchingRows(final KeyValue left, final short lrowlength, + private boolean matchingRows(final KeyValue left, final short lrowlength, final KeyValue right, final short rrowlength) { return lrowlength == rrowlength && Bytes.equals(left.getBuffer(), left.getRowOffset(), lrowlength, right.getBuffer(), right.getRowOffset(), rrowlength); } - public boolean matchingRows(final byte [] left, final int loffset, - final int llength, - final byte [] right, final int roffset, final int rlength) { - return Bytes.equals(left, loffset, llength, - right, roffset, rlength); + public byte[] calcIndexKey(byte[] lastKeyOfPreviousBlock, byte[] firstKeyInBlock) { + byte[] fakeKey = getShortMidpointKey(lastKeyOfPreviousBlock, firstKeyInBlock); + if (compareFlatKey(fakeKey, firstKeyInBlock) > 0) { + LOG.error("Unexpected getShortMidpointKey result, fakeKey:" + + Bytes.toStringBinary(fakeKey) + ", firstKeyInBlock:" + + Bytes.toStringBinary(firstKeyInBlock)); + return firstKeyInBlock; + } + if (lastKeyOfPreviousBlock != null && compareFlatKey(lastKeyOfPreviousBlock, fakeKey) >= 0) { + LOG.error("Unexpected getShortMidpointKey result, lastKeyOfPreviousBlock:" + + Bytes.toStringBinary(lastKeyOfPreviousBlock) + ", fakeKey:" + + Bytes.toStringBinary(fakeKey)); + return firstKeyInBlock; + } + return fakeKey; } /** - * Compares the row and timestamp of two keys - * Was called matchesWithoutColumn in HStoreKey. - * @param right Key to compare against. - * @return True if same row and timestamp is greater than the timestamp in - * right + * This is a HFile block index key optimization. + * @param leftKey + * @param rightKey + * @return 0 if equal, <0 if left smaller, >0 if right smaller */ - public boolean matchingRowsGreaterTimestamp(final KeyValue left, - final KeyValue right) { - short lrowlength = left.getRowLength(); - short rrowlength = right.getRowLength(); - if (!matchingRows(left, lrowlength, right, rrowlength)) { - return false; + public byte[] getShortMidpointKey(final byte[] leftKey, final byte[] rightKey) { + if (rightKey == null) { + throw new IllegalArgumentException("rightKey can not be null"); } - return left.getTimestamp() >= right.getTimestamp(); + if (leftKey == null) { + return Arrays.copyOf(rightKey, rightKey.length); + } + if (compareFlatKey(leftKey, rightKey) >= 0) { + throw new IllegalArgumentException("Unexpected input, leftKey:" + Bytes.toString(leftKey) + + ", rightKey:" + Bytes.toString(rightKey)); + } + + short leftRowLength = Bytes.toShort(leftKey, 0); + short rightRowLength = Bytes.toShort(rightKey, 0); + int leftCommonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + leftRowLength; + int rightCommonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + rightRowLength; + int leftCommonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + leftCommonLength; + int rightCommonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + rightCommonLength; + int leftColumnLength = leftKey.length - leftCommonLengthWithTSAndType; + int rightColumnLength = rightKey.length - rightCommonLengthWithTSAndType; + // rows are equal + if (leftRowLength == rightRowLength && compareRows(leftKey, ROW_LENGTH_SIZE, leftRowLength, + rightKey, ROW_LENGTH_SIZE, rightRowLength) == 0) { + // Compare family & qualifier together. + int comparison = Bytes.compareTo(leftKey, leftCommonLength, leftColumnLength, rightKey, + rightCommonLength, rightColumnLength); + // same with "row + family + qualifier", return rightKey directly + if (comparison == 0) { + return Arrays.copyOf(rightKey, rightKey.length); + } + // "family + qualifier" are different, generate a faked key per rightKey + byte[] newKey = Arrays.copyOf(rightKey, rightKey.length); + Bytes.putLong(newKey, rightKey.length - TIMESTAMP_TYPE_SIZE, HConstants.LATEST_TIMESTAMP); + Bytes.putByte(newKey, rightKey.length - TYPE_SIZE, Type.Maximum.getCode()); + return newKey; + } + // rows are different + short minLength = leftRowLength < rightRowLength ? leftRowLength : rightRowLength; + short diffIdx = 0; + while (diffIdx < minLength + && leftKey[ROW_LENGTH_SIZE + diffIdx] == rightKey[ROW_LENGTH_SIZE + diffIdx]) { + diffIdx++; + } + if (diffIdx >= minLength) { + // leftKey's row is prefix of rightKey's. we can optimize it in future + return Arrays.copyOf(rightKey, rightKey.length); + } + int diffByte = leftKey[ROW_LENGTH_SIZE + diffIdx]; + if ((0xff & diffByte) < 0xff && (diffByte + 1) < + (rightKey[ROW_LENGTH_SIZE + diffIdx] & 0xff)) { + byte[] newRowKey = new byte[diffIdx + 1]; + System.arraycopy(leftKey, ROW_LENGTH_SIZE, newRowKey, 0, diffIdx); + newRowKey[diffIdx] = (byte) (diffByte + 1); + int rightFamilyLength = rightKey[rightCommonLength - 1]; + byte[] family = null; + if (rightFamilyLength > 0) { + family = new byte[rightFamilyLength]; + System.arraycopy(rightKey, rightCommonLength, family, 0, rightFamilyLength); + } + int rightQualifierLength = rightColumnLength - rightFamilyLength; + byte[] qualifier = null; + if (rightQualifierLength > 0) { + qualifier = new byte[rightQualifierLength]; + System.arraycopy(rightKey, rightCommonLength + rightFamilyLength, qualifier, 0, + rightQualifierLength); + } + return new KeyValue(newRowKey, null, null, HConstants.LATEST_TIMESTAMP, + Type.Maximum).getKey(); + } + // the following is optimizable in future + return Arrays.copyOf(rightKey, rightKey.length); } @Override protected Object clone() throws CloneNotSupportedException { return new KVComparator(); } + } /** @@ -2316,10 +2380,10 @@ public class KeyValue implements Cell, HeapSize, Cloneable { /** * Comparator that compares row component only of a KeyValue. */ - public static class RowComparator implements Comparator { + public static class RowOnlyComparator implements Comparator { final KVComparator comparator; - public RowComparator(final KVComparator c) { + public RowOnlyComparator(final KVComparator c) { this.comparator = c; } @@ -2328,69 +2392,11 @@ public class KeyValue implements Cell, HeapSize, Cloneable { } } - /** - * Compare key portion of a {@link KeyValue} for keys in .META. - * table. - */ - public static class MetaKeyComparator extends KeyComparator { - public int compareRows(byte [] left, int loffset, int llength, - byte [] right, int roffset, int rlength) { - int leftDelimiter = getDelimiter(left, loffset, llength, - HConstants.DELIMITER); - int rightDelimiter = getDelimiter(right, roffset, rlength, - HConstants.DELIMITER); - if (leftDelimiter < 0 && rightDelimiter >= 0) { - // Nothing between .META. and regionid. Its first key. - return -1; - } else if (rightDelimiter < 0 && leftDelimiter >= 0) { - return 1; - } else if (leftDelimiter < 0 && rightDelimiter < 0) { - return 0; - } - // Compare up to the delimiter - int result = Bytes.compareTo(left, loffset, leftDelimiter - loffset, - right, roffset, rightDelimiter - roffset); - if (result != 0) { - return result; - } - // Compare middle bit of the row. - // Move past delimiter - leftDelimiter++; - rightDelimiter++; - int leftFarDelimiter = getRequiredDelimiterInReverse(left, leftDelimiter, - llength - (leftDelimiter - loffset), HConstants.DELIMITER); - int rightFarDelimiter = getRequiredDelimiterInReverse(right, - rightDelimiter, rlength - (rightDelimiter - roffset), - HConstants.DELIMITER); - // Now compare middlesection of row. - result = super.compareRows(left, leftDelimiter, - leftFarDelimiter - leftDelimiter, right, rightDelimiter, - rightFarDelimiter - rightDelimiter); - if (result != 0) { - return result; - } - // Compare last part of row, the rowid. - leftFarDelimiter++; - rightFarDelimiter++; - result = compareRowid(left, leftFarDelimiter, - llength - (leftFarDelimiter - loffset), - right, rightFarDelimiter, rlength - (rightFarDelimiter - roffset)); - return result; - } - - @Override - public byte[] getShortMidpointKey(final byte[] leftKey, final byte[] rightKey) { - return Arrays.copyOf(rightKey, rightKey.length); - } - - protected int compareRowid(byte[] left, int loffset, int llength, - byte[] right, int roffset, int rlength) { - return Bytes.compareTo(left, loffset, llength, right, roffset, rlength); - } - } /** * Avoids redundant comparisons for better performance. + * + * TODO get rid of this wart */ public interface SamePrefixComparator { /** @@ -2398,326 +2404,33 @@ public class KeyValue implements Cell, HeapSize, Cloneable { * @param commonPrefix How many bytes are the same. */ int compareIgnoringPrefix( - int commonPrefix, T left, int loffset, int llength, T right, int roffset, int rlength + int commonPrefix, byte[] left, int loffset, int llength, byte[] right, int roffset, int rlength ); } - /** - * Compare key portion of a {@link KeyValue}. - */ - public static class KeyComparator - implements RawComparator, SamePrefixComparator { - - @Override - public int compare(byte[] left, int loffset, int llength, byte[] right, - int roffset, int rlength) { - // Compare row - short lrowlength = Bytes.toShort(left, loffset); - short rrowlength = Bytes.toShort(right, roffset); - int compare = compareRows(left, loffset + Bytes.SIZEOF_SHORT, - lrowlength, right, roffset + Bytes.SIZEOF_SHORT, rrowlength); - if (compare != 0) { - return compare; - } - - // Compare the rest of the two KVs without making any assumptions about - // the common prefix. This function will not compare rows anyway, so we - // don't need to tell it that the common prefix includes the row. - return compareWithoutRow(0, left, loffset, llength, right, roffset, - rlength, rrowlength); - } - - /** - * Compare the two key-values, ignoring the prefix of the given length - * that is known to be the same between the two. - * @param commonPrefix the prefix length to ignore - */ - @Override - public int compareIgnoringPrefix(int commonPrefix, byte[] left, - int loffset, int llength, byte[] right, int roffset, int rlength) { - // Compare row - short lrowlength = Bytes.toShort(left, loffset); - short rrowlength; - - int comparisonResult = 0; - if (commonPrefix < ROW_LENGTH_SIZE) { - // almost nothing in common - rrowlength = Bytes.toShort(right, roffset); - comparisonResult = compareRows(left, loffset + ROW_LENGTH_SIZE, - lrowlength, right, roffset + ROW_LENGTH_SIZE, rrowlength); - } else { // the row length is the same - rrowlength = lrowlength; - if (commonPrefix < ROW_LENGTH_SIZE + rrowlength) { - // The rows are not the same. Exclude the common prefix and compare - // the rest of the two rows. - int common = commonPrefix - ROW_LENGTH_SIZE; - comparisonResult = compareRows( - left, loffset + common + ROW_LENGTH_SIZE, lrowlength - common, - right, roffset + common + ROW_LENGTH_SIZE, rrowlength - common); - } - } - if (comparisonResult != 0) { - return comparisonResult; - } - - assert lrowlength == rrowlength; - - return compareWithoutRow(commonPrefix, left, loffset, llength, right, - roffset, rlength, lrowlength); - } - - /** - * Compare columnFamily, qualifier, timestamp, and key type (everything - * except the row). This method is used both in the normal comparator and - * the "same-prefix" comparator. Note that we are assuming that row portions - * of both KVs have already been parsed and found identical, and we don't - * validate that assumption here. - * @param commonPrefix - * the length of the common prefix of the two key-values being - * compared, including row length and row - */ - private int compareWithoutRow(int commonPrefix, byte[] left, int loffset, - int llength, byte[] right, int roffset, int rlength, short rowlength) { - /*** - * KeyValue Format and commonLength: - * |_keyLen_|_valLen_|_rowLen_|_rowKey_|_famiLen_|_fami_|_Quali_|.... - * ------------------|-------commonLength--------|-------------- - */ - int commonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + rowlength; - - // commonLength + TIMESTAMP_TYPE_SIZE - int commonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + commonLength; - // ColumnFamily + Qualifier length. - int lcolumnlength = llength - commonLengthWithTSAndType; - int rcolumnlength = rlength - commonLengthWithTSAndType; - - byte ltype = left[loffset + (llength - 1)]; - byte rtype = right[roffset + (rlength - 1)]; - - // If the column is not specified, the "minimum" key type appears the - // latest in the sorted order, regardless of the timestamp. This is used - // for specifying the last key/value in a given row, because there is no - // "lexicographically last column" (it would be infinitely long). The - // "maximum" key type does not need this behavior. - if (lcolumnlength == 0 && ltype == Type.Minimum.getCode()) { - // left is "bigger", i.e. it appears later in the sorted order - return 1; - } - if (rcolumnlength == 0 && rtype == Type.Minimum.getCode()) { - return -1; - } - - int lfamilyoffset = commonLength + loffset; - int rfamilyoffset = commonLength + roffset; - - // Column family length. - int lfamilylength = left[lfamilyoffset - 1]; - int rfamilylength = right[rfamilyoffset - 1]; - // If left family size is not equal to right family size, we need not - // compare the qualifiers. - boolean sameFamilySize = (lfamilylength == rfamilylength); - int common = 0; - if (commonPrefix > 0) { - common = Math.max(0, commonPrefix - commonLength); - if (!sameFamilySize) { - // Common should not be larger than Math.min(lfamilylength, - // rfamilylength). - common = Math.min(common, Math.min(lfamilylength, rfamilylength)); - } else { - common = Math.min(common, Math.min(lcolumnlength, rcolumnlength)); - } - } - if (!sameFamilySize) { - // comparing column family is enough. - return Bytes.compareTo(left, lfamilyoffset + common, lfamilylength - - common, right, rfamilyoffset + common, rfamilylength - common); - } - // Compare family & qualifier together. - final int comparison = Bytes.compareTo(left, lfamilyoffset + common, - lcolumnlength - common, right, rfamilyoffset + common, - rcolumnlength - common); - if (comparison != 0) { - return comparison; - } - return compareTimestampAndType(left, loffset, llength, right, roffset, - rlength, ltype, rtype); - } - - private int compareTimestampAndType(byte[] left, int loffset, int llength, - byte[] right, int roffset, int rlength, byte ltype, byte rtype) { - int compare; - // Get timestamps. - long ltimestamp = Bytes.toLong(left, - loffset + (llength - TIMESTAMP_TYPE_SIZE)); - long rtimestamp = Bytes.toLong(right, - roffset + (rlength - TIMESTAMP_TYPE_SIZE)); - compare = compareTimestamps(ltimestamp, rtimestamp); - if (compare != 0) { - return compare; - } - - // Compare types. Let the delete types sort ahead of puts; i.e. types - // of higher numbers sort before those of lesser numbers. Maximum (255) - // appears ahead of everything, and minimum (0) appears after - // everything. - return (0xff & rtype) - (0xff & ltype); - } - - public int compare(byte[] left, byte[] right) { - return compare(left, 0, left.length, right, 0, right.length); - } - - public int compareRows(byte [] left, int loffset, int llength, - byte [] right, int roffset, int rlength) { - return Bytes.compareTo(left, loffset, llength, right, roffset, rlength); - } - - /** - * Generate a faked byte array if possible. It aims to: - * 1)reduce key length, which expects to reduce HFile index memory footprint - * 2)replace TS field with LATEST_TIMESTAMP(to avoid seeking previous block) - * see HBASE-7845 for more details - * we need to ensure: leftKey < newKey <= rightKey - * @param leftKey the previous block's real stop key usually - * @param rightKey the current block's real start key usually - * @return newKey: the newly generated faked key - */ - protected byte[] getShortMidpointKey(final byte[] leftKey, final byte[] rightKey) { - if (rightKey == null) { - throw new IllegalArgumentException("rightKey can not be null"); - } - if (leftKey == null) { - return Arrays.copyOf(rightKey, rightKey.length); - } - if (compare(leftKey, rightKey) >= 0) { - throw new IllegalArgumentException("Unexpected input, leftKey:" + Bytes.toString(leftKey) - + ", rightKey:" + Bytes.toString(rightKey)); - } - - short leftRowLength = Bytes.toShort(leftKey, 0); - short rightRowLength = Bytes.toShort(rightKey, 0); - int leftCommonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + leftRowLength; - int rightCommonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + rightRowLength; - int leftCommonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + leftCommonLength; - int rightCommonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + rightCommonLength; - int leftColumnLength = leftKey.length - leftCommonLengthWithTSAndType; - int rightColumnLength = rightKey.length - rightCommonLengthWithTSAndType; - // rows are equal - if (leftRowLength == rightRowLength && compareRows(leftKey, ROW_LENGTH_SIZE, leftRowLength, - rightKey, ROW_LENGTH_SIZE, rightRowLength) == 0) { - // Compare family & qualifier together. - int comparison = Bytes.compareTo(leftKey, leftCommonLength, leftColumnLength, rightKey, - rightCommonLength, rightColumnLength); - // same with "row + family + qualifier", return rightKey directly - if (comparison == 0) { - return Arrays.copyOf(rightKey, rightKey.length); - } - // "family + qualifier" are different, generate a faked key per rightKey - byte[] newKey = Arrays.copyOf(rightKey, rightKey.length); - Bytes.putLong(newKey, rightKey.length - TIMESTAMP_TYPE_SIZE, HConstants.LATEST_TIMESTAMP); - Bytes.putByte(newKey, rightKey.length - TYPE_SIZE, Type.Maximum.getCode()); - return newKey; - } - // rows are different - short minLength = leftRowLength < rightRowLength ? leftRowLength : rightRowLength; - short diffIdx = 0; - while (diffIdx < minLength - && leftKey[ROW_LENGTH_SIZE + diffIdx] == rightKey[ROW_LENGTH_SIZE + diffIdx]) { - diffIdx++; - } - if (diffIdx >= minLength) { - // leftKey's row is prefix of rightKey's. we can optimize it in future - return Arrays.copyOf(rightKey, rightKey.length); - } - int diffByte = leftKey[ROW_LENGTH_SIZE + diffIdx]; - if ((0xff & diffByte) < 0xff && (diffByte + 1) < - (rightKey[ROW_LENGTH_SIZE + diffIdx] & 0xff)) { - byte[] newRowKey = new byte[diffIdx + 1]; - System.arraycopy(leftKey, ROW_LENGTH_SIZE, newRowKey, 0, diffIdx); - newRowKey[diffIdx] = (byte) (diffByte + 1); - int rightFamilyLength = rightKey[rightCommonLength - 1]; - byte[] family = null; - if (rightFamilyLength > 0) { - family = new byte[rightFamilyLength]; - System.arraycopy(rightKey, rightCommonLength, family, 0, rightFamilyLength); - } - int rightQualifierLength = rightColumnLength - rightFamilyLength; - byte[] qualifier = null; - if (rightQualifierLength > 0) { - qualifier = new byte[rightQualifierLength]; - System.arraycopy(rightKey, rightCommonLength + rightFamilyLength, qualifier, 0, - rightQualifierLength); - } - return new KeyValue(newRowKey, null, null, HConstants.LATEST_TIMESTAMP, - Type.Maximum).getKey(); - } - // the following is optimizable in future - return Arrays.copyOf(rightKey, rightKey.length); - } - - protected int compareColumns( - byte [] left, int loffset, int llength, final int lfamilylength, - byte [] right, int roffset, int rlength, final int rfamilylength) { - return KeyValue.compareColumns(left, loffset, llength, lfamilylength, - right, roffset, rlength, rfamilylength); - } - - static int compareTimestamps(final long ltimestamp, final long rtimestamp) { - // The below older timestamps sorting ahead of newer timestamps looks - // wrong but it is intentional. This way, newer timestamps are first - // found when we iterate over a memstore and newer versions are the - // first we trip over when reading from a store file. - if (ltimestamp < rtimestamp) { - return 1; - } else if (ltimestamp > rtimestamp) { - return -1; - } - return 0; - } - - /** - * Generate a shorter faked key into index block. For example, consider a block boundary - * between the keys "the quick brown fox" and "the who test text". We can use "the r" as the - * key for the index block entry since it is > all entries in the previous block and <= all - * entries in subsequent blocks. - * - * @param lastKeyOfPreviousBlock - * @param firstKeyInBlock - * @return a shortened null key, or if there are unexpected results, the firstKeyIn (new) Block - */ - public byte[] calcIndexKey(byte[] lastKeyOfPreviousBlock, byte[] firstKeyInBlock) { - byte[] fakeKey = getShortMidpointKey(lastKeyOfPreviousBlock, firstKeyInBlock); - if (compare(fakeKey, firstKeyInBlock) > 0) { - LOG.error("Unexpected getShortMidpointKey result, fakeKey:" - + Bytes.toStringBinary(fakeKey) + ", firstKeyInBlock:" - + Bytes.toStringBinary(firstKeyInBlock)); - return firstKeyInBlock; - } - if (lastKeyOfPreviousBlock != null && compare(lastKeyOfPreviousBlock, fakeKey) >= 0) { - LOG.error("Unexpected getShortMidpointKey result, lastKeyOfPreviousBlock:" + - Bytes.toStringBinary(lastKeyOfPreviousBlock) + ", fakeKey:" + - Bytes.toStringBinary(fakeKey)); - return firstKeyInBlock; - } - return fakeKey; - } - } - /** * This is a TEST only Comparator used in TestSeekTo and TestReseekTo. */ - @Deprecated - public static class RawKeyComparator extends KeyComparator { - RawComparator getRawComparator() { return Bytes.BYTES_RAWCOMPARATOR; } - - public int compare(byte[] left, int loffset, int llength, byte[] right, - int roffset, int rlength) { - return getRawComparator().compare(left, loffset, llength, right, roffset, rlength); + public static class RawBytesComparator extends KVComparator { + /** + * The HFileV2 file format's trailer contains this class name. We reinterpret this and + * instantiate the appropriate comparator. + * TODO: With V3 consider removing this. + * @return legacy class name for FileFileTrailer#comparatorClassName + */ + public String getLegacyKeyComparatorName() { + return "org.apache.hadoop.hbase.util.Bytes$ByteArrayComparator"; } - + + public int compareFlatKey(byte[] left, int loffset, int llength, byte[] right, + int roffset, int rlength) { + return Bytes.BYTES_RAWCOMPARATOR.compare(left, loffset, llength, right, roffset, rlength); + } + public byte[] calcIndexKey(byte[] lastKeyOfPreviousBlock, byte[] firstKeyInBlock) { return firstKeyInBlock; } + } /** @@ -2738,6 +2451,8 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return ClassSize.align(sum); } + // ----- + // KV tags stubs @Override public int getTagsOffset() { throw new UnsupportedOperationException("Not implememnted"); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java index 2fbd6ebb80e..28825b1e03b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hbase; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.hbase.KeyValue.KeyComparator; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.util.Bytes; /** @@ -335,10 +335,10 @@ public final class TableName implements Comparable { * * @return The comparator. */ - public KeyComparator getRowComparator() { + public KVComparator getRowComparator() { if(TableName.META_TABLE_NAME.equals(this)) { - return KeyValue.META_COMPARATOR.getRawComparator(); + return KeyValue.META_COMPARATOR; } - return KeyValue.COMPARATOR.getRawComparator(); + return KeyValue.COMPARATOR; } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index af5fb2615d0..3eee7693c19 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -24,12 +24,12 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.WritableUtils; /** @@ -113,14 +113,14 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { BufferedEncodedSeeker implements EncodedSeeker { - protected final RawComparator comparator; + protected final KVComparator comparator; protected final SamePrefixComparator samePrefixComparator; protected ByteBuffer currentBuffer; protected STATE current = createSeekerState(); // always valid protected STATE previous = createSeekerState(); // may not be valid @SuppressWarnings("unchecked") - public BufferedEncodedSeeker(RawComparator comparator) { + public BufferedEncodedSeeker(KVComparator comparator) { this.comparator = comparator; if (comparator instanceof SamePrefixComparator) { this.samePrefixComparator = (SamePrefixComparator) comparator; @@ -207,7 +207,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { comp = samePrefixComparator.compareIgnoringPrefix(commonPrefix, key, offset, length, current.keyBuffer, 0, current.keyLength); } else { - comp = comparator.compare(key, offset, length, + comp = comparator.compareFlatKey(key, offset, length, current.keyBuffer, 0, current.keyLength); } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java index 023d731fc73..bc13465e992 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java @@ -22,9 +22,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.RawComparator; /** * Just copy data, do not do any kind of compression. Use for comparison and @@ -67,7 +67,7 @@ public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder { } @Override - public EncodedSeeker createSeeker(RawComparator comparator, + public EncodedSeeker createSeeker(KVComparator comparator, final boolean includesMemstoreTS) { return new BufferedEncodedSeeker(comparator) { @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java index 61496ec6d78..3f7df0db1f7 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java @@ -22,6 +22,7 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.io.RawComparator; @@ -106,7 +107,7 @@ public interface DataBlockEncoder { * @return A newly created seeker. */ EncodedSeeker createSeeker( - RawComparator comparator, boolean includesMemstoreTS + KVComparator comparator, boolean includesMemstoreTS ); /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java index 4c29120b3eb..e628d58c98f 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.RawComparator; @@ -422,7 +423,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder { } @Override - public EncodedSeeker createSeeker(RawComparator comparator, + public EncodedSeeker createSeeker(KVComparator comparator, final boolean includesMemstoreTS) { return new BufferedEncodedSeeker(comparator) { private byte[] familyNameWithSize; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java index 7c7c4b7660d..175735563b9 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java @@ -24,6 +24,7 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.RawComparator; @@ -417,7 +418,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder { } @Override - public EncodedSeeker createSeeker(RawComparator comparator, + public EncodedSeeker createSeeker(KVComparator comparator, final boolean includesMemstoreTS) { return new BufferedEncodedSeeker(comparator) { private void decode(boolean isFirst) { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java index 590baf59c45..74c09b5ee00 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.RawComparator; @@ -164,7 +165,7 @@ public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder { } @Override - public EncodedSeeker createSeeker(RawComparator comparator, + public EncodedSeeker createSeeker(KVComparator comparator, final boolean includesMemstoreTS) { return new BufferedEncodedSeeker(comparator) { @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java index 383bcfe3a3b..3413143d55e 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java @@ -1575,7 +1575,7 @@ public class Bytes { * ranging from -(N + 1) to N - 1. */ public static int binarySearch(byte [][]arr, byte []key, int offset, - int length, RawComparator comparator) { + int length, RawComparator comparator) { int low = 0; int high = arr.length - 1; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java index 2c0b7cb7d84..39038dfebed 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java @@ -28,7 +28,6 @@ import junit.framework.TestCase; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.KeyValue.KVComparator; -import org.apache.hadoop.hbase.KeyValue.KeyComparator; import org.apache.hadoop.hbase.KeyValue.MetaComparator; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.util.Bytes; @@ -94,24 +93,13 @@ public class TestKeyValue extends TestCase { final byte [] b = Bytes.toBytes("bbb"); final byte [] fam = Bytes.toBytes("col"); final byte [] qf = Bytes.toBytes("umn"); -// final byte [] column = Bytes.toBytes("col:umn"); KeyValue aaa = new KeyValue(a, fam, qf, a); KeyValue bbb = new KeyValue(b, fam, qf, b); - byte [] keyabb = aaa.getKey(); - byte [] keybbb = bbb.getKey(); assertTrue(KeyValue.COMPARATOR.compare(aaa, bbb) < 0); - assertTrue(KeyValue.KEY_COMPARATOR.compare(keyabb, 0, keyabb.length, keybbb, - 0, keybbb.length) < 0); assertTrue(KeyValue.COMPARATOR.compare(bbb, aaa) > 0); - assertTrue(KeyValue.KEY_COMPARATOR.compare(keybbb, 0, keybbb.length, keyabb, - 0, keyabb.length) > 0); // Compare breaks if passed same ByteBuffer as both left and right arguments. assertTrue(KeyValue.COMPARATOR.compare(bbb, bbb) == 0); - assertTrue(KeyValue.KEY_COMPARATOR.compare(keybbb, 0, keybbb.length, keybbb, - 0, keybbb.length) == 0); assertTrue(KeyValue.COMPARATOR.compare(aaa, aaa) == 0); - assertTrue(KeyValue.KEY_COMPARATOR.compare(keyabb, 0, keyabb.length, keyabb, - 0, keyabb.length) == 0); // Do compare with different timestamps. aaa = new KeyValue(a, fam, qf, 1, a); bbb = new KeyValue(a, fam, qf, 2, a); @@ -299,7 +287,7 @@ public class TestKeyValue extends TestCase { assertTrue(cmp > 0); } - private void assertKVLessWithoutRow(KeyValue.KeyComparator c, int common, KeyValue less, + private void assertKVLessWithoutRow(KeyValue.KVComparator c, int common, KeyValue less, KeyValue greater) { int cmp = c.compareIgnoringPrefix(common, less.getBuffer(), less.getOffset() + KeyValue.ROW_OFFSET, less.getKeyLength(), greater.getBuffer(), @@ -312,7 +300,7 @@ public class TestKeyValue extends TestCase { } public void testCompareWithoutRow() { - final KeyValue.KeyComparator c = KeyValue.KEY_COMPARATOR; + final KeyValue.KVComparator c = KeyValue.COMPARATOR; byte[] row = Bytes.toBytes("row"); byte[] fa = Bytes.toBytes("fa"); @@ -462,14 +450,14 @@ public class TestKeyValue extends TestCase { * See HBASE-7845 */ public void testGetShortMidpointKey() { - final KeyComparator keyComparator = new KeyValue.KeyComparator(); + final KVComparator keyComparator = KeyValue.COMPARATOR; //verify that faked shorter rowkey could be generated long ts = 5; KeyValue kv1 = new KeyValue(Bytes.toBytes("the quick brown fox"), family, qualA, ts, Type.Put); KeyValue kv2 = new KeyValue(Bytes.toBytes("the who test text"), family, qualA, ts, Type.Put); byte[] newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey()); - assertTrue(keyComparator.compare(kv1.getKey(), newKey) < 0); - assertTrue(keyComparator.compare(newKey, kv2.getKey()) < 0); + assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0); + assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) < 0); short newRowLength = Bytes.toShort(newKey, 0); byte[] expectedArray = Bytes.toBytes("the r"); Bytes.equals(newKey, KeyValue.ROW_LENGTH_SIZE, newRowLength, expectedArray, 0, @@ -478,44 +466,44 @@ public class TestKeyValue extends TestCase { //verify: same with "row + family + qualifier", return rightKey directly kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put); kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 0, Type.Put); - assertTrue(keyComparator.compare(kv1.getKey(), kv2.getKey()) < 0); + assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0); newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey()); - assertTrue(keyComparator.compare(kv1.getKey(), newKey) < 0); - assertTrue(keyComparator.compare(newKey, kv2.getKey()) == 0); + assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0); + assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) == 0); kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -5, Type.Put); kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, -10, Type.Put); - assertTrue(keyComparator.compare(kv1.getKey(), kv2.getKey()) < 0); + assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0); newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey()); - assertTrue(keyComparator.compare(kv1.getKey(), newKey) < 0); - assertTrue(keyComparator.compare(newKey, kv2.getKey()) == 0); + assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0); + assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) == 0); // verify: same with row, different with qualifier kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, 5, Type.Put); kv2 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualB, 5, Type.Put); - assertTrue(keyComparator.compare(kv1.getKey(), kv2.getKey()) < 0); + assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0); newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey()); - assertTrue(keyComparator.compare(kv1.getKey(), newKey) < 0); - assertTrue(keyComparator.compare(newKey, kv2.getKey()) < 0); + assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0); + assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) < 0); KeyValue newKeyValue = KeyValue.createKeyValueFromKey(newKey); assertTrue(Arrays.equals(newKeyValue.getFamily(),family)); assertTrue(Arrays.equals(newKeyValue.getQualifier(),qualB)); assertTrue(newKeyValue.getTimestamp() == HConstants.LATEST_TIMESTAMP); - assertTrue(newKeyValue.getType() == Type.Maximum.getCode()); + assertTrue(newKeyValue.getTypeByte() == Type.Maximum.getCode()); //verify metaKeyComparator's getShortMidpointKey output - final KeyComparator metaKeyComparator = new KeyValue.MetaKeyComparator(); + final KVComparator metaKeyComparator = KeyValue.META_COMPARATOR; kv1 = new KeyValue(Bytes.toBytes("ilovehbase123"), family, qualA, 5, Type.Put); kv2 = new KeyValue(Bytes.toBytes("ilovehbase234"), family, qualA, 0, Type.Put); newKey = metaKeyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey()); - assertTrue(metaKeyComparator.compare(kv1.getKey(), newKey) < 0); - assertTrue(metaKeyComparator.compare(newKey, kv2.getKey()) == 0); + assertTrue(metaKeyComparator.compareFlatKey(kv1.getKey(), newKey) < 0); + assertTrue(metaKeyComparator.compareFlatKey(newKey, kv2.getKey()) == 0); //verify common fix scenario kv1 = new KeyValue(Bytes.toBytes("ilovehbase"), family, qualA, ts, Type.Put); kv2 = new KeyValue(Bytes.toBytes("ilovehbaseandhdfs"), family, qualA, ts, Type.Put); - assertTrue(keyComparator.compare(kv1.getKey(), kv2.getKey()) < 0); + assertTrue(keyComparator.compareFlatKey(kv1.getKey(), kv2.getKey()) < 0); newKey = keyComparator.getShortMidpointKey(kv1.getKey(), kv2.getKey()); - assertTrue(keyComparator.compare(kv1.getKey(), newKey) < 0); - assertTrue(keyComparator.compare(newKey, kv2.getKey()) == 0); + assertTrue(keyComparator.compareFlatKey(kv1.getKey(), newKey) < 0); + assertTrue(keyComparator.compareFlatKey(newKey, kv2.getKey()) == 0); } } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java index 7ee4bc85ac6..2baab74315e 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java @@ -25,8 +25,9 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KeyComparator; -import org.apache.hadoop.hbase.KeyValue.MetaKeyComparator; +import org.apache.hadoop.hbase.KeyValue.KVComparator; +import org.apache.hadoop.hbase.KeyValue.MetaComparator; +import org.apache.hadoop.hbase.KeyValue.RawBytesComparator; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory; import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher; @@ -189,11 +190,10 @@ public class PrefixTreeCodec implements DataBlockEncoder{ * the way to this point. */ @Override - public EncodedSeeker createSeeker(RawComparator comparator, boolean includesMvccVersion) { - if(! (comparator instanceof KeyComparator)){ + public EncodedSeeker createSeeker(KVComparator comparator, boolean includesMvccVersion) { + if (comparator instanceof RawBytesComparator){ throw new IllegalArgumentException("comparator must be KeyValue.KeyComparator"); - } - if(comparator instanceof MetaKeyComparator){ + } else if (comparator instanceof MetaComparator){ throw new IllegalArgumentException("DataBlockEncoding.PREFIX_TREE not compatible with META " +"table"); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java index 154fb73645a..71324ef5145 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java @@ -160,7 +160,7 @@ public class HalfStoreFileReader extends StoreFile.Reader { // constrain the bottom. if (!top) { ByteBuffer bb = getKey(); - if (getComparator().compare(bb.array(), bb.arrayOffset(), bb.limit(), + if (getComparator().compareFlatKey(bb.array(), bb.arrayOffset(), bb.limit(), splitkey, 0, splitkey.length) >= 0) { atEnd = true; return false; @@ -179,13 +179,13 @@ public class HalfStoreFileReader extends StoreFile.Reader { byte[] fk = getFirstKey(); // This will be null when the file is empty in which we can not seekBefore to any key if (fk == null) return false; - if (getComparator().compare(key, offset, length, fk, 0, + if (getComparator().compareFlatKey(key, offset, length, fk, 0, fk.length) <= 0) { return false; } } else { // The equals sign isn't strictly necessary just here to be consistent with seekTo - if (getComparator().compare(key, offset, length, splitkey, 0, + if (getComparator().compareFlatKey(key, offset, length, splitkey, 0, splitkey.length) >= 0) { return this.delegate.seekBefore(splitkey, 0, splitkey.length); } @@ -216,7 +216,7 @@ public class HalfStoreFileReader extends StoreFile.Reader { // Check key. ByteBuffer k = this.delegate.getKey(); return this.delegate.getReader().getComparator(). - compare(k.array(), k.arrayOffset(), k.limit(), + compareFlatKey(k.array(), k.arrayOffset(), k.limit(), splitkey, 0, splitkey.length) < 0; } @@ -226,12 +226,12 @@ public class HalfStoreFileReader extends StoreFile.Reader { public int seekTo(byte[] key, int offset, int length) throws IOException { if (top) { - if (getComparator().compare(key, offset, length, splitkey, 0, + if (getComparator().compareFlatKey(key, offset, length, splitkey, 0, splitkey.length) < 0) { return -1; } } else { - if (getComparator().compare(key, offset, length, splitkey, 0, + if (getComparator().compareFlatKey(key, offset, length, splitkey, 0, splitkey.length) >= 0) { // we would place the scanner in the second half. // it might be an error to return false here ever... @@ -256,12 +256,12 @@ public class HalfStoreFileReader extends StoreFile.Reader { //This function is identical to the corresponding seekTo function except //that we call reseekTo (and not seekTo) on the delegate. if (top) { - if (getComparator().compare(key, offset, length, splitkey, 0, + if (getComparator().compareFlatKey(key, offset, length, splitkey, 0, splitkey.length) < 0) { return -1; } } else { - if (getComparator().compare(key, offset, length, splitkey, 0, + if (getComparator().compareFlatKey(key, offset, length, splitkey, 0, splitkey.length) >= 0) { // we would place the scanner in the second half. // it might be an error to return false here ever... diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java index 14109e5d4ec..b54b26c8477 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; @@ -71,7 +72,7 @@ public abstract class AbstractHFileReader implements HFile.Reader { protected int avgValueLen = -1; /** Key comparator */ - protected RawComparator comparator; + protected KVComparator comparator; /** Size of this file. */ protected final long fileSize; @@ -206,7 +207,7 @@ public abstract class AbstractHFileReader implements HFile.Reader { /** @return comparator */ @Override - public RawComparator getComparator() { + public KVComparator getComparator() { return comparator; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java index 75e7ffe9456..ea5f0c0c3a0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java @@ -33,7 +33,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KeyComparator; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; import org.apache.hadoop.hbase.util.Bytes; @@ -77,7 +77,7 @@ public abstract class AbstractHFileWriter implements HFile.Writer { protected long totalUncompressedBytes = 0; /** Key comparator. Used to ensure we write in order. */ - protected final KeyComparator comparator; + protected final KVComparator comparator; /** Meta block names. */ protected List metaNames = new ArrayList(); @@ -114,7 +114,7 @@ public abstract class AbstractHFileWriter implements HFile.Writer { FSDataOutputStream outputStream, Path path, int blockSize, Compression.Algorithm compressAlgo, HFileDataBlockEncoder dataBlockEncoder, - KeyComparator comparator) { + KVComparator comparator) { this.outputStream = outputStream; this.path = path; this.name = path != null ? path.getName() : outputStream.toString(); @@ -124,7 +124,7 @@ public abstract class AbstractHFileWriter implements HFile.Writer { this.blockEncoder = dataBlockEncoder != null ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE; this.comparator = comparator != null ? comparator - : KeyValue.KEY_COMPARATOR; + : KeyValue.COMPARATOR; closeOutputStream = path != null; this.cacheConf = cacheConf; @@ -198,8 +198,9 @@ public abstract class AbstractHFileWriter implements HFile.Writer { throw new IOException("Key cannot be null or empty"); } if (lastKeyBuffer != null) { - int keyComp = comparator.compare(lastKeyBuffer, lastKeyOffset, + int keyComp = comparator.compareFlatKey(lastKeyBuffer, lastKeyOffset, lastKeyLength, key, offset, length); + if (keyComp > 0) { throw new IOException("Added a key not lexically larger than" + " previous key=" diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java index 7ea23f3e7c8..38abbe30a2d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java @@ -31,10 +31,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.protobuf.generated.HFileProtos; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.RawComparator; import com.google.common.io.NullOutputStream; @@ -114,8 +114,8 @@ public class FixedFileTrailer { */ private long lastDataBlockOffset; - /** Raw key comparator class name in version 2 */ - private String comparatorClassName = KeyValue.KEY_COMPARATOR.getClass().getName(); + /** Raw key comparator class name in version 3 */ + private String comparatorClassName = KeyValue.COMPARATOR.getLegacyKeyComparatorName(); /** The {@link HFile} format major version. */ private final int majorVersion; @@ -214,6 +214,8 @@ public class FixedFileTrailer { .setNumDataIndexLevels(numDataIndexLevels) .setFirstDataBlockOffset(firstDataBlockOffset) .setLastDataBlockOffset(lastDataBlockOffset) + // TODO this is a classname encoded into an HFile's trailer. We are going to need to have + // some compat code here. .setComparatorClassName(comparatorClassName) .setCompressionCodec(compressionCodec.ordinal()) .build().writeDelimitedTo(baos); @@ -324,6 +326,8 @@ public class FixedFileTrailer { lastDataBlockOffset = builder.getLastDataBlockOffset(); } if (builder.hasComparatorClassName()) { + // TODO this is a classname encoded into an HFile's trailer. We are going to need to have + // some compat code here. setComparatorClass(getComparatorClass(builder.getComparatorClassName())); } if (builder.hasCompressionCodec()) { @@ -351,6 +355,8 @@ public class FixedFileTrailer { numDataIndexLevels = input.readInt(); firstDataBlockOffset = input.readLong(); lastDataBlockOffset = input.readLong(); + // TODO this is a classname encoded into an HFile's trailer. We are going to need to have + // some compat code here. setComparatorClass(getComparatorClass(Bytes.readStringFixedSize(input, MAX_COMPARATOR_NAME_LENGTH))); } @@ -555,30 +561,53 @@ public class FixedFileTrailer { return minorVersion; } - @SuppressWarnings("rawtypes") - public void setComparatorClass(Class klass) { - // Is the comparator instantiable + public void setComparatorClass(Class klass) { + // Is the comparator instantiable? try { - klass.newInstance(); + KVComparator comp = klass.newInstance(); + + // HFile V2 legacy comparator class names. + if (KeyValue.COMPARATOR.getClass().equals(klass)) { + comparatorClassName = KeyValue.COMPARATOR.getLegacyKeyComparatorName(); + } else if (KeyValue.META_COMPARATOR.getClass().equals(klass)) { + comparatorClassName = KeyValue.META_COMPARATOR.getLegacyKeyComparatorName(); + } else if (KeyValue.RAW_COMPARATOR.getClass().equals(klass)) { + comparatorClassName = KeyValue.RAW_COMPARATOR.getLegacyKeyComparatorName(); + } else { + // if the name wasn't one of the legacy names, maybe its a legit new kind of comparator. + comparatorClassName = klass.getName(); + } + } catch (Exception e) { throw new RuntimeException("Comparator class " + klass.getName() + " is not instantiable", e); } - comparatorClassName = klass.getName(); + } @SuppressWarnings("unchecked") - private static Class> getComparatorClass( + private static Class getComparatorClass( String comparatorClassName) throws IOException { try { - return (Class>) + // HFile V2 legacy comparator class names. + if (comparatorClassName.equals(KeyValue.COMPARATOR.getLegacyKeyComparatorName())) { + comparatorClassName = KeyValue.COMPARATOR.getClass().getName(); + } else if (comparatorClassName.equals(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName())) { + comparatorClassName = KeyValue.META_COMPARATOR.getClass().getName(); + } else if (comparatorClassName.equals(KeyValue.RAW_COMPARATOR.getLegacyKeyComparatorName())) { + comparatorClassName = KeyValue.RAW_COMPARATOR.getClass().getName(); + } + + // if the name wasn't one of the legacy names, maybe its a legit new kind of comparator. + + return (Class) Class.forName(comparatorClassName); } catch (ClassNotFoundException ex) { throw new IOException(ex); } } - public static RawComparator createComparator( + public static KVComparator createComparator( String comparatorClassName) throws IOException { try { return getComparatorClass(comparatorClassName).newInstance(); @@ -591,7 +620,7 @@ public class FixedFileTrailer { } } - RawComparator createComparator() throws IOException { + KVComparator createComparator() throws IOException { expectAtLeastMajorVersion(2); return createComparator(comparatorClassName); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 174fdb3d80f..85244ce6a82 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -53,7 +53,7 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KeyComparator; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; @@ -336,7 +336,7 @@ public class HFile { protected Compression.Algorithm compression = HFile.DEFAULT_COMPRESSION_ALGORITHM; protected HFileDataBlockEncoder encoder = NoOpDataBlockEncoder.INSTANCE; - protected KeyComparator comparator = KeyValue.KEY_COMPARATOR; + protected KVComparator comparator = KeyValue.COMPARATOR; protected InetSocketAddress[] favoredNodes; protected ChecksumType checksumType = HFile.DEFAULT_CHECKSUM_TYPE; protected int bytesPerChecksum = DEFAULT_BYTES_PER_CHECKSUM; @@ -384,7 +384,7 @@ public class HFile { return this; } - public WriterFactory withComparator(KeyComparator comparator) { + public WriterFactory withComparator(KVComparator comparator) { Preconditions.checkNotNull(comparator); this.comparator = comparator; return this; @@ -432,7 +432,7 @@ public class HFile { FSDataOutputStream ostream, int blockSize, Compression.Algorithm compress, HFileDataBlockEncoder dataBlockEncoder, - KeyComparator comparator, ChecksumType checksumType, + KVComparator comparator, ChecksumType checksumType, int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException; } @@ -489,7 +489,7 @@ public class HFile { */ String getName(); - RawComparator getComparator(); + KVComparator getComparator(); HFileScanner getScanner(boolean cacheBlocks, final boolean pread, final boolean isCompaction); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java index 19794e78bf6..74bbb2ed887 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java @@ -38,13 +38,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.HFile.CachingBlockReader; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.CompoundBloomFilterWriter; -import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.util.StringUtils; @@ -106,7 +106,7 @@ public class HFileBlockIndex { */ public static class BlockIndexReader implements HeapSize { /** Needed doing lookup on blocks. */ - private final RawComparator comparator; + private final KVComparator comparator; // Root-level data. private byte[][] blockKeys; @@ -132,13 +132,13 @@ public class HFileBlockIndex { /** A way to read {@link HFile} blocks at a given offset */ private CachingBlockReader cachingBlockReader; - public BlockIndexReader(final RawComparator c, final int treeLevel, + public BlockIndexReader(final KVComparator c, final int treeLevel, final CachingBlockReader cachingBlockReader) { this(c, treeLevel); this.cachingBlockReader = cachingBlockReader; } - public BlockIndexReader(final RawComparator c, final int treeLevel) + public BlockIndexReader(final KVComparator c, final int treeLevel) { comparator = c; searchTreeLevel = treeLevel; @@ -481,7 +481,7 @@ public class HFileBlockIndex { */ static int binarySearchNonRootIndex(byte[] key, int keyOffset, int keyLength, ByteBuffer nonRootIndex, - RawComparator comparator) { + KVComparator comparator) { int numEntries = nonRootIndex.getInt(0); int low = 0; @@ -516,7 +516,7 @@ public class HFileBlockIndex { // we have to compare in this order, because the comparator order // has special logic when the 'left side' is a special key. - int cmp = comparator.compare(key, keyOffset, keyLength, + int cmp = comparator.compareFlatKey(key, keyOffset, keyLength, nonRootIndex.array(), nonRootIndex.arrayOffset() + midKeyOffset, midLength); @@ -568,7 +568,7 @@ public class HFileBlockIndex { * */ static int locateNonRootIndexEntry(ByteBuffer nonRootBlock, byte[] key, - int keyOffset, int keyLength, RawComparator comparator) { + int keyOffset, int keyLength, KVComparator comparator) { int entryIndex = binarySearchNonRootIndex(key, keyOffset, keyLength, nonRootBlock, comparator); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java index b3a8a668ac2..f202f9aa2a9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java @@ -120,7 +120,7 @@ public class HFileReaderV2 extends AbstractHFileReader { dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator, trailer.getNumDataIndexLevels(), this); metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader( - Bytes.BYTES_RAWCOMPARATOR, 1); + KeyValue.RAW_COMPARATOR, 1); // Parse load-on-open data. @@ -500,7 +500,7 @@ public class HFileReaderV2 extends AbstractHFileReader { int compared; if (isSeeked()) { ByteBuffer bb = getKey(); - compared = reader.getComparator().compare(key, offset, + compared = reader.getComparator().compareFlatKey(key, offset, length, bb.array(), bb.arrayOffset(), bb.limit()); if (compared < 1) { // If the required key is less than or equal to current key, then @@ -509,7 +509,7 @@ public class HFileReaderV2 extends AbstractHFileReader { } else { if (this.nextIndexedKey != null && (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || - reader.getComparator().compare(key, offset, length, + reader.getComparator().compareFlatKey(key, offset, length, nextIndexedKey, 0, nextIndexedKey.length) < 0)) { // The reader shall continue to scan the current data block instead of querying the // block index as long as it knows the target key is strictly smaller than @@ -535,7 +535,7 @@ public class HFileReaderV2 extends AbstractHFileReader { } ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock); - if (reader.getComparator().compare(firstKey.array(), + if (reader.getComparator().compareFlatKey(firstKey.array(), firstKey.arrayOffset(), firstKey.limit(), key, offset, length) == 0) { long previousBlockOffset = seekToBlock.getPrevBlockOffset(); @@ -851,7 +851,7 @@ public class HFileReaderV2 extends AbstractHFileReader { int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE; - int comp = reader.getComparator().compare(key, offset, length, + int comp = reader.getComparator().compareFlatKey(key, offset, length, blockBuffer.array(), keyOffset, klen); if (comp == 0) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java index 571003ac434..db6dc9c6302 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java @@ -33,7 +33,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KeyComparator; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable; @@ -100,7 +100,7 @@ public class HFileWriterV2 extends AbstractHFileWriter { public Writer createWriter(FileSystem fs, Path path, FSDataOutputStream ostream, int blockSize, Compression.Algorithm compress, HFileDataBlockEncoder blockEncoder, - final KeyComparator comparator, final ChecksumType checksumType, + final KVComparator comparator, final ChecksumType checksumType, final int bytesPerChecksum, boolean includeMVCCReadpoint) throws IOException { return new HFileWriterV2(conf, cacheConf, fs, path, ostream, blockSize, compress, blockEncoder, comparator, checksumType, bytesPerChecksum, includeMVCCReadpoint); @@ -111,7 +111,7 @@ public class HFileWriterV2 extends AbstractHFileWriter { public HFileWriterV2(Configuration conf, CacheConfig cacheConf, FileSystem fs, Path path, FSDataOutputStream ostream, int blockSize, Compression.Algorithm compressAlgo, HFileDataBlockEncoder blockEncoder, - final KeyComparator comparator, final ChecksumType checksumType, + final KVComparator comparator, final ChecksumType checksumType, final int bytesPerChecksum, final boolean includeMVCCReadpoint) throws IOException { super(cacheConf, ostream == null ? createOutputStream(conf, fs, path, null) : ostream, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java index 9c786588e11..09f59700415 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/GetClosestRowBeforeTracker.java @@ -74,7 +74,7 @@ class GetClosestRowBeforeTracker { this.tablenamePlusDelimiterLength = metaregion? l + 1: -1; this.oldestts = System.currentTimeMillis() - ttl; this.kvcomparator = c; - KeyValue.RowComparator rc = new KeyValue.RowComparator(this.kvcomparator); + KeyValue.RowOnlyComparator rc = new KeyValue.RowOnlyComparator(this.kvcomparator); this.deletes = new TreeMap>(rc); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index f9902cc0e44..da7f858fd3d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -529,7 +529,7 @@ public class HRegionFileSystem { byte[] lastKey = f.createReader().getLastKey(); // If lastKey is null means storefile is empty. if (lastKey == null) return null; - if (f.getReader().getComparator().compare(splitKey.getBuffer(), + if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(), splitKey.getKeyOffset(), splitKey.getKeyLength(), lastKey, 0, lastKey.length) > 0) { return null; } @@ -539,7 +539,7 @@ public class HRegionFileSystem { byte[] firstKey = f.createReader().getFirstKey(); // If firstKey is null means storefile is empty. if (firstKey == null) return null; - if (f.getReader().getComparator().compare(splitKey.getBuffer(), + if (f.getReader().getComparator().compareFlatKey(splitKey.getBuffer(), splitKey.getKeyOffset(), splitKey.getKeyLength(), firstKey, 0, firstKey.length) < 0) { return null; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java index 354f5a618a1..ce769c4743e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java @@ -81,7 +81,7 @@ public class ScanQueryMatcher { private final KeyValue startKey; /** Row comparator for the region this query is for */ - private final KeyValue.KeyComparator rowComparator; + private final KeyValue.KVComparator rowComparator; /* row is not private for tests */ /** Row the query is on */ @@ -145,7 +145,7 @@ public class ScanQueryMatcher { NavigableSet columns, ScanType scanType, long readPointToUse, long earliestPutTs, long oldestUnexpiredTS) { this.tr = scan.getTimeRange(); - this.rowComparator = scanInfo.getComparator().getRawComparator(); + this.rowComparator = scanInfo.getComparator(); this.deletes = new ScanDeleteTracker(); this.stopRow = scan.getStopRow(); this.startKey = KeyValue.createFirstDeleteFamilyOnRow(scan.getStartRow(), diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java index 352819e0ff1..b1d92448432 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java @@ -798,7 +798,7 @@ public class StoreFile { .withBlockSize(blocksize) .withCompression(compress) .withDataBlockEncoder(this.dataBlockEncoder) - .withComparator(comparator.getRawComparator()) + .withComparator(comparator) .withChecksumType(checksumType) .withBytesPerChecksum(bytesPerChecksum) .withFavoredNodes(favoredNodes) @@ -877,7 +877,7 @@ public class StoreFile { * @param kv */ public void trackTimestamps(final KeyValue kv) { - if (KeyValue.Type.Put.getCode() == kv.getType()) { + if (KeyValue.Type.Put.getCode() == kv.getTypeByte()) { earliestPutTs = Math.min(earliestPutTs, kv.getTimestamp()); } if (!isTimeRangeTrackerSet) { @@ -939,7 +939,7 @@ public class StoreFile { } generalBloomFilterWriter.add(bloomKey, bloomKeyOffset, bloomKeyLen); if (lastBloomKey != null - && generalBloomFilterWriter.getComparator().compare(bloomKey, + && generalBloomFilterWriter.getComparator().compareFlatKey(bloomKey, bloomKeyOffset, bloomKeyLen, lastBloomKey, lastBloomKeyOffset, lastBloomKeyLen) <= 0) { throw new IOException("Non-increasing Bloom keys: " @@ -1105,7 +1105,7 @@ public class StoreFile { this.reader = null; } - public RawComparator getComparator() { + public KVComparator getComparator() { return reader.getComparator(); } @@ -1333,7 +1333,7 @@ public class StoreFile { // from the file info. For row-column Bloom filters this is not yet // a sufficient condition to return false. boolean keyIsAfterLast = lastBloomKey != null - && bloomFilter.getComparator().compare(key, lastBloomKey) > 0; + && bloomFilter.getComparator().compareFlatKey(key, lastBloomKey) > 0; if (bloomFilterType == BloomType.ROWCOL) { // Since a Row Delete is essentially a DeleteFamily applied to all @@ -1344,7 +1344,7 @@ public class StoreFile { null, 0, 0); if (keyIsAfterLast - && bloomFilter.getComparator().compare(rowBloomKey, + && bloomFilter.getComparator().compareFlatKey(rowBloomKey, lastBloomKey) > 0) { exists = false; } else { @@ -1388,9 +1388,9 @@ public class StoreFile { } KeyValue startKeyValue = KeyValue.createFirstOnRow(scan.getStartRow()); KeyValue stopKeyValue = KeyValue.createLastOnRow(scan.getStopRow()); - boolean nonOverLapping = (getComparator().compare(this.getFirstKey(), + boolean nonOverLapping = (getComparator().compareFlatKey(this.getFirstKey(), stopKeyValue.getKey()) > 0 && !Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) - || getComparator().compare(this.getLastKey(), startKeyValue.getKey()) < 0; + || getComparator().compareFlatKey(this.getLastKey(), startKeyValue.getKey()) < 0; return !nonOverLapping; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterBase.java index 57512d7d10b..7d00d13f701 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterBase.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hbase.util; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.RawComparator; +import org.apache.hadoop.hbase.KeyValue.KVComparator; /** * Common methods Bloom filter methods required at read and write time. @@ -52,6 +52,6 @@ public interface BloomFilterBase { /** * @return Bloom key comparator */ - RawComparator getComparator(); + KVComparator getComparator(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java index c35cb0f3923..92eda4280f6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java @@ -200,7 +200,7 @@ public final class BloomFilterFactory { // In case of compound Bloom filters we ignore the maxKeys hint. CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter(getBloomBlockSize(conf), err, Hash.getHashType(conf), maxFold, cacheConf.shouldCacheBloomsOnWrite(), - bloomType == BloomType.ROWCOL ? KeyValue.KEY_COMPARATOR : Bytes.BYTES_RAWCOMPARATOR); + bloomType == BloomType.ROWCOL ? KeyValue.COMPARATOR : KeyValue.RAW_COMPARATOR); writer.addInlineBlockWriter(bloomWriter); return bloomWriter; } @@ -231,7 +231,7 @@ public final class BloomFilterFactory { // In case of compound Bloom filters we ignore the maxKeys hint. CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter(getBloomBlockSize(conf), err, Hash.getHashType(conf), maxFold, cacheConf.shouldCacheBloomsOnWrite(), - Bytes.BYTES_RAWCOMPARATOR); + KeyValue.RAW_COMPARATOR); writer.addInlineBlockWriter(bloomWriter); return bloomWriter; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java index c393cce08ff..edc9168b3d0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java @@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.util; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.Writable; @@ -625,8 +627,9 @@ public class ByteBloomFilter implements BloomFilter, BloomFilterWriter { } @Override - public RawComparator getComparator() { - return Bytes.BYTES_RAWCOMPARATOR; + public KVComparator getComparator() { +// return Bytes.BYTES_RAWCOMPARATOR; + return KeyValue.RAW_COMPARATOR; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java index d78969d264b..911738a061f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java @@ -24,12 +24,12 @@ import java.io.IOException; import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.io.hfile.FixedFileTrailer; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileBlock; import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex; -import org.apache.hadoop.io.RawComparator; /** * A Bloom filter implementation built on top of {@link ByteBloomFilter}, @@ -131,7 +131,7 @@ public class CompoundBloomFilter extends CompoundBloomFilterBase } @Override - public RawComparator getComparator() { + public KVComparator getComparator() { return comparator; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterBase.java index e661321f96d..f1fdabff42f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterBase.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.util; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.io.RawComparator; @InterfaceAudience.Private @@ -51,7 +52,7 @@ public class CompoundBloomFilterBase implements BloomFilterBase { protected int hashType; /** Comparator used to compare Bloom filter keys */ - protected RawComparator comparator; + protected KVComparator comparator; @Override public long getMaxKeys() { @@ -89,7 +90,7 @@ public class CompoundBloomFilterBase implements BloomFilterBase { } @Override - public RawComparator getComparator() { + public KVComparator getComparator() { return comparator; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterWriter.java index dc02685864c..f6709bdd182 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilterWriter.java @@ -28,10 +28,10 @@ import java.util.Queue; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex; import org.apache.hadoop.hbase.io.hfile.InlineBlockWriter; -import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.Writable; /** @@ -89,7 +89,7 @@ public class CompoundBloomFilterWriter extends CompoundBloomFilterBase */ public CompoundBloomFilterWriter(int chunkByteSizeHint, float errorRate, int hashType, int maxFold, boolean cacheOnWrite, - RawComparator comparator) { + KVComparator comparator) { chunkByteSize = ByteBloomFilter.computeFoldableByteSize( chunkByteSizeHint * 8L, maxFold); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java index fa9b6274ff5..db9c9192f2a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java @@ -185,7 +185,7 @@ public class TestDataBlockEncoders { ByteBuffer.wrap(encodeBytes(encoding, originalBuffer)); DataBlockEncoder encoder = encoding.getEncoder(); DataBlockEncoder.EncodedSeeker seeker = - encoder.createSeeker(KeyValue.KEY_COMPARATOR, includesMemstoreTS); + encoder.createSeeker(KeyValue.COMPARATOR, includesMemstoreTS); seeker.setCurrentBuffer(encodedBuffer); encodedSeekers.add(seeker); } @@ -240,7 +240,7 @@ public class TestDataBlockEncoders { "Bug while encoding using '%s'", encoder.toString()), e); } DataBlockEncoder.EncodedSeeker seeker = - encoder.createSeeker(KeyValue.KEY_COMPARATOR, includesMemstoreTS); + encoder.createSeeker(KeyValue.COMPARATOR, includesMemstoreTS); seeker.setCurrentBuffer(encodedBuffer); int i = 0; do { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTreeEncoding.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTreeEncoding.java index 40b365f5b8e..b53596948d3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTreeEncoding.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTreeEncoding.java @@ -77,7 +77,7 @@ public class TestPrefixTreeEncoding { HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( Algorithm.NONE, DataBlockEncoding.PREFIX_TREE, new byte[0]); encoder.encodeKeyValues(dataBuffer, false, blkEncodingCtx); - EncodedSeeker seeker = encoder.createSeeker(KeyValue.KEY_COMPARATOR, false); + EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, false); byte[] onDiskBytes = blkEncodingCtx.getOnDiskBytesWithHeader(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length @@ -117,7 +117,7 @@ public class TestPrefixTreeEncoding { HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( Algorithm.NONE, DataBlockEncoding.PREFIX_TREE, new byte[0]); encoder.encodeKeyValues(dataBuffer, false, blkEncodingCtx); - EncodedSeeker seeker = encoder.createSeeker(KeyValue.KEY_COMPARATOR, false); + EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, false); byte[] onDiskBytes=blkEncodingCtx.getOnDiskBytesWithHeader(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length @@ -143,7 +143,7 @@ public class TestPrefixTreeEncoding { HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( Algorithm.NONE, DataBlockEncoding.PREFIX_TREE, new byte[0]); encoder.encodeKeyValues(dataBuffer, false, blkEncodingCtx); - EncodedSeeker seeker = encoder.createSeeker(KeyValue.KEY_COMPARATOR, false); + EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, false); byte[] onDiskBytes = blkEncodingCtx.getOnDiskBytesWithHeader(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, DataBlockEncoding.ID_SIZE, onDiskBytes.length @@ -159,7 +159,7 @@ public class TestPrefixTreeEncoding { HFileBlockEncodingContext blkEncodingCtx = new HFileBlockDefaultEncodingContext( Algorithm.NONE, DataBlockEncoding.PREFIX_TREE, new byte[0]); encoder.encodeKeyValues(dataBuffer, false, blkEncodingCtx); - EncodedSeeker seeker = encoder.createSeeker(KeyValue.KEY_COMPARATOR, + EncodedSeeker seeker = encoder.createSeeker(KeyValue.COMPARATOR, false); byte[] onDiskBytes = blkEncodingCtx.getOnDiskBytesWithHeader(); ByteBuffer readBuffer = ByteBuffer.wrap(onDiskBytes, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java index ac8e956e4bf..65bd9b48a8f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java @@ -92,7 +92,7 @@ public class TestFixedFileTrailer { t.setLastDataBlockOffset(291); t.setNumDataIndexLevels(3); - t.setComparatorClass(KeyValue.KEY_COMPARATOR.getClass()); + t.setComparatorClass(KeyValue.COMPARATOR.getClass()); t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic. t.setUncompressedDataIndexSize(827398717L); // Something random. @@ -209,7 +209,7 @@ public class TestFixedFileTrailer { assertEquals(expected.getFirstDataBlockOffset(), loaded.getFirstDataBlockOffset()); assertTrue( - expected.createComparator() instanceof KeyValue.KeyComparator); + expected.createComparator() instanceof KeyValue.KVComparator); assertEquals(expected.getUncompressedDataIndexSize(), loaded.getUncompressedDataIndexSize()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index f0f0aac5a0f..e229a141b82 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -35,7 +35,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestCase; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KeyComparator; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.hfile.HFile.Reader; @@ -216,7 +215,7 @@ public class TestHFile extends HBaseTestCase { .withBlockSize(minBlockSize) .withCompression(codec) // NOTE: This test is dependent on this deprecated nonstandard comparator - .withComparator(new KeyValue.RawKeyComparator()) + .withComparator(new KeyValue.RawBytesComparator()) .create(); LOG.info(writer); writeRecords(writer); @@ -350,36 +349,5 @@ public class TestHFile extends HBaseTestCase { assertTrue(Compression.Algorithm.LZ4.ordinal() == 4); } - // This can't be an anonymous class because the compiler will not generate - // a nullary constructor for it. - static class CustomKeyComparator extends KeyComparator { - @Override - public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, - int l2) { - return -Bytes.compareTo(b1, s1, l1, b2, s2, l2); - } - @Override - public int compare(byte[] o1, byte[] o2) { - return compare(o1, 0, o1.length, o2, 0, o2.length); - } - } - - public void testComparator() throws IOException { - if (cacheConf == null) cacheConf = new CacheConfig(conf); - Path mFile = new Path(ROOT_DIR, "meta.tfile"); - FSDataOutputStream fout = createFSOutput(mFile); - KeyComparator comparator = new CustomKeyComparator(); - Writer writer = HFile.getWriterFactory(conf, cacheConf) - .withOutputStream(fout) - .withBlockSize(minBlockSize) - .withComparator(comparator) - .create(); - writer.append("3".getBytes(), "0".getBytes()); - writer.append("2".getBytes(), "0".getBytes()); - writer.append("1".getBytes(), "0".getBytes()); - writer.close(); - } - - } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java index f578ef95bfa..9fc9e9c4d03 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java @@ -42,11 +42,13 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.compress.Compression; -import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader; import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexChunk; +import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.junit.Before; @@ -173,7 +175,7 @@ public class TestHFileBlockIndex { BlockReaderWrapper brw = new BlockReaderWrapper(blockReader); HFileBlockIndex.BlockIndexReader indexReader = new HFileBlockIndex.BlockIndexReader( - Bytes.BYTES_RAWCOMPARATOR, numLevels, brw); + KeyValue.RAW_COMPARATOR, numLevels, brw); indexReader.readRootIndex(blockReader.blockRange(rootIndexOffset, fileSize).nextBlockWithBlockType(BlockType.ROOT_INDEX), numRootEntries); @@ -355,7 +357,7 @@ public class TestHFileBlockIndex { int searchResult = BlockIndexReader.binarySearchNonRootIndex( arrayHoldingKey, searchKey.length / 2, searchKey.length, nonRootIndex, - Bytes.BYTES_RAWCOMPARATOR); + KeyValue.RAW_COMPARATOR); String lookupFailureMsg = "Failed to look up key #" + i + " (" + Bytes.toStringBinary(searchKey) + ")"; @@ -381,7 +383,7 @@ public class TestHFileBlockIndex { // higher-level API function.s boolean locateBlockResult = (BlockIndexReader.locateNonRootIndexEntry(nonRootIndex, arrayHoldingKey, - searchKey.length / 2, searchKey.length, Bytes.BYTES_RAWCOMPARATOR) != -1); + searchKey.length / 2, searchKey.length, KeyValue.RAW_COMPARATOR) != -1); if (i == 0) { assertFalse(locateBlockResult); @@ -441,7 +443,7 @@ public class TestHFileBlockIndex { long expected = ClassSize.estimateBase(cl, false); HFileBlockIndex.BlockIndexReader bi = - new HFileBlockIndex.BlockIndexReader(Bytes.BYTES_RAWCOMPARATOR, 1); + new HFileBlockIndex.BlockIndexReader(KeyValue.RAW_COMPARATOR, 1); long actual = bi.heapSize(); // Since the arrays in BlockIndex(byte [][] blockKeys, long [] blockOffsets, @@ -506,7 +508,7 @@ public class TestHFileBlockIndex { keyStrSet.add(Bytes.toStringBinary(k)); if (i > 0) { - assertTrue(KeyValue.KEY_COMPARATOR.compare(keys[i - 1], + assertTrue(KeyValue.COMPARATOR.compareFlatKey(keys[i - 1], keys[i]) < 0); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java index 8657b30a6a8..4276b2a101b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java @@ -166,7 +166,7 @@ public class TestHFilePerformance extends TestCase { .withOutputStream(fout) .withBlockSize(minBlockSize) .withCompression(codecName) - .withComparator(new KeyValue.RawKeyComparator()) + .withComparator(new KeyValue.RawBytesComparator()) .create(); // Writing value in one shot. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java index 9022719a847..57971220b74 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java @@ -131,7 +131,7 @@ public class TestHFileSeek extends TestCase { .withOutputStream(fout) .withBlockSize(options.minBlockSize) .withCompression(options.compress) - .withComparator(new KeyValue.RawKeyComparator()) + .withComparator(new KeyValue.RawBytesComparator()) .create(); try { BytesWritable key = new BytesWritable(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java index 5a45f6240bc..5beca602471 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java @@ -39,13 +39,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Writables; -import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableUtils; import org.junit.Before; @@ -137,13 +137,13 @@ public class TestHFileWriterV2 { HFileBlock.FSReader blockReader = new HFileBlock.FSReaderV2(fsdis, compressAlgo, fileSize); // Comparator class name is stored in the trailer in version 2. - RawComparator comparator = trailer.createComparator(); + KVComparator comparator = trailer.createComparator(); HFileBlockIndex.BlockIndexReader dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator, trailer.getNumDataIndexLevels()); HFileBlockIndex.BlockIndexReader metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader( - Bytes.BYTES_RAWCOMPARATOR, 1); + KeyValue.RAW_COMPARATOR, 1); HFileBlock.BlockIterator blockIter = blockReader.blockRange( trailer.getLoadOnOpenDataOffset(), diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java index 7f6a9ba2ab4..4505c7bfd57 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java @@ -51,7 +51,7 @@ public class TestReseekTo { .withOutputStream(fout) .withBlockSize(4000) // NOTE: This test is dependent on this deprecated nonstandard comparator - .withComparator(new KeyValue.RawKeyComparator()) + .withComparator(new KeyValue.RawBytesComparator()) .create(); int numberOfKeys = 1000; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java index 2c8f7fced91..4ea9d59ea4f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java @@ -23,7 +23,6 @@ import java.io.IOException; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.KeyValue.KeyComparator; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.RawComparator; import org.junit.experimental.categories.Category; @@ -51,7 +50,7 @@ public class TestSeekTo extends HBaseTestCase { .withOutputStream(fout) .withBlockSize(blocksize) // NOTE: This test is dependent on this deprecated nonstandard comparator - .withComparator(new KeyValue.RawKeyComparator()) + .withComparator(KeyValue.RAW_COMPARATOR) .create(); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java index 3a4db1e8be9..e5fffd9a66b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java @@ -536,9 +536,10 @@ public class TestMemStore extends TestCase { List results = new ArrayList(); for (int i = 0; scanner.next(results); i++) { int rowId = startRowId + i; + KeyValue left = results.get(0); + byte[] row1 = Bytes.toBytes(rowId); assertTrue("Row name", - KeyValue.COMPARATOR.compareRows(results.get(0), - Bytes.toBytes(rowId)) == 0); + KeyValue.COMPARATOR.compareRows(left.getBuffer(), left.getRowOffset(), (int) left.getRowLength(), row1, 0, row1.length) == 0); assertEquals("Count of columns", QUALIFIER_COUNT, results.size()); List row = new ArrayList(); for (KeyValue kv : results) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java index d09ad8bf7be..f8613f9810d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java @@ -327,7 +327,7 @@ public class TestStoreFile extends HBaseTestCase { (topScanner.isSeeked() && topScanner.next())) { key = topScanner.getKey(); - if (topScanner.getReader().getComparator().compare(key.array(), + if (topScanner.getReader().getComparator().compareFlatKey(key.array(), key.arrayOffset(), key.limit(), midkey, 0, midkey.length) < 0) { fail("key=" + Bytes.toStringBinary(key) + " < midkey=" + Bytes.toStringBinary(midkey)); @@ -377,7 +377,7 @@ public class TestStoreFile extends HBaseTestCase { while ((!topScanner.isSeeked() && topScanner.seekTo()) || topScanner.next()) { key = topScanner.getKey(); - assertTrue(topScanner.getReader().getComparator().compare(key.array(), + assertTrue(topScanner.getReader().getComparator().compareFlatKey(key.array(), key.arrayOffset(), key.limit(), badmidkey, 0, badmidkey.length) >= 0); if (first) { first = false;