HBASE-17831 Support small scan in thrift2 (Guangxu Cheng)

This commit is contained in:
tedyu 2017-03-27 10:19:46 -07:00
parent c77e2135db
commit 85fda44179
5 changed files with 352 additions and 9 deletions

View File

@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.RowMutations;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Scan.ReadType;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.ParseFilter;
import org.apache.hadoop.hbase.security.visibility.Authorizations;
@ -62,6 +63,7 @@ import org.apache.hadoop.hbase.thrift2.generated.THRegionLocation;
import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
import org.apache.hadoop.hbase.thrift2.generated.TMutation;
import org.apache.hadoop.hbase.thrift2.generated.TPut;
import org.apache.hadoop.hbase.thrift2.generated.TReadType;
import org.apache.hadoop.hbase.thrift2.generated.TResult;
import org.apache.hadoop.hbase.thrift2.generated.TRowMutations;
import org.apache.hadoop.hbase.thrift2.generated.TScan;
@ -445,6 +447,14 @@ public class ThriftUtilities {
}
}
if (in.isSetReadType()) {
out.setReadType(readTypeFromThrift(in.getReadType()));
}
if (in.isSetLimit()) {
out.setLimit(in.getLimit());
}
return out;
}
@ -560,4 +570,13 @@ public class ThriftUtilities {
default: return null;
}
}
private static ReadType readTypeFromThrift(TReadType tReadType) {
switch (tReadType.getValue()) {
case 1: return ReadType.DEFAULT;
case 2: return ReadType.STREAM;
case 3: return ReadType.PREAD;
default: return null;
}
}
}

View File

@ -0,0 +1,48 @@
/**
* Autogenerated by Thrift Compiler (0.9.3)
*
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated
*/
package org.apache.hadoop.hbase.thrift2.generated;
import java.util.Map;
import java.util.HashMap;
import org.apache.thrift.TEnum;
public enum TReadType implements org.apache.thrift.TEnum {
DEFAULT(1),
STREAM(2),
PREAD(3);
private final int value;
private TReadType(int value) {
this.value = value;
}
/**
* Get the integer value of this enum value, as defined in the Thrift IDL.
*/
public int getValue() {
return value;
}
/**
* Find a the enum type by its integer value, as defined in the Thrift IDL.
* @return null if the value is not found.
*/
public static TReadType findByValue(int value) {
switch (value) {
case 1:
return DEFAULT;
case 2:
return STREAM;
case 3:
return PREAD;
default:
return null;
}
}
}

View File

@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory;
* Any timestamps in the columns are ignored but the colFamTimeRangeMap included, use timeRange to select by timestamp.
* Max versions defaults to 1.
*/
@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2017-03-06")
@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2017-03-27")
public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, java.io.Serializable, Cloneable, Comparable<TScan> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TScan");
@ -55,6 +55,8 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
private static final org.apache.thrift.protocol.TField REVERSED_FIELD_DESC = new org.apache.thrift.protocol.TField("reversed", org.apache.thrift.protocol.TType.BOOL, (short)11);
private static final org.apache.thrift.protocol.TField CACHE_BLOCKS_FIELD_DESC = new org.apache.thrift.protocol.TField("cacheBlocks", org.apache.thrift.protocol.TType.BOOL, (short)12);
private static final org.apache.thrift.protocol.TField COL_FAM_TIME_RANGE_MAP_FIELD_DESC = new org.apache.thrift.protocol.TField("colFamTimeRangeMap", org.apache.thrift.protocol.TType.MAP, (short)13);
private static final org.apache.thrift.protocol.TField READ_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("readType", org.apache.thrift.protocol.TType.I32, (short)14);
private static final org.apache.thrift.protocol.TField LIMIT_FIELD_DESC = new org.apache.thrift.protocol.TField("limit", org.apache.thrift.protocol.TType.I32, (short)15);
private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
static {
@ -75,6 +77,12 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
public boolean reversed; // optional
public boolean cacheBlocks; // optional
public Map<ByteBuffer,TTimeRange> colFamTimeRangeMap; // optional
/**
*
* @see TReadType
*/
public TReadType readType; // optional
public int limit; // optional
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@ -90,7 +98,13 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
AUTHORIZATIONS((short)10, "authorizations"),
REVERSED((short)11, "reversed"),
CACHE_BLOCKS((short)12, "cacheBlocks"),
COL_FAM_TIME_RANGE_MAP((short)13, "colFamTimeRangeMap");
COL_FAM_TIME_RANGE_MAP((short)13, "colFamTimeRangeMap"),
/**
*
* @see TReadType
*/
READ_TYPE((short)14, "readType"),
LIMIT((short)15, "limit");
private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
@ -131,6 +145,10 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
return CACHE_BLOCKS;
case 13: // COL_FAM_TIME_RANGE_MAP
return COL_FAM_TIME_RANGE_MAP;
case 14: // READ_TYPE
return READ_TYPE;
case 15: // LIMIT
return LIMIT;
default:
return null;
}
@ -176,8 +194,9 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
private static final int __BATCHSIZE_ISSET_ID = 2;
private static final int __REVERSED_ISSET_ID = 3;
private static final int __CACHEBLOCKS_ISSET_ID = 4;
private static final int __LIMIT_ISSET_ID = 5;
private byte __isset_bitfield = 0;
private static final _Fields optionals[] = {_Fields.START_ROW,_Fields.STOP_ROW,_Fields.COLUMNS,_Fields.CACHING,_Fields.MAX_VERSIONS,_Fields.TIME_RANGE,_Fields.FILTER_STRING,_Fields.BATCH_SIZE,_Fields.ATTRIBUTES,_Fields.AUTHORIZATIONS,_Fields.REVERSED,_Fields.CACHE_BLOCKS,_Fields.COL_FAM_TIME_RANGE_MAP};
private static final _Fields optionals[] = {_Fields.START_ROW,_Fields.STOP_ROW,_Fields.COLUMNS,_Fields.CACHING,_Fields.MAX_VERSIONS,_Fields.TIME_RANGE,_Fields.FILTER_STRING,_Fields.BATCH_SIZE,_Fields.ATTRIBUTES,_Fields.AUTHORIZATIONS,_Fields.REVERSED,_Fields.CACHE_BLOCKS,_Fields.COL_FAM_TIME_RANGE_MAP,_Fields.READ_TYPE,_Fields.LIMIT};
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@ -208,10 +227,14 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
tmpMap.put(_Fields.CACHE_BLOCKS, new org.apache.thrift.meta_data.FieldMetaData("cacheBlocks", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
tmpMap.put(_Fields.COL_FAM_TIME_RANGE_MAP, new org.apache.thrift.meta_data.FieldMetaData("colFamTimeRangeMap", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true),
tmpMap.put(_Fields.COL_FAM_TIME_RANGE_MAP, new org.apache.thrift.meta_data.FieldMetaData("colFamTimeRangeMap", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true),
new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTimeRange.class))));
tmpMap.put(_Fields.READ_TYPE, new org.apache.thrift.meta_data.FieldMetaData("readType", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TReadType.class)));
tmpMap.put(_Fields.LIMIT, new org.apache.thrift.meta_data.FieldMetaData("limit", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TScan.class, metaDataMap);
}
@ -272,6 +295,10 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
}
this.colFamTimeRangeMap = __this__colFamTimeRangeMap;
}
if (other.isSetReadType()) {
this.readType = other.readType;
}
this.limit = other.limit;
}
public TScan deepCopy() {
@ -298,6 +325,9 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
setCacheBlocksIsSet(false);
this.cacheBlocks = false;
this.colFamTimeRangeMap = null;
this.readType = null;
setLimitIsSet(false);
this.limit = 0;
}
public byte[] getStartRow() {
@ -674,6 +704,61 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
}
}
/**
*
* @see TReadType
*/
public TReadType getReadType() {
return this.readType;
}
/**
*
* @see TReadType
*/
public TScan setReadType(TReadType readType) {
this.readType = readType;
return this;
}
public void unsetReadType() {
this.readType = null;
}
/** Returns true if field readType is set (has been assigned a value) and false otherwise */
public boolean isSetReadType() {
return this.readType != null;
}
public void setReadTypeIsSet(boolean value) {
if (!value) {
this.readType = null;
}
}
public int getLimit() {
return this.limit;
}
public TScan setLimit(int limit) {
this.limit = limit;
setLimitIsSet(true);
return this;
}
public void unsetLimit() {
__isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __LIMIT_ISSET_ID);
}
/** Returns true if field limit is set (has been assigned a value) and false otherwise */
public boolean isSetLimit() {
return EncodingUtils.testBit(__isset_bitfield, __LIMIT_ISSET_ID);
}
public void setLimitIsSet(boolean value) {
__isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __LIMIT_ISSET_ID, value);
}
public void setFieldValue(_Fields field, Object value) {
switch (field) {
case START_ROW:
@ -780,6 +865,22 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
}
break;
case READ_TYPE:
if (value == null) {
unsetReadType();
} else {
setReadType((TReadType)value);
}
break;
case LIMIT:
if (value == null) {
unsetLimit();
} else {
setLimit((Integer)value);
}
break;
}
}
@ -824,6 +925,12 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
case COL_FAM_TIME_RANGE_MAP:
return getColFamTimeRangeMap();
case READ_TYPE:
return getReadType();
case LIMIT:
return getLimit();
}
throw new IllegalStateException();
}
@ -861,6 +968,10 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
return isSetCacheBlocks();
case COL_FAM_TIME_RANGE_MAP:
return isSetColFamTimeRangeMap();
case READ_TYPE:
return isSetReadType();
case LIMIT:
return isSetLimit();
}
throw new IllegalStateException();
}
@ -995,6 +1106,24 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
return false;
}
boolean this_present_readType = true && this.isSetReadType();
boolean that_present_readType = true && that.isSetReadType();
if (this_present_readType || that_present_readType) {
if (!(this_present_readType && that_present_readType))
return false;
if (!this.readType.equals(that.readType))
return false;
}
boolean this_present_limit = true && this.isSetLimit();
boolean that_present_limit = true && that.isSetLimit();
if (this_present_limit || that_present_limit) {
if (!(this_present_limit && that_present_limit))
return false;
if (this.limit != that.limit)
return false;
}
return true;
}
@ -1067,6 +1196,16 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
if (present_colFamTimeRangeMap)
list.add(colFamTimeRangeMap);
boolean present_readType = true && (isSetReadType());
list.add(present_readType);
if (present_readType)
list.add(readType.getValue());
boolean present_limit = true && (isSetLimit());
list.add(present_limit);
if (present_limit)
list.add(limit);
return list.hashCode();
}
@ -1208,6 +1347,26 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetReadType()).compareTo(other.isSetReadType());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetReadType()) {
lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.readType, other.readType);
if (lastComparison != 0) {
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetLimit()).compareTo(other.isSetLimit());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetLimit()) {
lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.limit, other.limit);
if (lastComparison != 0) {
return lastComparison;
}
}
return 0;
}
@ -1337,6 +1496,22 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
}
first = false;
}
if (isSetReadType()) {
if (!first) sb.append(", ");
sb.append("readType:");
if (this.readType == null) {
sb.append("null");
} else {
sb.append(this.readType);
}
first = false;
}
if (isSetLimit()) {
if (!first) sb.append(", ");
sb.append("limit:");
sb.append(this.limit);
first = false;
}
sb.append(")");
return sb.toString();
}
@ -1526,7 +1701,23 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
iprot.readMapEnd();
}
struct.setColFamTimeRangeMapIsSet(true);
} else {
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 14: // READ_TYPE
if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
struct.readType = org.apache.hadoop.hbase.thrift2.generated.TReadType.findByValue(iprot.readI32());
struct.setReadTypeIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 15: // LIMIT
if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
struct.limit = iprot.readI32();
struct.setLimitIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
@ -1649,6 +1840,18 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
oprot.writeFieldEnd();
}
}
if (struct.readType != null) {
if (struct.isSetReadType()) {
oprot.writeFieldBegin(READ_TYPE_FIELD_DESC);
oprot.writeI32(struct.readType.getValue());
oprot.writeFieldEnd();
}
}
if (struct.isSetLimit()) {
oprot.writeFieldBegin(LIMIT_FIELD_DESC);
oprot.writeI32(struct.limit);
oprot.writeFieldEnd();
}
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@ -1706,7 +1909,13 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
if (struct.isSetColFamTimeRangeMap()) {
optionals.set(12);
}
oprot.writeBitSet(optionals, 13);
if (struct.isSetReadType()) {
optionals.set(13);
}
if (struct.isSetLimit()) {
optionals.set(14);
}
oprot.writeBitSet(optionals, 15);
if (struct.isSetStartRow()) {
oprot.writeBinary(struct.startRow);
}
@ -1766,12 +1975,18 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
}
}
}
if (struct.isSetReadType()) {
oprot.writeI32(struct.readType.getValue());
}
if (struct.isSetLimit()) {
oprot.writeI32(struct.limit);
}
}
@Override
public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
BitSet incoming = iprot.readBitSet(13);
BitSet incoming = iprot.readBitSet(15);
if (incoming.get(0)) {
struct.startRow = iprot.readBinary();
struct.setStartRowIsSet(true);
@ -1859,6 +2074,14 @@ public class TScan implements org.apache.thrift.TBase<TScan, TScan._Fields>, jav
}
struct.setColFamTimeRangeMapIsSet(true);
}
if (incoming.get(13)) {
struct.readType = org.apache.hadoop.hbase.thrift2.generated.TReadType.findByValue(iprot.readI32());
struct.setReadTypeIsSet(true);
}
if (incoming.get(14)) {
struct.limit = iprot.readI32();
struct.setLimitIsSet(true);
}
}
}

View File

@ -208,6 +208,12 @@ struct TAppend {
5: optional TCellVisibility cellVisibility
}
enum TReadType {
DEFAULT = 1,
STREAM = 2,
PREAD = 3
}
/**
* Any timestamps in the columns are ignored but the colFamTimeRangeMap included, use timeRange to select by timestamp.
* Max versions defaults to 1.
@ -226,6 +232,8 @@ struct TScan {
11: optional bool reversed
12: optional bool cacheBlocks
13: optional map<binary,TTimeRange> colFamTimeRangeMap
14: optional TReadType readType
15: optional i32 limit
}
/**

View File

@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.thrift2.generated.TIOError;
import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
import org.apache.hadoop.hbase.thrift2.generated.TPut;
import org.apache.hadoop.hbase.thrift2.generated.TReadType;
import org.apache.hadoop.hbase.thrift2.generated.TResult;
import org.apache.hadoop.hbase.thrift2.generated.TScan;
import org.apache.hadoop.hbase.thrift2.generated.TMutation;
@ -850,6 +851,50 @@ public class TestThriftHBaseServiceHandler {
}
}
@Test
public void testSmallScan() throws Exception {
ThriftHBaseServiceHandler handler = createHandler();
ByteBuffer table = wrap(tableAname);
// insert data
TColumnValue columnValue = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
wrap(valueAname));
List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
columnValues.add(columnValue);
for (int i = 0; i < 10; i++) {
TPut put = new TPut(wrap(("testSmallScan" + i).getBytes()), columnValues);
handler.put(table, put);
}
// small scan instance
TScan scan = new TScan();
scan.setStartRow("testSmallScan".getBytes());
scan.setStopRow("testSmallScan\uffff".getBytes());
scan.setReadType(TReadType.PREAD);
scan.setCaching(2);
// get scanner and rows
int scanId = handler.openScanner(table, scan);
List<TResult> results = handler.getScannerRows(scanId, 10);
assertEquals(10, results.size());
for (int i = 0; i < 10; i++) {
// check if the rows are returned and in order
assertArrayEquals(("testSmallScan" + i).getBytes(), results.get(i).getRow());
}
// check that we are at the end of the scan
results = handler.getScannerRows(scanId, 10);
assertEquals(0, results.size());
// close scanner and check that it was indeed closed
handler.closeScanner(scanId);
try {
handler.getScannerRows(scanId, 10);
fail("Scanner id should be invalid");
} catch (TIllegalArgument e) {
}
}
@Test
public void testPutTTL() throws Exception {
ThriftHBaseServiceHandler handler = createHandler();