From 85fda44179c0afba74f52944ae9bb5a38266678c Mon Sep 17 00:00:00 2001 From: tedyu Date: Mon, 27 Mar 2017 10:19:46 -0700 Subject: [PATCH] HBASE-17831 Support small scan in thrift2 (Guangxu Cheng) --- .../hadoop/hbase/thrift2/ThriftUtilities.java | 19 ++ .../hbase/thrift2/generated/TReadType.java | 48 ++++ .../hadoop/hbase/thrift2/generated/TScan.java | 241 +++++++++++++++++- .../apache/hadoop/hbase/thrift2/hbase.thrift | 8 + .../TestThriftHBaseServiceHandler.java | 45 ++++ 5 files changed, 352 insertions(+), 9 deletions(-) create mode 100644 hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TReadType.java diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java index 85d95eae605..69015ab9bdc 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Scan.ReadType; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.ParseFilter; import org.apache.hadoop.hbase.security.visibility.Authorizations; @@ -62,6 +63,7 @@ import org.apache.hadoop.hbase.thrift2.generated.THRegionLocation; import org.apache.hadoop.hbase.thrift2.generated.TIncrement; import org.apache.hadoop.hbase.thrift2.generated.TMutation; import org.apache.hadoop.hbase.thrift2.generated.TPut; +import org.apache.hadoop.hbase.thrift2.generated.TReadType; import org.apache.hadoop.hbase.thrift2.generated.TResult; import org.apache.hadoop.hbase.thrift2.generated.TRowMutations; import org.apache.hadoop.hbase.thrift2.generated.TScan; @@ -445,6 +447,14 @@ public class ThriftUtilities { } } + if (in.isSetReadType()) { + out.setReadType(readTypeFromThrift(in.getReadType())); + } + + if (in.isSetLimit()) { + out.setLimit(in.getLimit()); + } + return out; } @@ -560,4 +570,13 @@ public class ThriftUtilities { default: return null; } } + + private static ReadType readTypeFromThrift(TReadType tReadType) { + switch (tReadType.getValue()) { + case 1: return ReadType.DEFAULT; + case 2: return ReadType.STREAM; + case 3: return ReadType.PREAD; + default: return null; + } + } } diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TReadType.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TReadType.java new file mode 100644 index 00000000000..4a6cf3ebd59 --- /dev/null +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TReadType.java @@ -0,0 +1,48 @@ +/** + * Autogenerated by Thrift Compiler (0.9.3) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hbase.thrift2.generated; + + +import java.util.Map; +import java.util.HashMap; +import org.apache.thrift.TEnum; + +public enum TReadType implements org.apache.thrift.TEnum { + DEFAULT(1), + STREAM(2), + PREAD(3); + + private final int value; + + private TReadType(int value) { + this.value = value; + } + + /** + * Get the integer value of this enum value, as defined in the Thrift IDL. + */ + public int getValue() { + return value; + } + + /** + * Find a the enum type by its integer value, as defined in the Thrift IDL. + * @return null if the value is not found. + */ + public static TReadType findByValue(int value) { + switch (value) { + case 1: + return DEFAULT; + case 2: + return STREAM; + case 3: + return PREAD; + default: + return null; + } + } +} diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java index 7531052dc5c..18392075de8 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/generated/TScan.java @@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory; * Any timestamps in the columns are ignored but the colFamTimeRangeMap included, use timeRange to select by timestamp. * Max versions defaults to 1. */ -@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2017-03-06") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2017-03-27") public class TScan implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("TScan"); @@ -55,6 +55,8 @@ public class TScan implements org.apache.thrift.TBase, jav private static final org.apache.thrift.protocol.TField REVERSED_FIELD_DESC = new org.apache.thrift.protocol.TField("reversed", org.apache.thrift.protocol.TType.BOOL, (short)11); private static final org.apache.thrift.protocol.TField CACHE_BLOCKS_FIELD_DESC = new org.apache.thrift.protocol.TField("cacheBlocks", org.apache.thrift.protocol.TType.BOOL, (short)12); private static final org.apache.thrift.protocol.TField COL_FAM_TIME_RANGE_MAP_FIELD_DESC = new org.apache.thrift.protocol.TField("colFamTimeRangeMap", org.apache.thrift.protocol.TType.MAP, (short)13); + private static final org.apache.thrift.protocol.TField READ_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("readType", org.apache.thrift.protocol.TType.I32, (short)14); + private static final org.apache.thrift.protocol.TField LIMIT_FIELD_DESC = new org.apache.thrift.protocol.TField("limit", org.apache.thrift.protocol.TType.I32, (short)15); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -75,6 +77,12 @@ public class TScan implements org.apache.thrift.TBase, jav public boolean reversed; // optional public boolean cacheBlocks; // optional public Map colFamTimeRangeMap; // optional + /** + * + * @see TReadType + */ + public TReadType readType; // optional + public int limit; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -90,7 +98,13 @@ public class TScan implements org.apache.thrift.TBase, jav AUTHORIZATIONS((short)10, "authorizations"), REVERSED((short)11, "reversed"), CACHE_BLOCKS((short)12, "cacheBlocks"), - COL_FAM_TIME_RANGE_MAP((short)13, "colFamTimeRangeMap"); + COL_FAM_TIME_RANGE_MAP((short)13, "colFamTimeRangeMap"), + /** + * + * @see TReadType + */ + READ_TYPE((short)14, "readType"), + LIMIT((short)15, "limit"); private static final Map byName = new HashMap(); @@ -131,6 +145,10 @@ public class TScan implements org.apache.thrift.TBase, jav return CACHE_BLOCKS; case 13: // COL_FAM_TIME_RANGE_MAP return COL_FAM_TIME_RANGE_MAP; + case 14: // READ_TYPE + return READ_TYPE; + case 15: // LIMIT + return LIMIT; default: return null; } @@ -176,8 +194,9 @@ public class TScan implements org.apache.thrift.TBase, jav private static final int __BATCHSIZE_ISSET_ID = 2; private static final int __REVERSED_ISSET_ID = 3; private static final int __CACHEBLOCKS_ISSET_ID = 4; + private static final int __LIMIT_ISSET_ID = 5; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.START_ROW,_Fields.STOP_ROW,_Fields.COLUMNS,_Fields.CACHING,_Fields.MAX_VERSIONS,_Fields.TIME_RANGE,_Fields.FILTER_STRING,_Fields.BATCH_SIZE,_Fields.ATTRIBUTES,_Fields.AUTHORIZATIONS,_Fields.REVERSED,_Fields.CACHE_BLOCKS,_Fields.COL_FAM_TIME_RANGE_MAP}; + private static final _Fields optionals[] = {_Fields.START_ROW,_Fields.STOP_ROW,_Fields.COLUMNS,_Fields.CACHING,_Fields.MAX_VERSIONS,_Fields.TIME_RANGE,_Fields.FILTER_STRING,_Fields.BATCH_SIZE,_Fields.ATTRIBUTES,_Fields.AUTHORIZATIONS,_Fields.REVERSED,_Fields.CACHE_BLOCKS,_Fields.COL_FAM_TIME_RANGE_MAP,_Fields.READ_TYPE,_Fields.LIMIT}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -208,10 +227,14 @@ public class TScan implements org.apache.thrift.TBase, jav new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); tmpMap.put(_Fields.CACHE_BLOCKS, new org.apache.thrift.meta_data.FieldMetaData("cacheBlocks", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); - tmpMap.put(_Fields.COL_FAM_TIME_RANGE_MAP, new org.apache.thrift.meta_data.FieldMetaData("colFamTimeRangeMap", org.apache.thrift.TFieldRequirementType.OPTIONAL, - new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true), + tmpMap.put(_Fields.COL_FAM_TIME_RANGE_MAP, new org.apache.thrift.meta_data.FieldMetaData("colFamTimeRangeMap", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true), new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, TTimeRange.class)))); + tmpMap.put(_Fields.READ_TYPE, new org.apache.thrift.meta_data.FieldMetaData("readType", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TReadType.class))); + tmpMap.put(_Fields.LIMIT, new org.apache.thrift.meta_data.FieldMetaData("limit", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(TScan.class, metaDataMap); } @@ -272,6 +295,10 @@ public class TScan implements org.apache.thrift.TBase, jav } this.colFamTimeRangeMap = __this__colFamTimeRangeMap; } + if (other.isSetReadType()) { + this.readType = other.readType; + } + this.limit = other.limit; } public TScan deepCopy() { @@ -298,6 +325,9 @@ public class TScan implements org.apache.thrift.TBase, jav setCacheBlocksIsSet(false); this.cacheBlocks = false; this.colFamTimeRangeMap = null; + this.readType = null; + setLimitIsSet(false); + this.limit = 0; } public byte[] getStartRow() { @@ -674,6 +704,61 @@ public class TScan implements org.apache.thrift.TBase, jav } } + /** + * + * @see TReadType + */ + public TReadType getReadType() { + return this.readType; + } + + /** + * + * @see TReadType + */ + public TScan setReadType(TReadType readType) { + this.readType = readType; + return this; + } + + public void unsetReadType() { + this.readType = null; + } + + /** Returns true if field readType is set (has been assigned a value) and false otherwise */ + public boolean isSetReadType() { + return this.readType != null; + } + + public void setReadTypeIsSet(boolean value) { + if (!value) { + this.readType = null; + } + } + + public int getLimit() { + return this.limit; + } + + public TScan setLimit(int limit) { + this.limit = limit; + setLimitIsSet(true); + return this; + } + + public void unsetLimit() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __LIMIT_ISSET_ID); + } + + /** Returns true if field limit is set (has been assigned a value) and false otherwise */ + public boolean isSetLimit() { + return EncodingUtils.testBit(__isset_bitfield, __LIMIT_ISSET_ID); + } + + public void setLimitIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __LIMIT_ISSET_ID, value); + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case START_ROW: @@ -780,6 +865,22 @@ public class TScan implements org.apache.thrift.TBase, jav } break; + case READ_TYPE: + if (value == null) { + unsetReadType(); + } else { + setReadType((TReadType)value); + } + break; + + case LIMIT: + if (value == null) { + unsetLimit(); + } else { + setLimit((Integer)value); + } + break; + } } @@ -824,6 +925,12 @@ public class TScan implements org.apache.thrift.TBase, jav case COL_FAM_TIME_RANGE_MAP: return getColFamTimeRangeMap(); + case READ_TYPE: + return getReadType(); + + case LIMIT: + return getLimit(); + } throw new IllegalStateException(); } @@ -861,6 +968,10 @@ public class TScan implements org.apache.thrift.TBase, jav return isSetCacheBlocks(); case COL_FAM_TIME_RANGE_MAP: return isSetColFamTimeRangeMap(); + case READ_TYPE: + return isSetReadType(); + case LIMIT: + return isSetLimit(); } throw new IllegalStateException(); } @@ -995,6 +1106,24 @@ public class TScan implements org.apache.thrift.TBase, jav return false; } + boolean this_present_readType = true && this.isSetReadType(); + boolean that_present_readType = true && that.isSetReadType(); + if (this_present_readType || that_present_readType) { + if (!(this_present_readType && that_present_readType)) + return false; + if (!this.readType.equals(that.readType)) + return false; + } + + boolean this_present_limit = true && this.isSetLimit(); + boolean that_present_limit = true && that.isSetLimit(); + if (this_present_limit || that_present_limit) { + if (!(this_present_limit && that_present_limit)) + return false; + if (this.limit != that.limit) + return false; + } + return true; } @@ -1067,6 +1196,16 @@ public class TScan implements org.apache.thrift.TBase, jav if (present_colFamTimeRangeMap) list.add(colFamTimeRangeMap); + boolean present_readType = true && (isSetReadType()); + list.add(present_readType); + if (present_readType) + list.add(readType.getValue()); + + boolean present_limit = true && (isSetLimit()); + list.add(present_limit); + if (present_limit) + list.add(limit); + return list.hashCode(); } @@ -1208,6 +1347,26 @@ public class TScan implements org.apache.thrift.TBase, jav return lastComparison; } } + lastComparison = Boolean.valueOf(isSetReadType()).compareTo(other.isSetReadType()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetReadType()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.readType, other.readType); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetLimit()).compareTo(other.isSetLimit()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetLimit()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.limit, other.limit); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -1337,6 +1496,22 @@ public class TScan implements org.apache.thrift.TBase, jav } first = false; } + if (isSetReadType()) { + if (!first) sb.append(", "); + sb.append("readType:"); + if (this.readType == null) { + sb.append("null"); + } else { + sb.append(this.readType); + } + first = false; + } + if (isSetLimit()) { + if (!first) sb.append(", "); + sb.append("limit:"); + sb.append(this.limit); + first = false; + } sb.append(")"); return sb.toString(); } @@ -1526,7 +1701,23 @@ public class TScan implements org.apache.thrift.TBase, jav iprot.readMapEnd(); } struct.setColFamTimeRangeMapIsSet(true); - } else { + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 14: // READ_TYPE + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.readType = org.apache.hadoop.hbase.thrift2.generated.TReadType.findByValue(iprot.readI32()); + struct.setReadTypeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 15: // LIMIT + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.limit = iprot.readI32(); + struct.setLimitIsSet(true); + } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; @@ -1649,6 +1840,18 @@ public class TScan implements org.apache.thrift.TBase, jav oprot.writeFieldEnd(); } } + if (struct.readType != null) { + if (struct.isSetReadType()) { + oprot.writeFieldBegin(READ_TYPE_FIELD_DESC); + oprot.writeI32(struct.readType.getValue()); + oprot.writeFieldEnd(); + } + } + if (struct.isSetLimit()) { + oprot.writeFieldBegin(LIMIT_FIELD_DESC); + oprot.writeI32(struct.limit); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -1706,7 +1909,13 @@ public class TScan implements org.apache.thrift.TBase, jav if (struct.isSetColFamTimeRangeMap()) { optionals.set(12); } - oprot.writeBitSet(optionals, 13); + if (struct.isSetReadType()) { + optionals.set(13); + } + if (struct.isSetLimit()) { + optionals.set(14); + } + oprot.writeBitSet(optionals, 15); if (struct.isSetStartRow()) { oprot.writeBinary(struct.startRow); } @@ -1766,12 +1975,18 @@ public class TScan implements org.apache.thrift.TBase, jav } } } + if (struct.isSetReadType()) { + oprot.writeI32(struct.readType.getValue()); + } + if (struct.isSetLimit()) { + oprot.writeI32(struct.limit); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, TScan struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(13); + BitSet incoming = iprot.readBitSet(15); if (incoming.get(0)) { struct.startRow = iprot.readBinary(); struct.setStartRowIsSet(true); @@ -1859,6 +2074,14 @@ public class TScan implements org.apache.thrift.TBase, jav } struct.setColFamTimeRangeMapIsSet(true); } + if (incoming.get(13)) { + struct.readType = org.apache.hadoop.hbase.thrift2.generated.TReadType.findByValue(iprot.readI32()); + struct.setReadTypeIsSet(true); + } + if (incoming.get(14)) { + struct.limit = iprot.readI32(); + struct.setLimitIsSet(true); + } } } diff --git a/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift b/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift index 0bd8ecef3fc..e2e5b2994c0 100644 --- a/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift +++ b/hbase-thrift/src/main/resources/org/apache/hadoop/hbase/thrift2/hbase.thrift @@ -208,6 +208,12 @@ struct TAppend { 5: optional TCellVisibility cellVisibility } +enum TReadType { + DEFAULT = 1, + STREAM = 2, + PREAD = 3 +} + /** * Any timestamps in the columns are ignored but the colFamTimeRangeMap included, use timeRange to select by timestamp. * Max versions defaults to 1. @@ -226,6 +232,8 @@ struct TScan { 11: optional bool reversed 12: optional bool cacheBlocks 13: optional map colFamTimeRangeMap + 14: optional TReadType readType + 15: optional i32 limit } /** diff --git a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java index 4b202f69775..c3f59f61cc1 100644 --- a/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java +++ b/hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.thrift2.generated.TIOError; import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument; import org.apache.hadoop.hbase.thrift2.generated.TIncrement; import org.apache.hadoop.hbase.thrift2.generated.TPut; +import org.apache.hadoop.hbase.thrift2.generated.TReadType; import org.apache.hadoop.hbase.thrift2.generated.TResult; import org.apache.hadoop.hbase.thrift2.generated.TScan; import org.apache.hadoop.hbase.thrift2.generated.TMutation; @@ -850,6 +851,50 @@ public class TestThriftHBaseServiceHandler { } } + @Test + public void testSmallScan() throws Exception { + ThriftHBaseServiceHandler handler = createHandler(); + ByteBuffer table = wrap(tableAname); + + // insert data + TColumnValue columnValue = new TColumnValue(wrap(familyAname), wrap(qualifierAname), + wrap(valueAname)); + List columnValues = new ArrayList(); + columnValues.add(columnValue); + for (int i = 0; i < 10; i++) { + TPut put = new TPut(wrap(("testSmallScan" + i).getBytes()), columnValues); + handler.put(table, put); + } + + // small scan instance + TScan scan = new TScan(); + scan.setStartRow("testSmallScan".getBytes()); + scan.setStopRow("testSmallScan\uffff".getBytes()); + scan.setReadType(TReadType.PREAD); + scan.setCaching(2); + + // get scanner and rows + int scanId = handler.openScanner(table, scan); + List results = handler.getScannerRows(scanId, 10); + assertEquals(10, results.size()); + for (int i = 0; i < 10; i++) { + // check if the rows are returned and in order + assertArrayEquals(("testSmallScan" + i).getBytes(), results.get(i).getRow()); + } + + // check that we are at the end of the scan + results = handler.getScannerRows(scanId, 10); + assertEquals(0, results.size()); + + // close scanner and check that it was indeed closed + handler.closeScanner(scanId); + try { + handler.getScannerRows(scanId, 10); + fail("Scanner id should be invalid"); + } catch (TIllegalArgument e) { + } + } + @Test public void testPutTTL() throws Exception { ThriftHBaseServiceHandler handler = createHandler();