Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-9345 Add support for specifying filters in scan (Virag Kothari)

This commit is contained in:
Michael Stack 2014-06-19 16:11:52 -07:00
parent 9829bb9c24
commit b5db143280
4 changed files with 144 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hbase-server/src/main/java/org/apache/hadoop/hbase/rest/Constants.java
View File

@ -68,6 +68,8 @@ public interface Constants {
String SCAN_BATCH_SIZE = "batchsize";
String SCAN_LIMIT = "limit";
String SCAN_FETCH_SIZE = "hbase.rest.scan.fetchsize";
String SCAN_FILTER = "filter";
String CUSTOM_FILTERS = "hbase.rest.custom.filters";
String ROW_KEYS_PARAM_NAME = "row";
/** If this query parameter is present when processing row or scanner resources,

17
hbase-server/src/main/java/org/apache/hadoop/hbase/rest/RESTServlet.java
View File

@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.filter.ParseFilter;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@ -186,6 +187,7 @@ public class RESTServlet implements Constants {
this.realUser = realUser;
this.conf = conf;
registerCustomFilter(conf);
}
/**
@ -255,4 +257,19 @@ public class RESTServlet implements Constants {
}
return connInfo;
}
private void registerCustomFilter(Configuration conf) {
String[] filterList = conf.getStrings(Constants.CUSTOM_FILTERS);
if (filterList != null) {
for (String filterClass : filterList) {
String[] filterPart = filterClass.split(":");
if (filterPart.length != 2) {
LOG.warn(
"Invalid filter specification " + filterClass + " - skipping");
} else {
ParseFilter.registerFilter(filterPart[0], filterPart[1]);
}
}
}
}
}

20
hbase-server/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
View File

@ -38,6 +38,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.ParseFilter;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.util.Bytes;
@ -130,7 +132,8 @@ public class TableResource extends ResourceBase {
@DefaultValue("-1") @QueryParam(Constants.SCAN_BATCH_SIZE) int batchSize,
@DefaultValue("0") @QueryParam(Constants.SCAN_START_TIME) long startTime,
@DefaultValue(Long.MAX_VALUE + "") @QueryParam(Constants.SCAN_END_TIME) long endTime,
@DefaultValue("true") @QueryParam(Constants.SCAN_BATCH_SIZE) boolean cacheBlocks) {
@DefaultValue("true") @QueryParam(Constants.SCAN_BATCH_SIZE) boolean cacheBlocks,
@DefaultValue("") @QueryParam(Constants.SCAN_FILTER) String filters) {
try {
Filter filter = null;
if (scanSpec.indexOf('*') > 0) {
@ -164,7 +167,20 @@ public class TableResource extends ResourceBase {
tableScan.addFamily(Bytes.toBytes(familysplit[0]));
}
}
if (filter != null) {
FilterList filterList = null;
if (StringUtils.isNotEmpty(filters)) {
ParseFilter pf = new ParseFilter();
Filter filterParam = pf.parseFilterString(filters);
if (filter != null) {
filterList = new FilterList(filter, filterParam);
}
else {
filter = filterParam;
}
}
if (filterList != null) {
tableScan.setFilter(filterList);
} else if (filter != null) {
tableScan.setFilter(filter);
}
int fetchSize = this.servlet.getConfiguration().getInt(Constants.SCAN_FETCH_SIZE, 10);

108
hbase-server/src/test/java/org/apache/hadoop/hbase/rest/TestTableScan.java
View File

@ -27,6 +27,7 @@ import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
@ -48,6 +49,9 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.ParseFilter;
import org.apache.hadoop.hbase.filter.PrefixFilter;
import org.apache.hadoop.hbase.rest.client.Client;
import org.apache.hadoop.hbase.rest.client.Cluster;
import org.apache.hadoop.hbase.rest.client.Response;
@ -87,6 +91,7 @@ public class TestTableScan {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
conf = TEST_UTIL.getConfiguration();
conf.set(Constants.CUSTOM_FILTERS, "CustomFilter:" + CustomFilter.class.getName());
TEST_UTIL.startMiniCluster();
REST_TEST_UTIL.startServletContainer(conf);
client = new Client(new Cluster().add("localhost",
@ -184,7 +189,6 @@ public class TestTableScan {
count = TestScannerResource.countCellSet(model);
assertEquals(15, count);
checkRowsNotNull(model);
}
@Test
@ -459,6 +463,108 @@ public class TestTableScan {
int count = TestScannerResource.countCellSet(model);
assertEquals(0, count);
}
@Test
public void testSimpleFilter() throws IOException, JAXBException {
StringBuilder builder = new StringBuilder();
builder = new StringBuilder();
builder.append("/*");
builder.append("?");
builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
builder.append("&");
builder.append(Constants.SCAN_START_ROW + "=aaa");
builder.append("&");
builder.append(Constants.SCAN_END_ROW + "=aay");
builder.append("&");
builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("PrefixFilter('aab')", "UTF-8"));
Response response =
client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
Unmarshaller ush = ctx.createUnmarshaller();
CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream());
int count = TestScannerResource.countCellSet(model);
assertEquals(1, count);
assertEquals("aab", new String(model.getRows().get(0).getCells().get(0).getValue()));
}
@Test
public void testCompoundFilter() throws IOException, JAXBException {
StringBuilder builder = new StringBuilder();
builder = new StringBuilder();
builder.append("/*");
builder.append("?");
builder.append(Constants.SCAN_FILTER + "="
+ URLEncoder.encode("PrefixFilter('abc') AND QualifierFilter(=,'binary:1')", "UTF-8"));
Response response =
client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
Unmarshaller ush = ctx.createUnmarshaller();
CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream());
int count = TestScannerResource.countCellSet(model);
assertEquals(1, count);
assertEquals("abc", new String(model.getRows().get(0).getCells().get(0).getValue()));
}
@Test
public void testCustomFilter() throws IOException, JAXBException {
StringBuilder builder = new StringBuilder();
builder = new StringBuilder();
builder.append("/a*");
builder.append("?");
builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
builder.append("&");
builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8"));
Response response =
client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
Unmarshaller ush = ctx.createUnmarshaller();
CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream());
int count = TestScannerResource.countCellSet(model);
assertEquals(1, count);
assertEquals("abc", new String(model.getRows().get(0).getCells().get(0).getValue()));
}
@Test
public void testNegativeCustomFilter() throws IOException, JAXBException {
StringBuilder builder = new StringBuilder();
builder = new StringBuilder();
builder.append("/b*");
builder.append("?");
builder.append(Constants.SCAN_COLUMN + "=" + COLUMN_1);
builder.append("&");
builder.append(Constants.SCAN_FILTER + "=" + URLEncoder.encode("CustomFilter('abc')", "UTF-8"));
Response response =
client.get("/" + TABLE + builder.toString(), Constants.MIMETYPE_XML);
assertEquals(200, response.getCode());
JAXBContext ctx = JAXBContext.newInstance(CellSetModel.class);
Unmarshaller ush = ctx.createUnmarshaller();
CellSetModel model = (CellSetModel) ush.unmarshal(response.getStream());
int count = TestScannerResource.countCellSet(model);
// Should return no rows as the filters conflict
assertEquals(0, count);
}
public static class CustomFilter extends PrefixFilter {
private byte[] key = null;
public CustomFilter(byte[] key) {
super(key);
}
@Override
public boolean filterRowKey(byte[] buffer, int offset, int length) {
int cmp = Bytes.compareTo(buffer, offset, length, this.key, 0, this.key.length);
return cmp != 0;
}
public static Filter createFilterFromArguments(ArrayList<byte[]> filterArguments) {
byte[] prefix = ParseFilter.removeQuotesFromByteArray(filterArguments.get(0));
return new CustomFilter(prefix);
}
}
/**
* The Class ClientSideCellSetModel which mimics cell set model, and contains listener to perform