HBASE-2438 Addition of a Column Pagination Filter
git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@936070 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack
parent
0f17cbf487
commit
2b2f1d1670
|
|
@ -546,6 +546,7 @@ Release 0.21.0 - Unreleased
|
|||
HBASE-2327 [EC2] Allocate elastic IP addresses for ZK and master nodes
|
||||
HBASE-2319 [stargate] multiuser mode: request shaping
|
||||
HBASE-2403 [stargate] client HTable interface to REST connector
|
||||
HBASE-2438 Addition of a Column Pagination Filter (Paul Kist via Stack)
|
||||
|
||||
OPTIMIZATIONS
|
||||
HBASE-410 [testing] Speed up the test suite
|
||||
|
|
|
|||
|
|
@ -0,0 +1,97 @@
|
|||
/**
|
||||
* Copyright 2007 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.filter;
|
||||
|
||||
import java.io.DataInput;
|
||||
import java.io.DataOutput;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
|
||||
/**
|
||||
* A filter, based on the ColumnCountGetFilter, takes two arguments: limit and offset.
|
||||
* This filter can be used for row-based indexing, where references to other tables are stored across many columns,
|
||||
* in order to efficient lookups and paginated results for end users.
|
||||
*/
|
||||
public class ColumnPaginationFilter implements Filter
|
||||
{
|
||||
private int limit = 0;
|
||||
private int offset = 0;
|
||||
private int count = 0;
|
||||
|
||||
/**
|
||||
* Used during serialization. Do not use.
|
||||
*/
|
||||
public ColumnPaginationFilter()
|
||||
{
|
||||
super();
|
||||
}
|
||||
|
||||
public ColumnPaginationFilter(final int limit, final int offset)
|
||||
{
|
||||
this.limit = limit;
|
||||
this.offset = offset;
|
||||
}
|
||||
|
||||
public boolean filterAllRemaining()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
public ReturnCode filterKeyValue(KeyValue v)
|
||||
{
|
||||
if(count >= offset + limit)
|
||||
{
|
||||
return ReturnCode.NEXT_ROW;
|
||||
}
|
||||
|
||||
ReturnCode code = count < offset ? ReturnCode.SKIP : ReturnCode.INCLUDE;
|
||||
count++;
|
||||
return code;
|
||||
}
|
||||
|
||||
public boolean filterRow()
|
||||
{
|
||||
this.count = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean filterRowKey(byte[] buffer, int offset, int length)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
public void reset()
|
||||
{
|
||||
this.count = 0;
|
||||
}
|
||||
|
||||
public void readFields(DataInput in) throws IOException
|
||||
{
|
||||
this.limit = in.readInt();
|
||||
this.offset = in.readInt();
|
||||
}
|
||||
|
||||
public void write(DataOutput out) throws IOException
|
||||
{
|
||||
out.writeInt(this.limit);
|
||||
out.writeInt(this.offset);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
/**
|
||||
* Copyright 2007 The Apache Software Foundation
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.filter;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.DataOutputStream;
|
||||
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
/**
|
||||
* Test for the ColumnPaginationFilter, used mainly to test the successful serialization of the filter.
|
||||
* More test functionality can be found within {@link org.apache.hadoop.hbase.filter.TestFilter#testColumnPaginationFilter()}
|
||||
*/
|
||||
public class TestColumnPaginationFilter extends TestCase
|
||||
{
|
||||
private static final byte[] ROW = Bytes.toBytes("row_1_test");
|
||||
private static final byte[] COLUMN_FAMILY = Bytes.toBytes("test");
|
||||
private static final byte[] VAL_1 = Bytes.toBytes("a");
|
||||
private static final byte [] COLUMN_QUALIFIER = Bytes.toBytes("foo");
|
||||
|
||||
private Filter columnPaginationFilter;
|
||||
|
||||
@Override
|
||||
protected void setUp() throws Exception {
|
||||
super.setUp();
|
||||
columnPaginationFilter = getColumnPaginationFilter();
|
||||
|
||||
}
|
||||
private Filter getColumnPaginationFilter() {
|
||||
return new ColumnPaginationFilter(1,0);
|
||||
}
|
||||
|
||||
private Filter serializationTest(Filter filter) throws Exception {
|
||||
ByteArrayOutputStream stream = new ByteArrayOutputStream();
|
||||
DataOutputStream out = new DataOutputStream(stream);
|
||||
filter.write(out);
|
||||
out.close();
|
||||
byte[] buffer = stream.toByteArray();
|
||||
|
||||
DataInputStream in =
|
||||
new DataInputStream(new ByteArrayInputStream(buffer));
|
||||
Filter newFilter = new ColumnPaginationFilter();
|
||||
newFilter.readFields(in);
|
||||
|
||||
return newFilter;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* The more specific functionality tests are contained within the TestFilters class. This class is mainly for testing
|
||||
* serialization
|
||||
*
|
||||
* @param filter
|
||||
* @throws Exception
|
||||
*/
|
||||
private void basicFilterTests(ColumnPaginationFilter filter) throws Exception
|
||||
{
|
||||
KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_1);
|
||||
assertTrue("basicFilter1", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests serialization
|
||||
* @throws Exception
|
||||
*/
|
||||
public void testSerialization() throws Exception {
|
||||
Filter newFilter = serializationTest(columnPaginationFilter);
|
||||
basicFilterTests((ColumnPaginationFilter)newFilter);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1103,4 +1103,97 @@ public class TestFilter extends HBaseTestCase {
|
|||
assertEquals("Expected " + kvs.length + " total keys but scanned " + idx,
|
||||
kvs.length, idx);
|
||||
}
|
||||
|
||||
|
||||
public void testColumnPaginationFilter() throws Exception {
|
||||
|
||||
// Set of KVs (page: 1; pageSize: 1) - the first set of 1 column per row
|
||||
KeyValue [] expectedKVs = {
|
||||
// testRowOne-0
|
||||
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
|
||||
// testRowOne-2
|
||||
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
|
||||
// testRowOne-3
|
||||
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
|
||||
// testRowTwo-0
|
||||
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
|
||||
// testRowTwo-2
|
||||
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
|
||||
// testRowTwo-3
|
||||
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1])
|
||||
};
|
||||
|
||||
|
||||
// Set of KVs (page: 3; pageSize: 1) - the third set of 1 column per row
|
||||
KeyValue [] expectedKVs2 = {
|
||||
// testRowOne-0
|
||||
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
|
||||
// testRowOne-2
|
||||
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
|
||||
// testRowOne-3
|
||||
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
|
||||
// testRowTwo-0
|
||||
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
|
||||
// testRowTwo-2
|
||||
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
|
||||
// testRowTwo-3
|
||||
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
|
||||
};
|
||||
|
||||
// Set of KVs (page: 2; pageSize 2) - the 2nd set of 2 columns per row
|
||||
KeyValue [] expectedKVs3 = {
|
||||
// testRowOne-0
|
||||
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
|
||||
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
|
||||
// testRowOne-2
|
||||
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
|
||||
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
|
||||
// testRowOne-3
|
||||
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
|
||||
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
|
||||
// testRowTwo-0
|
||||
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
|
||||
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
|
||||
// testRowTwo-2
|
||||
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
|
||||
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
|
||||
// testRowTwo-3
|
||||
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
|
||||
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
|
||||
};
|
||||
|
||||
|
||||
// Set of KVs (page: 2; pageSize 2) - the 2nd set of 2 columns per row
|
||||
KeyValue [] expectedKVs4 = {
|
||||
|
||||
};
|
||||
|
||||
long expectedRows = this.numRows;
|
||||
long expectedKeys = 1;
|
||||
Scan s = new Scan();
|
||||
|
||||
|
||||
// Page 1; 1 Column per page (Limit 1, Offset 0)
|
||||
s.setFilter(new ColumnPaginationFilter(1,0));
|
||||
verifyScan(s, expectedRows, expectedKeys);
|
||||
this.verifyScanFull(s, expectedKVs);
|
||||
|
||||
// Page 3; 1 Result per page (Limit 1, Offset 2)
|
||||
s.setFilter(new ColumnPaginationFilter(1,2));
|
||||
verifyScan(s, expectedRows, expectedKeys);
|
||||
this.verifyScanFull(s, expectedKVs2);
|
||||
|
||||
// Page 2; 2 Results per page (Limit 2, Offset 2)
|
||||
s.setFilter(new ColumnPaginationFilter(2,2));
|
||||
expectedKeys = 2;
|
||||
verifyScan(s, expectedRows, expectedKeys);
|
||||
this.verifyScanFull(s, expectedKVs3);
|
||||
|
||||
// Page 8; 20 Results per page (no results) (Limit 20, Offset 140)
|
||||
s.setFilter(new ColumnPaginationFilter(20,140));
|
||||
expectedKeys = 0;
|
||||
expectedRows = 0;
|
||||
verifyScan(s, expectedRows, 0);
|
||||
this.verifyScanFull(s, expectedKVs4);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue