From 2b2f1d1670040f2e0d27921a3f002099ff6ba447 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Tue, 20 Apr 2010 21:28:49 +0000 Subject: [PATCH] HBASE-2438 Addition of a Column Pagination Filter git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@936070 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 1 + .../hbase/filter/ColumnPaginationFilter.java | 97 +++++++++++++++++++ .../filter/TestColumnPaginationFilter.java | 94 ++++++++++++++++++ .../hadoop/hbase/filter/TestFilter.java | 93 ++++++++++++++++++ 4 files changed, 285 insertions(+) create mode 100644 core/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java create mode 100644 core/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java diff --git a/CHANGES.txt b/CHANGES.txt index 97d779329aa..67702339cd4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -546,6 +546,7 @@ Release 0.21.0 - Unreleased HBASE-2327 [EC2] Allocate elastic IP addresses for ZK and master nodes HBASE-2319 [stargate] multiuser mode: request shaping HBASE-2403 [stargate] client HTable interface to REST connector + HBASE-2438 Addition of a Column Pagination Filter (Paul Kist via Stack) OPTIMIZATIONS HBASE-410 [testing] Speed up the test suite diff --git a/core/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java b/core/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java new file mode 100644 index 00000000000..4d12b55e0e0 --- /dev/null +++ b/core/src/main/java/org/apache/hadoop/hbase/filter/ColumnPaginationFilter.java @@ -0,0 +1,97 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.KeyValue; + +/** + * A filter, based on the ColumnCountGetFilter, takes two arguments: limit and offset. + * This filter can be used for row-based indexing, where references to other tables are stored across many columns, + * in order to efficient lookups and paginated results for end users. + */ +public class ColumnPaginationFilter implements Filter +{ + private int limit = 0; + private int offset = 0; + private int count = 0; + + /** + * Used during serialization. Do not use. + */ + public ColumnPaginationFilter() + { + super(); + } + + public ColumnPaginationFilter(final int limit, final int offset) + { + this.limit = limit; + this.offset = offset; + } + + public boolean filterAllRemaining() + { + return false; + } + + public ReturnCode filterKeyValue(KeyValue v) + { + if(count >= offset + limit) + { + return ReturnCode.NEXT_ROW; + } + + ReturnCode code = count < offset ? ReturnCode.SKIP : ReturnCode.INCLUDE; + count++; + return code; + } + + public boolean filterRow() + { + this.count = 0; + return false; + } + + public boolean filterRowKey(byte[] buffer, int offset, int length) + { + return false; + } + + public void reset() + { + this.count = 0; + } + + public void readFields(DataInput in) throws IOException + { + this.limit = in.readInt(); + this.offset = in.readInt(); + } + + public void write(DataOutput out) throws IOException + { + out.writeInt(this.limit); + out.writeInt(this.offset); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java b/core/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java new file mode 100644 index 00000000000..0a47f9513dc --- /dev/null +++ b/core/src/test/java/org/apache/hadoop/hbase/filter/TestColumnPaginationFilter.java @@ -0,0 +1,94 @@ +/** + * Copyright 2007 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; + +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; + +import junit.framework.TestCase; + +/** + * Test for the ColumnPaginationFilter, used mainly to test the successful serialization of the filter. + * More test functionality can be found within {@link org.apache.hadoop.hbase.filter.TestFilter#testColumnPaginationFilter()} + */ +public class TestColumnPaginationFilter extends TestCase +{ + private static final byte[] ROW = Bytes.toBytes("row_1_test"); + private static final byte[] COLUMN_FAMILY = Bytes.toBytes("test"); + private static final byte[] VAL_1 = Bytes.toBytes("a"); + private static final byte [] COLUMN_QUALIFIER = Bytes.toBytes("foo"); + + private Filter columnPaginationFilter; + + @Override + protected void setUp() throws Exception { + super.setUp(); + columnPaginationFilter = getColumnPaginationFilter(); + + } + private Filter getColumnPaginationFilter() { + return new ColumnPaginationFilter(1,0); + } + + private Filter serializationTest(Filter filter) throws Exception { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(stream); + filter.write(out); + out.close(); + byte[] buffer = stream.toByteArray(); + + DataInputStream in = + new DataInputStream(new ByteArrayInputStream(buffer)); + Filter newFilter = new ColumnPaginationFilter(); + newFilter.readFields(in); + + return newFilter; + } + + + /** + * The more specific functionality tests are contained within the TestFilters class. This class is mainly for testing + * serialization + * + * @param filter + * @throws Exception + */ + private void basicFilterTests(ColumnPaginationFilter filter) throws Exception + { + KeyValue kv = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_1); + assertTrue("basicFilter1", filter.filterKeyValue(kv) == Filter.ReturnCode.INCLUDE); + } + + /** + * Tests serialization + * @throws Exception + */ + public void testSerialization() throws Exception { + Filter newFilter = serializationTest(columnPaginationFilter); + basicFilterTests((ColumnPaginationFilter)newFilter); + } + + +} diff --git a/core/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/core/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java index 71f7d69127d..ab0846d455d 100644 --- a/core/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java +++ b/core/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java @@ -1103,4 +1103,97 @@ public class TestFilter extends HBaseTestCase { assertEquals("Expected " + kvs.length + " total keys but scanned " + idx, kvs.length, idx); } + + + public void testColumnPaginationFilter() throws Exception { + + // Set of KVs (page: 1; pageSize: 1) - the first set of 1 column per row + KeyValue [] expectedKVs = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]) + }; + + + // Set of KVs (page: 3; pageSize: 1) - the third set of 1 column per row + KeyValue [] expectedKVs2 = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + }; + + // Set of KVs (page: 2; pageSize 2) - the 2nd set of 2 columns per row + KeyValue [] expectedKVs3 = { + // testRowOne-0 + new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + // testRowOne-2 + new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + // testRowOne-3 + new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]), + new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]), + // testRowTwo-0 + new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + // testRowTwo-2 + new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + // testRowTwo-3 + new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), + new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]), + }; + + + // Set of KVs (page: 2; pageSize 2) - the 2nd set of 2 columns per row + KeyValue [] expectedKVs4 = { + + }; + + long expectedRows = this.numRows; + long expectedKeys = 1; + Scan s = new Scan(); + + + // Page 1; 1 Column per page (Limit 1, Offset 0) + s.setFilter(new ColumnPaginationFilter(1,0)); + verifyScan(s, expectedRows, expectedKeys); + this.verifyScanFull(s, expectedKVs); + + // Page 3; 1 Result per page (Limit 1, Offset 2) + s.setFilter(new ColumnPaginationFilter(1,2)); + verifyScan(s, expectedRows, expectedKeys); + this.verifyScanFull(s, expectedKVs2); + + // Page 2; 2 Results per page (Limit 2, Offset 2) + s.setFilter(new ColumnPaginationFilter(2,2)); + expectedKeys = 2; + verifyScan(s, expectedRows, expectedKeys); + this.verifyScanFull(s, expectedKVs3); + + // Page 8; 20 Results per page (no results) (Limit 20, Offset 140) + s.setFilter(new ColumnPaginationFilter(20,140)); + expectedKeys = 0; + expectedRows = 0; + verifyScan(s, expectedRows, 0); + this.verifyScanFull(s, expectedKVs4); + } }