HBASE-16691 Optimize KeyOnlyFilter by utilizing KeyOnlyCell.(binlijin)
This commit is contained in:
anoopsamjohn
parent
aa4a678ebf
commit
b0fcca6d7b
|
|
@ -20,11 +20,12 @@ package org.apache.hadoop.hbase.filter;
|
|||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.apache.hadoop.hbase.ByteBufferedCell;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.KeyValueUtil;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.hbase.classification.InterfaceStability;
|
||||
import org.apache.hadoop.hbase.exceptions.DeserializationException;
|
||||
|
|
@ -61,19 +62,11 @@ public class KeyOnlyFilter extends FilterBase {
|
|||
}
|
||||
|
||||
private Cell createKeyOnlyCell(Cell c) {
|
||||
// KV format: <keylen:4><valuelen:4><key:keylen><value:valuelen>
|
||||
// Rebuild as: <keylen:4><0:4><key:keylen>
|
||||
int dataLen = lenAsVal ? Bytes.SIZEOF_INT : 0;
|
||||
int keyOffset = (2 * Bytes.SIZEOF_INT);
|
||||
int keyLen = KeyValueUtil.keyLength(c);
|
||||
byte[] newBuffer = new byte[keyLen + keyOffset + dataLen];
|
||||
Bytes.putInt(newBuffer, 0, keyLen);
|
||||
Bytes.putInt(newBuffer, Bytes.SIZEOF_INT, dataLen);
|
||||
KeyValueUtil.appendKeyTo(c, newBuffer, keyOffset);
|
||||
if (lenAsVal) {
|
||||
Bytes.putInt(newBuffer, newBuffer.length - dataLen, c.getValueLength());
|
||||
if (c instanceof ByteBufferedCell) {
|
||||
return new KeyOnlyByteBufferedCell((ByteBufferedCell) c, lenAsVal);
|
||||
} else {
|
||||
return new KeyOnlyCell(c, lenAsVal);
|
||||
}
|
||||
return new KeyValue(newBuffer);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -130,4 +123,275 @@ public class KeyOnlyFilter extends FilterBase {
|
|||
KeyOnlyFilter other = (KeyOnlyFilter)o;
|
||||
return this.lenAsVal == other.lenAsVal;
|
||||
}
|
||||
|
||||
static class KeyOnlyCell implements Cell {
|
||||
private Cell cell;
|
||||
private boolean lenAsVal;
|
||||
|
||||
public KeyOnlyCell(Cell c, boolean lenAsVal) {
|
||||
this.cell = c;
|
||||
this.lenAsVal = lenAsVal;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getRowArray() {
|
||||
return cell.getRowArray();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getRowOffset() {
|
||||
return cell.getRowOffset();
|
||||
}
|
||||
|
||||
@Override
|
||||
public short getRowLength() {
|
||||
return cell.getRowLength();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getFamilyArray() {
|
||||
return cell.getFamilyArray();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getFamilyOffset() {
|
||||
return cell.getFamilyOffset();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte getFamilyLength() {
|
||||
return cell.getFamilyLength();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getQualifierArray() {
|
||||
return cell.getQualifierArray();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getQualifierOffset() {
|
||||
return cell.getQualifierOffset();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getQualifierLength() {
|
||||
return cell.getQualifierLength();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTimestamp() {
|
||||
return cell.getTimestamp();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte getTypeByte() {
|
||||
return cell.getTypeByte();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSequenceId() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getValueArray() {
|
||||
if (lenAsVal) {
|
||||
return Bytes.toBytes(cell.getValueLength());
|
||||
} else {
|
||||
return HConstants.EMPTY_BYTE_ARRAY;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getValueOffset() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getValueLength() {
|
||||
if (lenAsVal) {
|
||||
return Bytes.SIZEOF_INT;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getTagsArray() {
|
||||
return HConstants.EMPTY_BYTE_ARRAY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getTagsOffset() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getTagsLength() {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static class KeyOnlyByteBufferedCell extends ByteBufferedCell {
|
||||
private ByteBufferedCell cell;
|
||||
private boolean lenAsVal;
|
||||
|
||||
public KeyOnlyByteBufferedCell(ByteBufferedCell c, boolean lenAsVal) {
|
||||
this.cell = c;
|
||||
this.lenAsVal = lenAsVal;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getRowArray() {
|
||||
return cell.getRowArray();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getRowOffset() {
|
||||
return cell.getRowOffset();
|
||||
}
|
||||
|
||||
@Override
|
||||
public short getRowLength() {
|
||||
return cell.getRowLength();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getFamilyArray() {
|
||||
return cell.getFamilyArray();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getFamilyOffset() {
|
||||
return cell.getFamilyOffset();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte getFamilyLength() {
|
||||
return cell.getFamilyLength();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getQualifierArray() {
|
||||
return cell.getQualifierArray();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getQualifierOffset() {
|
||||
return cell.getQualifierOffset();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getQualifierLength() {
|
||||
return cell.getQualifierLength();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTimestamp() {
|
||||
return cell.getTimestamp();
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte getTypeByte() {
|
||||
return cell.getTypeByte();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSequenceId() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getValueArray() {
|
||||
if (lenAsVal) {
|
||||
return Bytes.toBytes(cell.getValueLength());
|
||||
} else {
|
||||
return HConstants.EMPTY_BYTE_ARRAY;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getValueOffset() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getValueLength() {
|
||||
if (lenAsVal) {
|
||||
return Bytes.SIZEOF_INT;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getTagsArray() {
|
||||
return HConstants.EMPTY_BYTE_ARRAY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getTagsOffset() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getTagsLength() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ByteBuffer getRowByteBuffer() {
|
||||
return cell.getRowByteBuffer();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getRowPosition() {
|
||||
return cell.getRowPosition();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ByteBuffer getFamilyByteBuffer() {
|
||||
return cell.getFamilyByteBuffer();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getFamilyPosition() {
|
||||
return cell.getFamilyPosition();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ByteBuffer getQualifierByteBuffer() {
|
||||
return cell.getQualifierByteBuffer();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getQualifierPosition() {
|
||||
return cell.getQualifierPosition();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ByteBuffer getValueByteBuffer() {
|
||||
if (lenAsVal) {
|
||||
return ByteBuffer.wrap(Bytes.toBytes(cell.getValueLength()));
|
||||
} else {
|
||||
return HConstants.EMPTY_BYTE_BUFFER;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getValuePosition() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ByteBuffer getTagsByteBuffer() {
|
||||
return HConstants.EMPTY_BYTE_BUFFER;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getTagsPosition() {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,127 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.filter;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.hbase.CellUtil;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.KeyValue.Type;
|
||||
import org.apache.hadoop.hbase.KeyValueUtil;
|
||||
import org.apache.hadoop.hbase.TestCellUtil.ByteBufferedCellImpl;
|
||||
import org.apache.hadoop.hbase.filter.KeyOnlyFilter.KeyOnlyByteBufferedCell;
|
||||
import org.apache.hadoop.hbase.filter.KeyOnlyFilter.KeyOnlyCell;
|
||||
import org.apache.hadoop.hbase.testclassification.MiscTests;
|
||||
import org.apache.hadoop.hbase.testclassification.SmallTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
import org.junit.runners.Parameterized.Parameters;
|
||||
|
||||
@Category({ MiscTests.class, SmallTests.class })
|
||||
@RunWith(Parameterized.class)
|
||||
public class TestKeyOnlyFilter {
|
||||
|
||||
private final boolean lenAsVal;
|
||||
|
||||
@Parameters
|
||||
public static Collection<Object[]> parameters() {
|
||||
List<Object[]> paramList = new ArrayList<Object[]>();
|
||||
{
|
||||
paramList.add(new Object[] { false });
|
||||
paramList.add(new Object[] { true });
|
||||
}
|
||||
return paramList;
|
||||
}
|
||||
|
||||
public TestKeyOnlyFilter(boolean lenAsVal) {
|
||||
this.lenAsVal = lenAsVal;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testKeyOnly() throws Exception {
|
||||
byte[] r = Bytes.toBytes("row1");
|
||||
byte[] f = Bytes.toBytes("cf1");
|
||||
byte[] q = Bytes.toBytes("qual1");
|
||||
byte[] v = Bytes.toBytes("val1");
|
||||
byte[] tags = Bytes.toBytes("tag1");
|
||||
KeyValue kv = new KeyValue(r, f, q, 0, q.length, 1234L, Type.Put, v, 0,
|
||||
v.length, tags);
|
||||
|
||||
ByteBuffer buffer = ByteBuffer.wrap(kv.getBuffer());
|
||||
ByteBufferedCellImpl bbCell = new ByteBufferedCellImpl(buffer, 0,
|
||||
buffer.remaining());
|
||||
|
||||
// KV format: <keylen:4><valuelen:4><key:keylen><value:valuelen>
|
||||
// Rebuild as: <keylen:4><0:4><key:keylen>
|
||||
int dataLen = lenAsVal ? Bytes.SIZEOF_INT : 0;
|
||||
int keyOffset = (2 * Bytes.SIZEOF_INT);
|
||||
int keyLen = KeyValueUtil.keyLength(kv);
|
||||
byte[] newBuffer = new byte[keyLen + keyOffset + dataLen];
|
||||
Bytes.putInt(newBuffer, 0, keyLen);
|
||||
Bytes.putInt(newBuffer, Bytes.SIZEOF_INT, dataLen);
|
||||
KeyValueUtil.appendKeyTo(kv, newBuffer, keyOffset);
|
||||
if (lenAsVal) {
|
||||
Bytes.putInt(newBuffer, newBuffer.length - dataLen, kv.getValueLength());
|
||||
}
|
||||
KeyValue KeyOnlyKeyValue = new KeyValue(newBuffer);
|
||||
|
||||
KeyOnlyCell keyOnlyCell = new KeyOnlyCell(kv, lenAsVal);
|
||||
KeyOnlyByteBufferedCell keyOnlyByteBufferedCell = new KeyOnlyByteBufferedCell(
|
||||
bbCell, lenAsVal);
|
||||
|
||||
assertTrue(CellUtil.matchingRows(KeyOnlyKeyValue, keyOnlyCell));
|
||||
assertTrue(CellUtil.matchingRows(KeyOnlyKeyValue, keyOnlyByteBufferedCell));
|
||||
|
||||
assertTrue(CellUtil.matchingFamily(KeyOnlyKeyValue, keyOnlyCell));
|
||||
assertTrue(CellUtil
|
||||
.matchingFamily(KeyOnlyKeyValue, keyOnlyByteBufferedCell));
|
||||
|
||||
assertTrue(CellUtil.matchingQualifier(KeyOnlyKeyValue, keyOnlyCell));
|
||||
assertTrue(CellUtil.matchingQualifier(KeyOnlyKeyValue,
|
||||
keyOnlyByteBufferedCell));
|
||||
|
||||
assertTrue(CellUtil.matchingValue(KeyOnlyKeyValue, keyOnlyCell));
|
||||
assertTrue(KeyOnlyKeyValue.getValueLength() == keyOnlyByteBufferedCell
|
||||
.getValueLength());
|
||||
if (keyOnlyByteBufferedCell.getValueLength() > 0) {
|
||||
assertTrue(CellUtil.matchingValue(KeyOnlyKeyValue,
|
||||
keyOnlyByteBufferedCell));
|
||||
}
|
||||
|
||||
assertTrue(KeyOnlyKeyValue.getTimestamp() == keyOnlyCell.getTimestamp());
|
||||
assertTrue(KeyOnlyKeyValue.getTimestamp() == keyOnlyByteBufferedCell
|
||||
.getTimestamp());
|
||||
|
||||
assertTrue(KeyOnlyKeyValue.getTypeByte() == keyOnlyCell.getTypeByte());
|
||||
assertTrue(KeyOnlyKeyValue.getTypeByte() == keyOnlyByteBufferedCell
|
||||
.getTypeByte());
|
||||
|
||||
assertTrue(KeyOnlyKeyValue.getTagsLength() == keyOnlyCell.getTagsLength());
|
||||
assertTrue(KeyOnlyKeyValue.getTagsLength() == keyOnlyByteBufferedCell
|
||||
.getTagsLength());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1660,7 +1660,7 @@ public class TestFilter {
|
|||
int i = 0;
|
||||
for (boolean done = true; done; i++) {
|
||||
done = scanner.next(results);
|
||||
Arrays.sort(results.toArray(new KeyValue[results.size()]),
|
||||
Arrays.sort(results.toArray(new Cell[results.size()]),
|
||||
CellComparator.COMPARATOR);
|
||||
LOG.info("counter=" + i + ", " + results);
|
||||
if (results.isEmpty()) break;
|
||||
|
|
@ -1682,7 +1682,7 @@ public class TestFilter {
|
|||
int i = 0;
|
||||
for (boolean done = true; done; i++) {
|
||||
done = scanner.next(results);
|
||||
Arrays.sort(results.toArray(new KeyValue[results.size()]),
|
||||
Arrays.sort(results.toArray(new Cell[results.size()]),
|
||||
CellComparator.COMPARATOR);
|
||||
LOG.info("counter=" + i + ", " + results);
|
||||
if(results.isEmpty()) break;
|
||||
|
|
@ -1704,7 +1704,7 @@ public class TestFilter {
|
|||
int idx = 0;
|
||||
for (boolean done = true; done; row++) {
|
||||
done = scanner.next(results);
|
||||
Arrays.sort(results.toArray(new KeyValue[results.size()]),
|
||||
Arrays.sort(results.toArray(new Cell[results.size()]),
|
||||
CellComparator.COMPARATOR);
|
||||
if(results.isEmpty()) break;
|
||||
assertTrue("Scanned too many keys! Only expected " + kvs.length +
|
||||
|
|
@ -1735,7 +1735,7 @@ public class TestFilter {
|
|||
int idx = 0;
|
||||
for (boolean more = true; more; row++) {
|
||||
more = scanner.next(results);
|
||||
Arrays.sort(results.toArray(new KeyValue[results.size()]),
|
||||
Arrays.sort(results.toArray(new Cell[results.size()]),
|
||||
CellComparator.COMPARATOR);
|
||||
if(results.isEmpty()) break;
|
||||
assertTrue("Scanned too many keys! Only expected " + kvs.length +
|
||||
|
|
|
|||
Loading…
Reference in New Issue