Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-9959 Remove some array copy - server side 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1543434 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
nkeywal 2013-11-19 14:01:30 +00:00
parent b2582665f1
commit dc8ecd9a9a
10 changed files with 225 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
View File

@ -93,6 +93,7 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
public static final String BLOOMFILTER = "BLOOMFILTER";
public static final String FOREVER = "FOREVER";
public static final String REPLICATION_SCOPE = "REPLICATION_SCOPE";
public static final byte[] REPLICATION_SCOPE_BYTES = Bytes.toBytes(REPLICATION_SCOPE);
public static final String MIN_VERSIONS = "MIN_VERSIONS";
public static final String KEEP_DELETED_CELLS = "KEEP_DELETED_CELLS";
public static final String COMPRESS_TAGS = "COMPRESS_TAGS";
@ -823,9 +824,9 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
* @return the scope tag
*/
public int getScope() {
String value = getValue(REPLICATION_SCOPE);
byte[] value = getValue(REPLICATION_SCOPE_BYTES);
if (value != null) {
return Integer.valueOf(value).intValue();
return Integer.valueOf(Bytes.toString(value));
}
return DEFAULT_REPLICATION_SCOPE;
}

27
hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
View File

@ -18,7 +18,9 @@
package org.apache.hadoop.hbase.client;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -123,6 +125,18 @@ public abstract class Mutation extends OperationWithAttributes implements Row, C
return kvWithTag;
}
/*
* Create a KeyValue with this objects row key and the Put identifier.
*
* @return a KeyValue with this objects row key and the Put identifier.
*/
KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value,
Tag[] tags) {
return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length,
family, 0, family == null ? 0 : family.length,
qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null);
}
/**
* Compile the column family (i.e. schema) information
* into a Map. Useful for parsing and aggregation by debugging,
@ -407,4 +421,17 @@ public abstract class Mutation extends OperationWithAttributes implements Row, C
}
return row;
}
static void checkRow(ByteBuffer row) {
if (row == null) {
throw new IllegalArgumentException("Row buffer is null");
}
if (row.remaining() == 0) {
throw new IllegalArgumentException("Row length is 0");
}
if (row.remaining() > HConstants.MAX_ROW_LENGTH) {
throw new IllegalArgumentException("Row length " + row.remaining() + " is > " +
HConstants.MAX_ROW_LENGTH);
}
}
}

63
hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java
View File

@ -20,6 +20,7 @@
package org.apache.hadoop.hbase.client;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -73,6 +74,27 @@ public class Put extends Mutation implements HeapSize, Comparable<Row> {
this(rowArray, rowOffset, rowLength, HConstants.LATEST_TIMESTAMP);
}
/**
* @param row row key; we make a copy of what we are passed to keep local.
* @param ts timestamp
*/
public Put(ByteBuffer row, long ts) {
if (ts < 0) {
throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + ts);
}
checkRow(row);
this.row = new byte[row.remaining()];
row.get(this.row);
this.ts = ts;
}
/**
* @param row row key; we make a copy of what we are passed to keep local.
*/
public Put(ByteBuffer row) {
this(row, HConstants.LATEST_TIMESTAMP);
}
/**
* We make a copy of the passed in row key to keep local.
* @param rowArray
@ -152,6 +174,47 @@ public class Put extends Mutation implements HeapSize, Comparable<Row> {
return this;
}
/**
* Add the specified column and value, with the specified timestamp as
* its version to this Put operation.
* @param family family name
* @param qualifier column qualifier
* @param ts version timestamp
* @param value column value
* @param tag the tags
* @return this
*/
public Put add(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value, Tag[] tag) {
if (ts < 0) {
throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + ts);
}
List<Cell> list = getCellList(family);
KeyValue kv = createPutKeyValue(family, qualifier, ts, value, tag);
list.add(kv);
familyMap.put(kv.getFamily(), list);
return this;
}
/**
* Add the specified column and value, with the specified timestamp as
* its version to this Put operation.
* @param family family name
* @param qualifier column qualifier
* @param ts version timestamp
* @param value column value
* @return this
*/
public Put add(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value) {
if (ts < 0) {
throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + ts);
}
List<Cell> list = getCellList(family);
KeyValue kv = createPutKeyValue(family, qualifier, ts, value, null);
list.add(kv);
familyMap.put(kv.getFamily(), list);
return this;
}
/**
* Add the specified KeyValue to this Put operation. Operation assumes that
* the passed KeyValue is immutable and its backing array will not be modified

16
hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
View File

@ -27,6 +27,7 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@ -481,7 +482,6 @@ public final class ProtobufUtil {
// TODO: Server-side at least why do we convert back to the Client types? Why not just pb it?
MutationType type = proto.getMutateType();
assert type == MutationType.PUT: type.name();
byte [] row = proto.hasRow()? proto.getRow().toByteArray(): null;
long timestamp = proto.hasTimestamp()? proto.getTimestamp(): HConstants.LATEST_TIMESTAMP;
Put put = null;
int cellCount = proto.hasAssociatedCellCount()? proto.getAssociatedCellCount(): 0;
@ -503,17 +503,23 @@ public final class ProtobufUtil {
put.add(KeyValueUtil.ensureKeyValue(cell));
}
} else {
put = new Put(row, timestamp);
if (proto.hasRow()) {
put = new Put(proto.getRow().asReadOnlyByteBuffer(), timestamp);
} else {
throw new IllegalArgumentException("row cannot be null");
}
// The proto has the metadata and the data itself
for (ColumnValue column: proto.getColumnValueList()) {
byte[] family = column.getFamily().toByteArray();
for (QualifierValue qv: column.getQualifierValueList()) {
byte[] qualifier = qv.getQualifier().toByteArray();
if (!qv.hasValue()) {
throw new DoNotRetryIOException(
"Missing required field: qualifer value");
"Missing required field: qualifier value");
}
byte[] value = qv.getValue().toByteArray();
ByteBuffer qualifier =
qv.hasQualifier() ? qv.getQualifier().asReadOnlyByteBuffer() : null;
ByteBuffer value =
qv.hasValue() ? qv.getValue().asReadOnlyByteBuffer() : null;
long ts = timestamp;
if (qv.hasTimestamp()) {
ts = qv.getTimestamp();

52
hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestOperation.java
View File

@ -21,10 +21,16 @@ package org.apache.hadoop.hbase.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.SmallTests;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@ -368,5 +374,51 @@ public class TestOperation {
Bytes.toStringBinary(QUALIFIER), kvMap.get("qualifier"));
}
@Test
public void testPutCreationWithByteBuffer() {
Put p = new Put(ROW);
List<Cell> c = p.get(FAMILY, QUALIFIER);
Assert.assertEquals(0, c.size());
Assert.assertEquals(HConstants.LATEST_TIMESTAMP, p.getTimeStamp());
p.add(FAMILY, ByteBuffer.wrap(QUALIFIER), 1984L, ByteBuffer.wrap(VALUE));
c = p.get(FAMILY, QUALIFIER);
Assert.assertEquals(1, c.size());
Assert.assertEquals(1984L, c.get(0).getTimestamp());
Assert.assertArrayEquals(VALUE, CellUtil.cloneValue(c.get(0)));
Assert.assertEquals(HConstants.LATEST_TIMESTAMP, p.getTimeStamp());
Assert.assertEquals(0, KeyValue.COMPARATOR.compare(c.get(0), new KeyValue(c.get(0))));
p = new Put(ROW);
p.add(FAMILY, ByteBuffer.wrap(QUALIFIER), 2013L, null);
c = p.get(FAMILY, QUALIFIER);
Assert.assertEquals(1, c.size());
Assert.assertEquals(2013L, c.get(0).getTimestamp());
Assert.assertArrayEquals(new byte[]{}, CellUtil.cloneValue(c.get(0)));
Assert.assertEquals(HConstants.LATEST_TIMESTAMP, p.getTimeStamp());
Assert.assertEquals(0, KeyValue.COMPARATOR.compare(c.get(0), new KeyValue(c.get(0))));
p = new Put(ByteBuffer.wrap(ROW));
p.add(FAMILY, ByteBuffer.wrap(QUALIFIER), 2001L, null);
c = p.get(FAMILY, QUALIFIER);
Assert.assertEquals(1, c.size());
Assert.assertEquals(2001L, c.get(0).getTimestamp());
Assert.assertArrayEquals(new byte[]{}, CellUtil.cloneValue(c.get(0)));
Assert.assertArrayEquals(ROW, CellUtil.cloneRow(c.get(0)));
Assert.assertEquals(HConstants.LATEST_TIMESTAMP, p.getTimeStamp());
Assert.assertEquals(0, KeyValue.COMPARATOR.compare(c.get(0), new KeyValue(c.get(0))));
p = new Put(ByteBuffer.wrap(ROW), 1970L);
p.add(FAMILY, ByteBuffer.wrap(QUALIFIER), 2001L, null);
c = p.get(FAMILY, QUALIFIER);
Assert.assertEquals(1, c.size());
Assert.assertEquals(2001L, c.get(0).getTimestamp());
Assert.assertArrayEquals(new byte[]{}, CellUtil.cloneValue(c.get(0)));
Assert.assertArrayEquals(ROW, CellUtil.cloneRow(c.get(0)));
Assert.assertEquals(1970L, p.getTimeStamp());
Assert.assertEquals(0, KeyValue.COMPARATOR.compare(c.get(0), new KeyValue(c.get(0))));
}
}

55
hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
View File

@ -143,6 +143,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable {
public static final int KEYVALUE_WITH_TAGS_INFRASTRUCTURE_SIZE = ROW_OFFSET + TAGS_LENGTH_SIZE;
/**
* Computes the number of bytes that a <code>KeyValue</code> instance with the provided
* characteristics would take up for its underlying data structure.
@ -430,7 +431,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable {
public KeyValue(final byte[] row, final byte[] family,
final byte[] qualifier, final long timestamp, final byte[] value,
final Tag[] tags) {
this(row, family, qualifier, timestamp, value, Arrays.asList(tags));
this(row, family, qualifier, timestamp, value, tags != null ? Arrays.asList(tags) : null);
}
/**
@ -718,6 +719,17 @@ public class KeyValue implements Cell, HeapSize, Cloneable {
this.offset = 0;
}
public KeyValue(byte[] row, int roffset, int rlength,
byte[] family, int foffset, int flength,
ByteBuffer qualifier, long ts, Type type, ByteBuffer value, List<Tag> tags) {
this.bytes = createByteArray(row, roffset, rlength, family, foffset, flength,
qualifier, 0, qualifier == null ? 0 : qualifier.remaining(), ts, type,
value, 0, value == null ? 0 : value.remaining(), tags);
this.length = bytes.length;
this.offset = 0;
}
public KeyValue(Cell c) {
this(c.getRowArray(), c.getRowOffset(), (int)c.getRowLength(),
c.getFamilyArray(), c.getFamilyOffset(), (int)c.getFamilyLength(),
@ -790,17 +802,13 @@ public class KeyValue implements Cell, HeapSize, Cloneable {
* @param rlength row length
* @param family family name
* @param flength family length
* @param qualifier column qualifier
* @param qlength qualifier length
* @param value column value
* @param vlength value length
*
* @throws IllegalArgumentException an illegal value was passed
*/
private static void checkParameters(final byte [] row, final int rlength,
final byte [] family, int flength,
final byte [] qualifier, int qlength,
final byte [] value, int vlength)
final byte [] family, int flength, int qlength, int vlength)
throws IllegalArgumentException {
if (rlength > Short.MAX_VALUE) {
throw new IllegalArgumentException("Row > " + Short.MAX_VALUE);
@ -814,7 +822,6 @@ public class KeyValue implements Cell, HeapSize, Cloneable {
throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE);
}
// Qualifier length
qlength = qualifier == null ? 0 : qlength;
if (qlength > Integer.MAX_VALUE - rlength - flength) {
throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
}
@ -825,7 +832,6 @@ public class KeyValue implements Cell, HeapSize, Cloneable {
Integer.MAX_VALUE);
}
// Value length
vlength = value == null? 0 : vlength;
if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON
throw new IllegalArgumentException("Value length " + vlength + " > " +
HConstants.MAXIMUM_VALUE_LENGTH);
@ -864,7 +870,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable {
final long timestamp, final Type type,
final byte [] value, final int voffset, int vlength, Tag[] tags) {
checkParameters(row, rlength, family, flength, qualifier, qlength, value, vlength);
checkParameters(row, rlength, family, flength, qlength, vlength);
// Calculate length of tags area
int tagsLength = 0;
@ -941,7 +947,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable {
final byte [] value, final int voffset,
int vlength, byte[] tags, int tagsOffset, int tagsLength) {
checkParameters(row, rlength, family, flength, qualifier, qlength, value, vlength);
checkParameters(row, rlength, family, flength, qlength, vlength);
checkForTagsLength(tagsLength);
// Allocate right-sized byte array.
int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
@ -972,14 +978,18 @@ public class KeyValue implements Cell, HeapSize, Cloneable {
}
return bytes;
}
/**
* @param qualifier can be a ByteBuffer or a byte[], or null.
* @param value can be a ByteBuffer or a byte[], or null.
*/
private static byte [] createByteArray(final byte [] row, final int roffset,
final int rlength, final byte [] family, final int foffset, int flength,
final byte [] qualifier, final int qoffset, int qlength,
final Object qualifier, final int qoffset, int qlength,
final long timestamp, final Type type,
final byte [] value, final int voffset, int vlength, List<Tag> tags) {
final Object value, final int voffset, int vlength, List<Tag> tags) {
checkParameters(row, rlength, family, flength, qualifier, qlength, value, vlength);
checkParameters(row, rlength, family, flength, qlength, vlength);
// Calculate length of tags area
int tagsLength = 0;
@ -1005,13 +1015,21 @@ public class KeyValue implements Cell, HeapSize, Cloneable {
if(flength != 0) {
pos = Bytes.putBytes(bytes, pos, family, foffset, flength);
}
if(qlength != 0) {
pos = Bytes.putBytes(bytes, pos, qualifier, qoffset, qlength);
if (qlength > 0) {
if (qualifier instanceof ByteBuffer) {
pos = Bytes.putByteBuffer(bytes, pos, (ByteBuffer) qualifier);
} else {
pos = Bytes.putBytes(bytes, pos, (byte[]) qualifier, qoffset, qlength);
}
}
pos = Bytes.putLong(bytes, pos, timestamp);
pos = Bytes.putByte(bytes, pos, type.getCode());
if (value != null && value.length > 0) {
pos = Bytes.putBytes(bytes, pos, value, voffset, vlength);
if (vlength > 0) {
if (value instanceof ByteBuffer) {
pos = Bytes.putByteBuffer(bytes, pos, (ByteBuffer) value);
} else {
pos = Bytes.putBytes(bytes, pos, (byte[]) value, voffset, vlength);
}
}
// Add the tags after the value part
if (tagsLength > 0) {
@ -1023,7 +1041,6 @@ public class KeyValue implements Cell, HeapSize, Cloneable {
return bytes;
}
/**
* Needed doing 'contains' on List. Only compares the key portion, not the value.
*/

13
hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
View File

@ -305,6 +305,19 @@ public class Bytes {
return offset + 1;
}
/**
* Add the whole content of the ByteBuffer to the bytes arrays. The ByteBuffer is modified.
* @param bytes the byte array
* @param offset position in the array
* @param buf ByteBuffer to write out
* @return incremented offset
*/
public static int putByteBuffer(byte[] bytes, int offset, ByteBuffer buf) {
int len = buf.remaining();
buf.get(bytes, offset, len);
return offset + len;
}
/**
* Returns a new byte array, copied from the given {@code buf},
* from the index 0 (inclusive) to the limit (exclusive),

10
hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
View File

@ -463,5 +463,15 @@ public class TestBytes extends TestCase {
assertFalse(array[i] == 0);
}
}
public void testPutBuffer() {
byte[] b = new byte[100];
for (byte i = 0; i < 100; i++) {
Bytes.putByteBuffer(b, i, ByteBuffer.wrap(new byte[]{i}));
}
for (byte i = 0; i < 100; i++) {
Assert.assertEquals(i, b[i]);
}
}
}

3
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -3305,11 +3305,12 @@ public class HRegionServer implements ClientProtos.ClientService.BlockingInterfa
// this will contain all the cells that we need to return. It's created later, if needed.
List<CellScannable> cellsToReturn = null;
MultiResponse.Builder responseBuilder = MultiResponse.newBuilder();
RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder();
for (RegionAction regionAction : request.getRegionActionList()) {
this.requestCount.add(regionAction.getActionCount());
RegionActionResult.Builder regionActionResultBuilder = RegionActionResult.newBuilder();
HRegion region;
regionActionResultBuilder.clear();
try {
region = getRegion(regionAction.getRegion());
} catch (IOException e) {

16
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
View File

@ -1927,17 +1927,17 @@ public class TestHRegion {
// Putting data in Region
Put put = null;
put = new Put(row1);
put.add(fam1, null, ts, null);
put.add(fam2, null, ts, null);
put.add(fam3, null, ts, null);
put.add(fam4, null, ts, null);
put.add(fam1, (byte[]) null, ts, null);
put.add(fam2, (byte[]) null, ts, null);
put.add(fam3, (byte[]) null, ts, null);
put.add(fam4, (byte[]) null, ts, null);
region.put(put);
put = new Put(row2);
put.add(fam1, null, ts, null);
put.add(fam2, null, ts, null);
put.add(fam3, null, ts, null);
put.add(fam4, null, ts, null);
put.add(fam1, (byte[]) null, ts, null);
put.add(fam2, (byte[]) null, ts, null);
put.add(fam3, (byte[]) null, ts, null);
put.add(fam4, (byte[]) null, ts, null);
region.put(put);
Scan scan = new Scan();