diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java index c59b947f52d..cd663129bc9 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java @@ -296,8 +296,8 @@ public class ByteBufferKeyValue extends ByteBufferCell implements ExtendedCell { } @Override - public void setTimestamp(byte[] ts, int tsOffset) throws IOException { - ByteBufferUtils.copyFromArrayToBuffer(this.buf, this.getTimestampOffset(), ts, tsOffset, + public void setTimestamp(byte[] ts) throws IOException { + ByteBufferUtils.copyFromArrayToBuffer(this.buf, this.getTimestampOffset(), ts, 0, Bytes.SIZEOF_LONG); } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index a74905708f6..75225b483e7 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -34,16 +34,16 @@ import java.util.List; import java.util.Map.Entry; import java.util.NavigableMap; import java.util.Optional; - import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.io.HeapSize; -import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceAudience.Private; +import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting; + /** * Utility methods helpful for slinging {@link Cell} instances. Some methods below are for internal * use only and are marked InterfaceAudience.Private at the method level. Note that all such methods @@ -505,7 +505,7 @@ public final class CellUtil { } /** - * Marked as audience Private as of 1.2.0. + * Marked as audience Private as of 1.2.0. * Creating a Cell with tags and a memstoreTS/mvcc is an * internal implementation detail not for public use. * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. Use @@ -529,7 +529,7 @@ public final class CellUtil { } /** - * Marked as audience Private as of 1.2.0. + * Marked as audience Private as of 1.2.0. * Creating a Cell with tags is an internal implementation detail not for public use. * @deprecated As of release 2.0.0, this will be removed in HBase 3.0.0. Use * {@link ExtendedCellBuilder} instead @@ -1169,34 +1169,24 @@ public final class CellUtil { * sequenceid is an internal implementation detail not for general public use. * @param cell * @param seqId - * @throws IOException when the passed cell is not of type {@link SettableSequenceId} + * @throws IOException when the passed cell is not of type {@link ExtendedCell} * @deprecated As of HBase-2.0. Will be removed in HBase-3.0 */ @Deprecated public static void setSequenceId(Cell cell, long seqId) throws IOException { - if (cell instanceof SettableSequenceId) { - ((SettableSequenceId) cell).setSequenceId(seqId); - } else { - throw new IOException(new UnsupportedOperationException( - "Cell is not of type " + SettableSequenceId.class.getName())); - } + PrivateCellUtil.setSequenceId(cell, seqId); } /** * Sets the given timestamp to the cell. * @param cell * @param ts - * @throws IOException when the passed cell is not of type {@link SettableTimestamp} + * @throws IOException when the passed cell is not of type {@link ExtendedCell} * @deprecated As of HBase-2.0. Will be removed in HBase-3.0 */ @Deprecated public static void setTimestamp(Cell cell, long ts) throws IOException { - if (cell instanceof SettableTimestamp) { - ((SettableTimestamp) cell).setTimestamp(ts); - } else { - throw new IOException(new UnsupportedOperationException( - "Cell is not of type " + SettableTimestamp.class.getName())); - } + PrivateCellUtil.setTimestamp(cell, ts); } /** @@ -1204,17 +1194,12 @@ public final class CellUtil { * @param cell * @param ts buffer containing the timestamp value * @param tsOffset offset to the new timestamp - * @throws IOException when the passed cell is not of type {@link SettableTimestamp} + * @throws IOException when the passed cell is not of type {@link ExtendedCell} * @deprecated As of HBase-2.0. Will be removed in HBase-3.0 */ @Deprecated public static void setTimestamp(Cell cell, byte[] ts, int tsOffset) throws IOException { - if (cell instanceof SettableTimestamp) { - ((SettableTimestamp) cell).setTimestamp(ts, tsOffset); - } else { - throw new IOException(new UnsupportedOperationException( - "Cell is not of type " + SettableTimestamp.class.getName())); - } + PrivateCellUtil.setTimestamp(cell, Bytes.toLong(ts, tsOffset)); } /** @@ -1223,16 +1208,12 @@ public final class CellUtil { * @param cell * @param ts * @return True if cell timestamp is modified. - * @throws IOException when the passed cell is not of type {@link SettableTimestamp} + * @throws IOException when the passed cell is not of type {@link ExtendedCell} * @deprecated As of HBase-2.0. Will be removed in HBase-3.0 */ @Deprecated public static boolean updateLatestStamp(Cell cell, long ts) throws IOException { - if (cell.getTimestamp() == HConstants.LATEST_TIMESTAMP) { - PrivateCellUtil.setTimestamp(cell, ts); - return true; - } - return false; + return PrivateCellUtil.updateLatestStamp(cell, ts); } /** @@ -1242,16 +1223,12 @@ public final class CellUtil { * @param ts buffer containing the timestamp value * @param tsOffset offset to the new timestamp * @return True if cell timestamp is modified. - * @throws IOException when the passed cell is not of type {@link SettableTimestamp} + * @throws IOException when the passed cell is not of type {@link ExtendedCell} * @deprecated As of HBase-2.0. Will be removed in HBase-3.0 */ @Deprecated public static boolean updateLatestStamp(Cell cell, byte[] ts, int tsOffset) throws IOException { - if (cell.getTimestamp() == HConstants.LATEST_TIMESTAMP) { - PrivateCellUtil.setTimestamp(cell, ts, tsOffset); - return true; - } - return false; + return PrivateCellUtil.updateLatestStamp(cell, Bytes.toLong(ts, tsOffset)); } /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ExtendedCell.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ExtendedCell.java index b5ce0958074..73fbf674dc1 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ExtendedCell.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ExtendedCell.java @@ -20,21 +20,18 @@ package org.apache.hadoop.hbase; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; - import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.yetus.audience.InterfaceAudience; /** * Extension to {@link Cell} with server side required functions. Server side Cell implementations * must implement this. - * @see SettableSequenceId - * @see SettableTimestamp */ @InterfaceAudience.Private -public interface ExtendedCell extends RawCell, SettableSequenceId, SettableTimestamp, HeapSize, - Cloneable { +public interface ExtendedCell extends RawCell, HeapSize, Cloneable { - public static int CELL_NOT_BASED_ON_CHUNK = -1; + int CELL_NOT_BASED_ON_CHUNK = -1; /** * Write this cell to an OutputStream in a {@link KeyValue} format. *
KeyValue format
@@ -48,7 +45,29 @@ public interface ExtendedCell extends RawCell, SettableSequenceId, SettableTimes * @throws IOException */ // TODO remove the boolean param once HBASE-16706 is done. - int write(OutputStream out, boolean withTags) throws IOException; + default int write(OutputStream out, boolean withTags) throws IOException { + // Key length and then value length + ByteBufferUtils.putInt(out, KeyValueUtil.keyLength(this)); + ByteBufferUtils.putInt(out, getValueLength()); + + // Key + PrivateCellUtil.writeFlatKey(this, out); + + // Value + out.write(getValueArray()); + + // Tags length and tags byte array + if (withTags && getTagsLength() > 0) { + // Tags length + out.write((byte)(0xff & (getTagsLength() >> 8))); + out.write((byte)(0xff & getTagsLength())); + + // Tags byte array + out.write(getTagsArray(), getTagsOffset(), getTagsLength()); + } + + return getSerializedSize(withTags); + } /** * @param withTags Whether to write tags. @@ -60,20 +79,28 @@ public interface ExtendedCell extends RawCell, SettableSequenceId, SettableTimes * <tags> */ // TODO remove the boolean param once HBASE-16706 is done. - int getSerializedSize(boolean withTags); + default int getSerializedSize(boolean withTags) { + return KeyValueUtil.length(getRowLength(), getFamilyLength(), getQualifierLength(), + getValueLength(), getTagsLength(), withTags); + } /** * Write this Cell into the given buf's offset in a {@link KeyValue} format. * @param buf The buffer where to write the Cell. * @param offset The offset within buffer, to write the Cell. */ - void write(ByteBuffer buf, int offset); + default void write(ByteBuffer buf, int offset) { + KeyValueUtil.appendTo(this, buf, offset, true); + } /** * Does a deep copy of the contents to a new memory area and returns it as a new cell. * @return The deep cloned cell */ - ExtendedCell deepClone(); + default ExtendedCell deepClone() { + // When being added to the memstore, deepClone() is called and KeyValue has less heap overhead. + return new KeyValue(this); + } /** * Extracts the id of the backing bytebuffer of this cell if it was obtained from fixed sized @@ -83,4 +110,22 @@ public interface ExtendedCell extends RawCell, SettableSequenceId, SettableTimes default int getChunkId() { return CELL_NOT_BASED_ON_CHUNK; } + + /** + * Sets with the given seqId. + * @param seqId sequence ID + */ + void setSequenceId(long seqId) throws IOException; + + /** + * Sets with the given timestamp. + * @param ts timestamp + */ + void setTimestamp(long ts) throws IOException; + + /** + * Sets with the given timestamp. + * @param ts buffer containing the timestamp value + */ + void setTimestamp(byte[] ts) throws IOException; } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java index 78ad5783946..4049e3dd36b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/IndividualBytesFieldCell.java @@ -18,16 +18,10 @@ package org.apache.hadoop.hbase; -import org.apache.yetus.audience.InterfaceAudience; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; - import org.apache.hadoop.hbase.util.ArrayUtils; -import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; +import org.apache.yetus.audience.InterfaceAudience; @InterfaceAudience.Private public class IndividualBytesFieldCell implements ExtendedCell { @@ -136,42 +130,6 @@ public class IndividualBytesFieldCell implements ExtendedCell { } } - @Override - public int write(OutputStream out, boolean withTags) throws IOException { - // Key length and then value length - ByteBufferUtils.putInt(out, KeyValueUtil.keyLength(this)); - ByteBufferUtils.putInt(out, getValueLength()); - - // Key - PrivateCellUtil.writeFlatKey(this, out); - - // Value - out.write(getValueArray()); - - // Tags length and tags byte array - if (withTags && getTagsLength() > 0) { - // Tags length - out.write((byte)(0xff & (getTagsLength() >> 8))); - out.write((byte)(0xff & getTagsLength())); - - // Tags byte array - out.write(tags, tagsOffset, tagsLength); - } - - return getSerializedSize(withTags); - } - - @Override - public void write(ByteBuffer buf, int offset) { - KeyValueUtil.appendTo(this, buf, offset, true); - } - - @Override - public int getSerializedSize(boolean withTags) { - return KeyValueUtil.length(getRowLength(), getFamilyLength(), getQualifierLength(), - getValueLength(), getTagsLength(), withTags); - } - private long heapOverhead() { return FIXED_OVERHEAD + ClassSize.ARRAY // row , can not be null @@ -181,12 +139,6 @@ public class IndividualBytesFieldCell implements ExtendedCell { + ((tags == null) ? 0 : ClassSize.ARRAY); // tags , can be null } - @Override - public ExtendedCell deepClone() { - // When being added to the memstore, deepClone() is called and KeyValue has less heap overhead. - return new KeyValue(this); - } - /** * Implement Cell interface */ @@ -320,9 +272,6 @@ public class IndividualBytesFieldCell implements ExtendedCell { return super.clone(); // only a shadow copy } - /** - * Implement SettableSequenceId interface - */ @Override public void setSequenceId(long seqId) { if (seqId < 0) { @@ -331,9 +280,6 @@ public class IndividualBytesFieldCell implements ExtendedCell { this.seqId = seqId; } - /** - * Implement SettableTimestamp interface - */ @Override public void setTimestamp(long ts) { if (ts < 0) { @@ -343,8 +289,8 @@ public class IndividualBytesFieldCell implements ExtendedCell { } @Override - public void setTimestamp(byte[] ts, int tsOffset) { - setTimestamp(Bytes.toLong(ts, tsOffset)); + public void setTimestamp(byte[] ts) { + setTimestamp(Bytes.toLong(ts, 0)); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java index fa027b4d5d5..727eede7046 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -1445,8 +1445,8 @@ public class KeyValue implements ExtendedCell { } @Override - public void setTimestamp(byte[] ts, int tsOffset) { - Bytes.putBytes(this.bytes, this.getTimestampOffset(), ts, tsOffset, Bytes.SIZEOF_LONG); + public void setTimestamp(byte[] ts) { + Bytes.putBytes(this.bytes, this.getTimestampOffset(), ts, 0, Bytes.SIZEOF_LONG); } //--------------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java index 1acb4904afe..79b8b314e2b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase; import static org.apache.hadoop.hbase.HConstants.EMPTY_BYTE_ARRAY; import static org.apache.hadoop.hbase.Tag.TAG_LENGTH_SIZE; +import com.google.common.annotations.VisibleForTesting; import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; @@ -30,7 +31,6 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Optional; - import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.filter.ByteArrayComparable; import org.apache.hadoop.hbase.io.HeapSize; @@ -43,8 +43,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.yetus.audience.InterfaceAudience; -import com.google.common.annotations.VisibleForTesting; - /** * Utility methods helpful slinging {@link Cell} instances. It has more powerful and * rich set of APIs than those in {@link CellUtil} for internal usage. @@ -260,19 +258,19 @@ public final class PrivateCellUtil { @Override public void setTimestamp(long ts) throws IOException { - // The incoming cell is supposed to be SettableTimestamp type. + // The incoming cell is supposed to be ExtendedCell type. PrivateCellUtil.setTimestamp(cell, ts); } @Override - public void setTimestamp(byte[] ts, int tsOffset) throws IOException { - // The incoming cell is supposed to be SettableTimestamp type. - PrivateCellUtil.setTimestamp(cell, ts, tsOffset); + public void setTimestamp(byte[] ts) throws IOException { + // The incoming cell is supposed to be ExtendedCell type. + PrivateCellUtil.setTimestamp(cell, ts); } @Override public void setSequenceId(long seqId) throws IOException { - // The incoming cell is supposed to be SettableSequenceId type. + // The incoming cell is supposed to be ExtendedCell type. PrivateCellUtil.setSequenceId(cell, seqId); } @@ -442,8 +440,8 @@ public final class PrivateCellUtil { } @Override - public void setTimestamp(byte[] ts, int tsOffset) throws IOException { - PrivateCellUtil.setTimestamp(this.cell, ts, tsOffset); + public void setTimestamp(byte[] ts) throws IOException { + PrivateCellUtil.setTimestamp(this.cell, ts); } @Override @@ -1251,13 +1249,23 @@ public final class PrivateCellUtil { * These cells are used in reseeks/seeks to improve the read performance. They are not real cells * that are returned back to the clients */ - private static abstract class EmptyCell implements Cell, SettableSequenceId { + private static abstract class EmptyCell implements ExtendedCell { @Override public void setSequenceId(long seqId) { // Fake cells don't need seqId, so leaving it as a noop. } + @Override + public void setTimestamp(long ts) { + // Fake cells can't be changed timestamp, so leaving it as a noop. + } + + @Override + public void setTimestamp(byte[] ts) { + // Fake cells can't be changed timestamp, so leaving it as a noop. + } + @Override public byte[] getRowArray() { return EMPTY_BYTE_ARRAY; @@ -1344,13 +1352,23 @@ public final class PrivateCellUtil { * that are returned back to the clients */ private static abstract class EmptyByteBufferCell extends ByteBufferCell - implements SettableSequenceId { + implements ExtendedCell { @Override public void setSequenceId(long seqId) { // Fake cells don't need seqId, so leaving it as a noop. } + @Override + public void setTimestamp(long ts) { + // Fake cells can't be changed timestamp, so leaving it as a noop. + } + + @Override + public void setTimestamp(byte[] ts) { + // Fake cells can't be changed timestamp, so leaving it as a noop. + } + @Override public byte[] getRowArray() { return CellUtil.cloneRow(this); @@ -1483,6 +1501,11 @@ public final class PrivateCellUtil { } private static class FirstOnRowCell extends EmptyCell { + private static final long FIXED_HEAPSIZE = + ClassSize.OBJECT // object + + ClassSize.REFERENCE // row array + + Bytes.SIZEOF_INT // row offset + + Bytes.SIZEOF_SHORT; // row length private final byte[] rowArray; private final int roffset; private final short rlength; @@ -1493,6 +1516,13 @@ public final class PrivateCellUtil { this.rlength = rlength; } + @Override + public long heapSize() { + return ClassSize.align(FIXED_HEAPSIZE) + // array overhead + + (rlength == 0 ? ClassSize.sizeOfByteArray(rlength) : rlength); + } + @Override public byte[] getRowArray() { return this.rowArray; @@ -1520,6 +1550,11 @@ public final class PrivateCellUtil { } private static class FirstOnRowByteBufferCell extends EmptyByteBufferCell { + private static final int FIXED_OVERHEAD = + ClassSize.OBJECT // object + + ClassSize.REFERENCE // row buffer + + Bytes.SIZEOF_INT // row offset + + Bytes.SIZEOF_SHORT; // row length private final ByteBuffer rowBuff; private final int roffset; private final short rlength; @@ -1530,6 +1565,14 @@ public final class PrivateCellUtil { this.rlength = rlength; } + @Override + public long heapSize() { + if (this.rowBuff.hasArray()) { + return ClassSize.align(FIXED_OVERHEAD + rlength); + } + return ClassSize.align(FIXED_OVERHEAD); + } + @Override public ByteBuffer getRowByteBuffer() { return this.rowBuff; @@ -1557,6 +1600,11 @@ public final class PrivateCellUtil { } private static class LastOnRowByteBufferCell extends EmptyByteBufferCell { + private static final int FIXED_OVERHEAD = + ClassSize.OBJECT // object + + ClassSize.REFERENCE // rowBuff + + Bytes.SIZEOF_INT // roffset + + Bytes.SIZEOF_SHORT; // rlength private final ByteBuffer rowBuff; private final int roffset; private final short rlength; @@ -1567,6 +1615,14 @@ public final class PrivateCellUtil { this.rlength = rlength; } + @Override + public long heapSize() { + if (this.rowBuff.hasArray()) { + return ClassSize.align(FIXED_OVERHEAD + rlength); + } + return ClassSize.align(FIXED_OVERHEAD); + } + @Override public ByteBuffer getRowByteBuffer() { return this.rowBuff; @@ -1594,6 +1650,11 @@ public final class PrivateCellUtil { } private static class FirstOnRowColByteBufferCell extends FirstOnRowByteBufferCell { + private static final int FIXED_OVERHEAD = + FirstOnRowByteBufferCell.FIXED_OVERHEAD + + ClassSize.REFERENCE * 2 // family buffer and column buffer + + Bytes.SIZEOF_INT * 3 // famOffset, colOffset, colLength + + Bytes.SIZEOF_BYTE; // famLength private final ByteBuffer famBuff; private final int famOffset; private final byte famLength; @@ -1613,6 +1674,19 @@ public final class PrivateCellUtil { this.colLength = colLength; } + @Override + public long heapSize() { + if (famBuff.hasArray() && colBuff.hasArray()) { + return ClassSize.align(FIXED_OVERHEAD + famLength + colLength); + } else if (famBuff.hasArray()) { + return ClassSize.align(FIXED_OVERHEAD + famLength); + } else if (colBuff.hasArray()) { + return ClassSize.align(FIXED_OVERHEAD + colLength); + } else { + return ClassSize.align(FIXED_OVERHEAD); + } + } + @Override public ByteBuffer getFamilyByteBuffer() { return this.famBuff; @@ -1645,6 +1719,11 @@ public final class PrivateCellUtil { } private static class FirstOnRowColCell extends FirstOnRowCell { + private static final long FIXED_HEAPSIZE = + FirstOnRowCell.FIXED_HEAPSIZE + + Bytes.SIZEOF_BYTE // flength + + Bytes.SIZEOF_INT * 3 // foffset, qoffset, qlength + + ClassSize.REFERENCE * 2; // fArray, qArray private final byte[] fArray; private final int foffset; private final byte flength; @@ -1663,6 +1742,14 @@ public final class PrivateCellUtil { this.qlength = qlength; } + @Override + public long heapSize() { + return ClassSize.align(FIXED_HEAPSIZE) + // array overhead + + (flength == 0 ? ClassSize.sizeOfByteArray(flength) : flength) + + (qlength == 0 ? ClassSize.sizeOfByteArray(qlength) : qlength); + } + @Override public byte[] getFamilyArray() { return this.fArray; @@ -1695,7 +1782,9 @@ public final class PrivateCellUtil { } private static class FirstOnRowColTSCell extends FirstOnRowColCell { - + private static final long FIXED_HEAPSIZE = + FirstOnRowColCell.FIXED_HEAPSIZE + + Bytes.SIZEOF_LONG; // ts private long ts; public FirstOnRowColTSCell(byte[] rArray, int roffset, short rlength, byte[] fArray, @@ -1708,10 +1797,17 @@ public final class PrivateCellUtil { public long getTimestamp() { return this.ts; } + + @Override + public long heapSize() { + return ClassSize.align(FIXED_HEAPSIZE); + } } private static class FirstOnRowColTSByteBufferCell extends FirstOnRowColByteBufferCell { - + private static final int FIXED_OVERHEAD = + FirstOnRowColByteBufferCell.FIXED_OVERHEAD + + Bytes.SIZEOF_LONG; // ts private long ts; public FirstOnRowColTSByteBufferCell(ByteBuffer rBuffer, int roffset, short rlength, @@ -1725,9 +1821,19 @@ public final class PrivateCellUtil { public long getTimestamp() { return this.ts; } + + @Override + public long heapSize() { + return ClassSize.align(FIXED_OVERHEAD + super.heapSize()); + } } private static class LastOnRowCell extends EmptyCell { + private static final long FIXED_OVERHEAD = + ClassSize.OBJECT // object + + ClassSize.REFERENCE // row array + + Bytes.SIZEOF_INT // row offset + + Bytes.SIZEOF_SHORT; // row length private final byte[] rowArray; private final int roffset; private final short rlength; @@ -1738,6 +1844,13 @@ public final class PrivateCellUtil { this.rlength = rlength; } + @Override + public long heapSize() { + return ClassSize.align(FIXED_OVERHEAD) + // array overhead + + (rlength == 0 ? ClassSize.sizeOfByteArray(rlength) : rlength); + } + @Override public byte[] getRowArray() { return this.rowArray; @@ -1765,6 +1878,10 @@ public final class PrivateCellUtil { } private static class LastOnRowColCell extends LastOnRowCell { + private static final long FIXED_OVERHEAD = LastOnRowCell.FIXED_OVERHEAD + + ClassSize.REFERENCE * 2 // fArray and qArray + + Bytes.SIZEOF_INT * 3 // foffset, qoffset, qlength + + Bytes.SIZEOF_BYTE; // flength private final byte[] fArray; private final int foffset; private final byte flength; @@ -1783,6 +1900,14 @@ public final class PrivateCellUtil { this.qlength = qlength; } + @Override + public long heapSize() { + return ClassSize.align(FIXED_OVERHEAD) + // array overhead + + (flength == 0 ? ClassSize.sizeOfByteArray(flength) : flength) + + (qlength == 0 ? ClassSize.sizeOfByteArray(qlength) : qlength); + } + @Override public byte[] getFamilyArray() { return this.fArray; @@ -1815,6 +1940,11 @@ public final class PrivateCellUtil { } private static class LastOnRowColByteBufferCell extends LastOnRowByteBufferCell { + private static final int FIXED_OVERHEAD = + LastOnRowByteBufferCell.FIXED_OVERHEAD + + ClassSize.REFERENCE * 2 // fBuffer and qBuffer + + Bytes.SIZEOF_INT * 3 // foffset, qoffset, qlength + + Bytes.SIZEOF_BYTE; // flength private final ByteBuffer fBuffer; private final int foffset; private final byte flength; @@ -1834,6 +1964,19 @@ public final class PrivateCellUtil { this.qlength = qlength; } + @Override + public long heapSize() { + if (fBuffer.hasArray() && qBuffer.hasArray()) { + return ClassSize.align(FIXED_OVERHEAD + flength + qlength); + } else if (fBuffer.hasArray()) { + return ClassSize.align(FIXED_OVERHEAD + flength); + } else if (qBuffer.hasArray()) { + return ClassSize.align(FIXED_OVERHEAD + qlength); + } else { + return ClassSize.align(FIXED_OVERHEAD); + } + } + @Override public ByteBuffer getFamilyByteBuffer() { return this.fBuffer; @@ -1866,6 +2009,11 @@ public final class PrivateCellUtil { } private static class FirstOnRowDeleteFamilyCell extends EmptyCell { + private static final int FIXED_OVERHEAD = + ClassSize.OBJECT // object + + ClassSize.REFERENCE * 2 // fBuffer and qBuffer + + Bytes.SIZEOF_INT * 3 // foffset, qoffset, qlength + + Bytes.SIZEOF_BYTE; // flength private final byte[] row; private final byte[] fam; @@ -1874,6 +2022,15 @@ public final class PrivateCellUtil { this.fam = fam; } + @Override + public long heapSize() { + return ClassSize.align(FIXED_OVERHEAD) + // array overhead + + (getRowLength() == 0 ? ClassSize.sizeOfByteArray(getRowLength()) : getRowLength()) + + (getFamilyLength() == 0 ? + ClassSize.sizeOfByteArray(getFamilyLength()) : getFamilyLength()); + } + @Override public byte[] getRowArray() { return this.row; @@ -2035,14 +2192,14 @@ public final class PrivateCellUtil { * sequenceid is an internal implementation detail not for general public use. * @param cell * @param seqId - * @throws IOException when the passed cell is not of type {@link SettableSequenceId} + * @throws IOException when the passed cell is not of type {@link ExtendedCell} */ public static void setSequenceId(Cell cell, long seqId) throws IOException { - if (cell instanceof SettableSequenceId) { - ((SettableSequenceId) cell).setSequenceId(seqId); + if (cell instanceof ExtendedCell) { + ((ExtendedCell) cell).setSequenceId(seqId); } else { throw new IOException(new UnsupportedOperationException( - "Cell is not of type " + SettableSequenceId.class.getName())); + "Cell is not of type " + ExtendedCell.class.getName())); } } @@ -2050,14 +2207,14 @@ public final class PrivateCellUtil { * Sets the given timestamp to the cell. * @param cell * @param ts - * @throws IOException when the passed cell is not of type {@link SettableTimestamp} + * @throws IOException when the passed cell is not of type {@link ExtendedCell} */ public static void setTimestamp(Cell cell, long ts) throws IOException { - if (cell instanceof SettableTimestamp) { - ((SettableTimestamp) cell).setTimestamp(ts); + if (cell instanceof ExtendedCell) { + ((ExtendedCell) cell).setTimestamp(ts); } else { throw new IOException(new UnsupportedOperationException( - "Cell is not of type " + SettableTimestamp.class.getName())); + "Cell is not of type " + ExtendedCell.class.getName())); } } @@ -2065,15 +2222,14 @@ public final class PrivateCellUtil { * Sets the given timestamp to the cell. * @param cell * @param ts buffer containing the timestamp value - * @param tsOffset offset to the new timestamp - * @throws IOException when the passed cell is not of type {@link SettableTimestamp} + * @throws IOException when the passed cell is not of type {@link ExtendedCell} */ - public static void setTimestamp(Cell cell, byte[] ts, int tsOffset) throws IOException { - if (cell instanceof SettableTimestamp) { - ((SettableTimestamp) cell).setTimestamp(ts, tsOffset); + public static void setTimestamp(Cell cell, byte[] ts) throws IOException { + if (cell instanceof ExtendedCell) { + ((ExtendedCell) cell).setTimestamp(ts); } else { throw new IOException(new UnsupportedOperationException( - "Cell is not of type " + SettableTimestamp.class.getName())); + "Cell is not of type " + ExtendedCell.class.getName())); } } @@ -2083,7 +2239,7 @@ public final class PrivateCellUtil { * @param cell * @param ts * @return True if cell timestamp is modified. - * @throws IOException when the passed cell is not of type {@link SettableTimestamp} + * @throws IOException when the passed cell is not of type {@link ExtendedCell} */ public static boolean updateLatestStamp(Cell cell, long ts) throws IOException { if (cell.getTimestamp() == HConstants.LATEST_TIMESTAMP) { @@ -2098,13 +2254,12 @@ public final class PrivateCellUtil { * {@link HConstants#LATEST_TIMESTAMP}. * @param cell * @param ts buffer containing the timestamp value - * @param tsOffset offset to the new timestamp * @return True if cell timestamp is modified. - * @throws IOException when the passed cell is not of type {@link SettableTimestamp} + * @throws IOException when the passed cell is not of type {@link ExtendedCell} */ - public static boolean updateLatestStamp(Cell cell, byte[] ts, int tsOffset) throws IOException { + public static boolean updateLatestStamp(Cell cell, byte[] ts) throws IOException { if (cell.getTimestamp() == HConstants.LATEST_TIMESTAMP) { - setTimestamp(cell, ts, tsOffset); + setTimestamp(cell, ts); return true; } return false; @@ -2517,7 +2672,7 @@ public final class PrivateCellUtil { // Serialization is probably preceded by a length (it is in the KeyValueCodec at least). Bytes.SIZEOF_INT; } - + /** * @param cell * @return Sum of the lengths of all the elements in a Cell; does not count in any infrastructure diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableSequenceId.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableSequenceId.java deleted file mode 100644 index c55b7cf0048..00000000000 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableSequenceId.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase; - -import java.io.IOException; - -import org.apache.yetus.audience.InterfaceAudience; - -/** - * Using this Interface one can mark a Cell as Sequence stampable.
- * Note : Make sure to make Cell implementation of this type in server side. - * @deprecated as of 2.0 and will be removed in 3.0. Use {@link ExtendedCell} instead - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) -@Deprecated -public interface SettableSequenceId { - - /** - * Sets with the given seqId. - * @param seqId - */ - void setSequenceId(long seqId) throws IOException; -} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableTimestamp.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableTimestamp.java deleted file mode 100644 index 73c14883f54..00000000000 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/SettableTimestamp.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase; - -import java.io.IOException; - -import org.apache.yetus.audience.InterfaceAudience; - -/** - * Using this Interface one can mark a Cell as timestamp changeable.
- * Note : Server side Cell implementations in write path must implement this. - * @deprecated as of 2.0 and will be removed in 3.0. Use {@link ExtendedCell} instead - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) -@Deprecated -public interface SettableTimestamp { - - /** - * Sets with the given timestamp. - * @param ts - */ - void setTimestamp(long ts) throws IOException; - - /** - * Sets with the given timestamp. - * @param ts buffer containing the timestamp value - * @param tsOffset offset to the new timestamp - */ - void setTimestamp(byte[] ts, int tsOffset) throws IOException; -} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index e224046baf9..67a337d32eb 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -21,18 +21,16 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; - import org.apache.hadoop.hbase.ByteBufferCell; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.ExtendedCell; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.io.TagCompressionContext; import org.apache.hadoop.hbase.io.util.LRUDictionary; import org.apache.hadoop.hbase.io.util.StreamUtils; @@ -42,6 +40,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.ObjectIntPair; import org.apache.hadoop.io.WritableUtils; +import org.apache.yetus.audience.InterfaceAudience; /** * Base class for all data block encoders that use a buffer. @@ -278,7 +277,7 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder { * represented by the valueOffset and valueLength */ // We return this as a Cell to the upper layers of read flow and might try setting a new SeqId - // there. So this has to be an instance of SettableSequenceId. + // there. So this has to be an instance of ExtendedCell. protected static class OnheapDecodedCell implements ExtendedCell { private static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT + (3 * ClassSize.REFERENCE) + (2 * Bytes.SIZEOF_LONG) + (7 * Bytes.SIZEOF_INT) @@ -465,7 +464,7 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder { } @Override - public void setTimestamp(byte[] ts, int tsOffset) throws IOException { + public void setTimestamp(byte[] ts) throws IOException { // This is not used in actual flow. Throwing UnsupportedOperationException throw new UnsupportedOperationException(); } @@ -704,7 +703,7 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder { } @Override - public void setTimestamp(byte[] ts, int tsOffset) throws IOException { + public void setTimestamp(byte[] ts) throws IOException { // This is not used in actual flow. Throwing UnsupportedOperationException throw new UnsupportedOperationException(); } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestIndividualBytesFieldCell.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestIndividualBytesFieldCell.java index 0fde1a3409f..f98c68e8654 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestIndividualBytesFieldCell.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestIndividualBytesFieldCell.java @@ -18,9 +18,12 @@ package org.apache.hadoop.hbase; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import java.io.IOException; import java.nio.ByteBuffer; - import org.apache.hadoop.hbase.io.ByteArrayOutputStream; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.SmallTests; @@ -29,10 +32,6 @@ import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - @Category({MiscTests.class, SmallTests.class}) public class TestIndividualBytesFieldCell { private static IndividualBytesFieldCell ic0 = null; @@ -168,16 +167,10 @@ public class TestIndividualBytesFieldCell { assertEquals(kv1.getTagsLength() , ic1.getTagsLength()); } - // Verify if SettableSequenceId interface is implemented + // Verify if ExtendedCell interface is implemented @Test - public void testIfSettableSequenceIdImplemented() { - assertTrue(ic0 instanceof SettableSequenceId); - } - - // Verify if SettableTimestamp interface is implemented - @Test - public void testIfSettableTimestampImplemented() { - assertTrue(ic0 instanceof SettableTimestamp); + public void testIfExtendedCellImplemented() { + assertTrue(ic0 instanceof ExtendedCell); } @Test(expected = IllegalArgumentException.class) diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 954118b1de3..ffe1c85b1da 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -334,7 +334,6 @@ public class HFileOutputFormat2 } // we now have the proper WAL writer. full steam ahead - // TODO : Currently in SettableTimeStamp but this will also move to ExtendedCell PrivateCellUtil.updateLatestStamp(cell, this.now); wl.writer.append(kv); wl.written += length; diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/MapReduceCell.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/MapReduceCell.java index f3bac52bbab..38ff59b4093 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/MapReduceCell.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/util/MapReduceCell.java @@ -236,8 +236,8 @@ public class MapReduceCell extends ByteBufferCell implements ExtendedCell { } @Override - public void setTimestamp(byte[] ts, int tsOffset) throws IOException { - PrivateCellUtil.setTimestamp(cell, ts, tsOffset); + public void setTimestamp(byte[] ts) throws IOException { + PrivateCellUtil.setTimestamp(cell, ts); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java index 20ec8eee201..3ed7e7d9d8b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java @@ -119,7 +119,7 @@ public interface BlockCache extends Iterable { * @return number of blocks in the cache */ long getBlockCount(); - + /** * Returns the number of data blocks currently cached in the block cache. * @return number of blocks in the cache diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index e66ad59e050..1b7416669e4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -2876,7 +2876,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi updateDeleteLatestVersionTimeStamp(cell, get, count, byteNow); } } else { - PrivateCellUtil.updateLatestStamp(cell, byteNow, 0); + PrivateCellUtil.updateLatestStamp(cell, byteNow); } } } @@ -2888,7 +2888,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi if (result.size() < count) { // Nothing to delete - PrivateCellUtil.updateLatestStamp(cell, byteNow, 0); + PrivateCellUtil.updateLatestStamp(cell, byteNow); return; } if (result.size() > count) { @@ -4104,7 +4104,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi assert cells instanceof RandomAccess; int listSize = cells.size(); for (int i = 0; i < listSize; i++) { - PrivateCellUtil.updateLatestStamp(cells.get(i), now, 0); + PrivateCellUtil.updateLatestStamp(cells.get(i), now); } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java index ba2e22cf4ce..c6f81c4c65b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java @@ -405,7 +405,7 @@ public class VisibilityController implements MasterCoprocessor, RegionCoprocesso if (result.size() < get.getMaxVersions()) { // Nothing to delete - PrivateCellUtil.updateLatestStamp(cell, byteNow, 0); + PrivateCellUtil.updateLatestStamp(cell, byteNow); return; } if (result.size() > get.getMaxVersions()) {