Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-23705 Add CellComparator to HFileContext (#1062) 

Codecs don't have access to what CellComparator to use.  Backfill.

M hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java
 Adds a new compareRows with default implementation that takes a ByteBuffer.
 Needed by the index in a block encoder implementation.

M hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java
 Adds implementation for meta of new compareRows method. Adds utility
 method for figuring comparator based off tablename.

M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java
 Comparator is in context. Remove redundant handling.

M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
 Comparator is in context. Remove redundant handling. Clean javadoc.

M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java
 Clean javadoc.

M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java
 Cache context so can use it to get comparator to use later.

M hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java
 Cache cellcomparator to use. Javdoc on diff between HFileContext and
 HFileInfo.
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java
 Add CellComparator

M hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java
M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
 Remove comparator caching. Get from context instead.

M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
 Skip a reflection if we can.

M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
 Javadoc. Removed unused filed.

Signed-off-by: Anoop Sam John <anoopsamjohn@apacher.org>
Signed-off-by: Ramkrishna <ramkrishna.s.vasudevan@intel.com>
Signed-off-by: Jan Hentschel <janh@apache.org>
This commit is contained in:
Michael Stack 2020-01-22 20:00:21 -08:00 committed by GitHub
parent a58f2a4a55
commit 6cdc4b1f05
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
50 changed files with 511 additions and 624 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java
View File

@ -17,8 +17,9 @@
*/
package org.apache.hadoop.hbase;
import java.nio.ByteBuffer;
import java.util.Comparator;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
@ -31,11 +32,12 @@ import org.apache.yetus.audience.InterfaceStability;
public interface CellComparator extends Comparator<Cell> {
/**
* A comparator for ordering cells in user-space tables. Useful when writing cells in sorted
* order as necessary for bulk import (i.e. via MapReduce)
* order as necessary for bulk import (i.e. via MapReduce).
* <p>
* CAUTION: This comparator may provide inaccurate ordering for cells from system tables,
* and should not be relied upon in that case.
*/
// For internal use, see CellComparatorImpl utility methods.
static CellComparator getInstance() {
return CellComparatorImpl.COMPARATOR;
}
@ -80,6 +82,24 @@ public interface CellComparator extends Comparator<Cell> {
*/
int compareRows(Cell cell, byte[] bytes, int offset, int length);
/**
* @param row ByteBuffer that wraps a row; will read from current position and will reading all
* remaining; will not disturb the ByteBuffer internal state.
* @return greater than 0 if leftCell is bigger, less than 0 if rightCell is bigger, 0 if both
* cells are equal
*/
default int compareRows(ByteBuffer row, Cell cell) {
if (cell instanceof ByteBufferExtendedCell) {
return ByteBufferUtils.compareTo(row, row.position(), row.remaining(),
((ByteBufferExtendedCell) cell).getRowByteBuffer(),
((ByteBufferExtendedCell) cell).getRowPosition(),
cell.getRowLength());
}
return ByteBufferUtils.compareTo(row, row.position(), row.remaining(),
cell.getRowArray(), cell.getRowOffset(),
cell.getRowLength());
}
/**
* Lexographically compares the two cells excluding the row part. It compares family, qualifier,
* timestamp and the type

43
hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java
View File

@ -17,8 +17,8 @@
*/
package org.apache.hadoop.hbase;
import java.nio.ByteBuffer;
import java.util.Comparator;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
import org.apache.hadoop.hbase.util.Bytes;
@ -26,7 +26,6 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.primitives.Longs;
/**
@ -377,6 +376,26 @@ public class CellComparatorImpl implements CellComparator {
return result;
}
@Override
public int compareRows(ByteBuffer row, Cell cell) {
byte [] array;
int offset;
int len = row.remaining();
if (row.hasArray()) {
array = row.array();
offset = row.position() + row.arrayOffset();
} else {
// We copy the row array if offheap just so we can do a compare. We do this elsewhere too
// in BBUtils when Cell is backed by an offheap ByteBuffer. Needs fixing so no copy. TODO.
array = new byte[len];
offset = 0;
ByteBufferUtils.copyFromBufferToArray(array, row, row.position(),
0, len);
}
// Reverse result since we swap the order of the params we pass below.
return -compareRows(cell, array, offset, len);
}
@Override
public Comparator getSimpleComparator() {
return this;
@ -387,4 +406,24 @@ public class CellComparatorImpl implements CellComparator {
public Comparator getSimpleComparator() {
return new BBKVComparator(this);
}
/**
* Utility method that makes a guess at comparator to use based off passed tableName.
* Use in extreme when no comparator specified.
* @return CellComparator to use going off the {@code tableName} passed.
*/
public static CellComparator getCellComparator(TableName tableName) {
return getCellComparator(tableName.toBytes());
}
/**
* Utility method that makes a guess at comparator to use based off passed tableName.
* Use in extreme when no comparator specified.
* @return CellComparator to use going off the {@code tableName} passed.
*/
public static CellComparator getCellComparator(byte [] tableName) {
// FYI, TableName.toBytes does not create an array; just returns existing array pointer.
return Bytes.equals(tableName, TableName.META_TABLE_NAME.toBytes())?
CellComparatorImpl.META_COMPARATOR: CellComparatorImpl.COMPARATOR;
}
}

3
hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java
View File

@ -292,6 +292,9 @@ public final class TableName implements Comparable<TableName> {
return qualifierAsString;
}
/**
* @return A pointer to TableName as String bytes.
*/
public byte[] toBytes() {
return name;
}

15
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java
View File

@ -18,10 +18,8 @@ package org.apache.hadoop.hbase.io.encoding;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.ByteBufferKeyOnlyKeyValue;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.hfile.BlockType;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
@ -59,14 +57,13 @@ public abstract class AbstractDataBlockEncoder implements DataBlockEncoder {
}
}
protected abstract static class AbstractEncodedSeeker implements
EncodedSeeker {
/**
* Decorates EncodedSeeker with a {@link HFileBlockDecodingContext}
*/
protected abstract static class AbstractEncodedSeeker implements EncodedSeeker {
protected HFileBlockDecodingContext decodingCtx;
protected final CellComparator comparator;
public AbstractEncodedSeeker(CellComparator comparator,
HFileBlockDecodingContext decodingCtx) {
this.comparator = comparator;
public AbstractEncodedSeeker(HFileBlockDecodingContext decodingCtx) {
this.decodingCtx = decodingCtx;
}
@ -77,7 +74,5 @@ public abstract class AbstractDataBlockEncoder implements DataBlockEncoder {
protected boolean includesTags() {
return this.decodingCtx.getHFileContext().isIncludesTags();
}
}
}

13
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
View File

@ -732,9 +732,8 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder {
protected final ObjectIntPair<ByteBuffer> tmpPair = new ObjectIntPair<>();
protected STATE current, previous;
public BufferedEncodedSeeker(CellComparator comparator,
HFileBlockDecodingContext decodingCtx) {
super(comparator, decodingCtx);
public BufferedEncodedSeeker(HFileBlockDecodingContext decodingCtx) {
super(decodingCtx);
if (decodingCtx.getHFileContext().isCompressTags()) {
try {
tagCompressionContext = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE);
@ -1008,11 +1007,7 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder {
}
/**
* @param cell
* @param out
* @param encodingCtx
* @return unencoded size added
* @throws IOException
*/
protected final int afterEncodingKeyValue(Cell cell, DataOutputStream out,
HFileBlockDefaultEncodingContext encodingCtx) throws IOException {
@ -1154,8 +1149,8 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder {
.getEncodingState();
// Write the unencodedDataSizeWritten (with header size)
Bytes.putInt(uncompressedBytesWithHeader,
HConstants.HFILEBLOCK_HEADER_SIZE + DataBlockEncoding.ID_SIZE, state.unencodedDataSizeWritten
);
HConstants.HFILEBLOCK_HEADER_SIZE + DataBlockEncoding.ID_SIZE,
state.unencodedDataSizeWritten);
postEncoding(encodingCtx);
}

10
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
View File

@ -20,9 +20,7 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
import org.apache.hadoop.hbase.util.Bytes;
@ -48,7 +46,8 @@ public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder {
+ "encoding context.");
}
HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
HFileBlockDefaultEncodingContext encodingCtx =
(HFileBlockDefaultEncodingContext) blkEncodingCtx;
encodingCtx.prepareEncoding(out);
NoneEncoder encoder = new NoneEncoder(out, encodingCtx);
@ -81,9 +80,8 @@ public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder {
}
@Override
public EncodedSeeker createSeeker(CellComparator comparator,
final HFileBlockDecodingContext decodingCtx) {
return new BufferedEncodedSeeker<SeekerState>(comparator, decodingCtx) {
public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) {
return new BufferedEncodedSeeker<SeekerState>(decodingCtx) {
@Override
protected void decodeNext() {
current.keyLength = currentBuffer.getInt();

19
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
View File

@ -20,7 +20,6 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
@ -45,20 +44,13 @@ public interface DataBlockEncoder {
* Starts encoding for a block of KeyValues. Call
* {@link #endBlockEncoding(HFileBlockEncodingContext, DataOutputStream, byte[])} to finish
* encoding of a block.
* @param encodingCtx
* @param out
* @throws IOException
*/
void startBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out)
throws IOException;
/**
* Encodes a KeyValue.
* @param cell
* @param encodingCtx
* @param out
* @return unencoded kv size written
* @throws IOException
*/
int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out)
throws IOException;
@ -66,10 +58,6 @@ public interface DataBlockEncoder {
/**
* Ends encoding for a block of KeyValues. Gives a chance for the encoder to do the finishing
* stuff for the encoded block. It must be called at the end of block encoding.
* @param encodingCtx
* @param out
* @param uncompressedBytesWithHeader
* @throws IOException
*/
void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out,
byte[] uncompressedBytesWithHeader) throws IOException;
@ -77,7 +65,6 @@ public interface DataBlockEncoder {
/**
* Decode.
* @param source Compressed stream of KeyValues.
* @param decodingCtx
* @return Uncompressed block of KeyValues.
* @throws IOException If there is an error in source.
*/
@ -96,11 +83,9 @@ public interface DataBlockEncoder {
/**
* Create a HFileBlock seeker which find KeyValues within a block.
* @param comparator what kind of comparison should be used
* @param decodingCtx
* @return A newly created seeker.
*/
EncodedSeeker createSeeker(CellComparator comparator, HFileBlockDecodingContext decodingCtx);
EncodedSeeker createSeeker(HFileBlockDecodingContext decodingCtx);
/**
* Creates a encoder specific encoding context
@ -188,8 +173,6 @@ public interface DataBlockEncoder {
/**
* Compare the given key against the current key
* @param comparator
* @param key
* @return -1 is the passed key is smaller than the current key, 0 if equal and 1 if greater
*/
public int compareKey(CellComparator comparator, Cell key);

7
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
View File

@ -20,9 +20,7 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.PrivateCellUtil;
@ -382,9 +380,8 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
}
@Override
public EncodedSeeker createSeeker(CellComparator comparator,
HFileBlockDecodingContext decodingCtx) {
return new BufferedEncodedSeeker<DiffSeekerState>(comparator, decodingCtx) {
public EncodedSeeker createSeeker(HFileBlockDecodingContext decodingCtx) {
return new BufferedEncodedSeeker<DiffSeekerState>(decodingCtx) {
private byte[] familyNameWithSize;
private static final int TIMESTAMP_WITH_TYPE_LENGTH =
Bytes.SIZEOF_LONG + Bytes.SIZEOF_BYTE;

7
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
View File

@ -21,9 +21,7 @@ import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.PrivateCellUtil;
@ -397,9 +395,8 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
}
@Override
public EncodedSeeker createSeeker(CellComparator comparator,
final HFileBlockDecodingContext decodingCtx) {
return new BufferedEncodedSeeker<FastDiffSeekerState>(comparator, decodingCtx) {
public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) {
return new BufferedEncodedSeeker<FastDiffSeekerState>(decodingCtx) {
private void decode(boolean isFirst) {
byte flag = currentBuffer.get();
if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {

5
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java
View File

@ -17,20 +17,18 @@
package org.apache.hadoop.hbase.io.encoding;
import java.io.IOException;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.yetus.audience.InterfaceAudience;
/**
* A decoding context that is created by a reader's encoder, and is shared
* across the reader's all read operations.
* across all of the reader's read operations.
*
* @see HFileBlockEncodingContext for encoding
*/
@InterfaceAudience.Private
public interface HFileBlockDecodingContext {
/**
* Perform all actions that need to be done before the encoder's real decoding
* process. Decompression needs to be done if
@ -46,7 +44,6 @@ public interface HFileBlockDecodingContext {
* ByteBuffer pointed after the header but before the data
* @param onDiskBlock
* on disk data to be decoded
* @throws IOException
*/
void prepareDecoding(
int onDiskSizeWithoutHeader,

4
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java
View File

@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.io.encoding;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.io.ByteBuffInputStream;
import org.apache.hadoop.hbase.io.TagCompressionContext;
@ -41,8 +40,7 @@ import org.apache.yetus.audience.InterfaceAudience;
*
*/
@InterfaceAudience.Private
public class HFileBlockDefaultDecodingContext implements
HFileBlockDecodingContext {
public class HFileBlockDefaultDecodingContext implements HFileBlockDecodingContext {
private final HFileContext fileContext;
private TagCompressionContext tagCompressionContext;

13
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java
View File

@ -17,13 +17,11 @@
package org.apache.hadoop.hbase.io.encoding;
import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.NONE;
import java.io.ByteArrayInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.SecureRandom;
import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
import org.apache.hadoop.hbase.io.TagCompressionContext;
import org.apache.hadoop.hbase.io.compress.Compression;
@ -36,7 +34,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.io.compress.Compressor;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
/**
@ -47,8 +44,7 @@ import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
*
*/
@InterfaceAudience.Private
public class HFileBlockDefaultEncodingContext implements
HFileBlockEncodingContext {
public class HFileBlockDefaultEncodingContext implements HFileBlockEncodingContext {
private BlockType blockType;
private final DataBlockEncoding encodingAlgo;
@ -113,7 +109,6 @@ public class HFileBlockDefaultEncodingContext implements
/**
* prepare to start a new encoding.
* @throws IOException
*/
public void prepareEncoding(DataOutputStream out) throws IOException {
if (encodingAlgo != null && encodingAlgo != DataBlockEncoding.NONE) {
@ -133,7 +128,8 @@ public class HFileBlockDefaultEncodingContext implements
}
private Bytes compressAfterEncoding(byte[] uncompressedBytesWithHeaderBuffer,
int uncompressedBytesWithHeaderOffset, int uncompressedBytesWithHeaderLength, byte[] headerBytes)
int uncompressedBytesWithHeaderOffset, int uncompressedBytesWithHeaderLength,
byte[] headerBytes)
throws IOException {
Encryption.Context cryptoContext = fileContext.getEncryptionContext();
if (cryptoContext != Encryption.Context.NONE) {
@ -158,7 +154,8 @@ public class HFileBlockDefaultEncodingContext implements
compressedByteStream.reset();
compressionStream.resetState();
compressionStream.write(uncompressedBytesWithHeaderBuffer,
headerBytes.length + uncompressedBytesWithHeaderOffset, uncompressedBytesWithHeaderLength - headerBytes.length);
headerBytes.length + uncompressedBytesWithHeaderOffset,
uncompressedBytesWithHeaderLength - headerBytes.length);
compressionStream.flush();
compressionStream.finish();
byte[] plaintext = compressedByteStream.toByteArray();

5
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockEncodingContext.java
View File

@ -17,7 +17,6 @@
package org.apache.hadoop.hbase.io.encoding;
import java.io.IOException;
import org.apache.hadoop.hbase.io.hfile.BlockType;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.util.Bytes;
@ -46,9 +45,6 @@ public interface HFileBlockEncodingContext {
/**
* Do any action that needs to be performed after the encoding.
* Compression is also included if a non-null compression algorithm is used
*
* @param blockType
* @throws IOException
*/
void postEncoding(BlockType blockType) throws IOException;
@ -64,7 +60,6 @@ public interface HFileBlockEncodingContext {
/**
* Sets the encoding state.
* @param state
*/
void setEncodingState(EncodingState state);

7
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
View File

@ -21,9 +21,7 @@ import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.PrivateCellUtil;
@ -195,9 +193,8 @@ public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder {
}
@Override
public EncodedSeeker createSeeker(CellComparator comparator,
final HFileBlockDecodingContext decodingCtx) {
return new BufferedEncodedSeeker<SeekerState>(comparator, decodingCtx) {
public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) {
return new BufferedEncodedSeeker<SeekerState>(decodingCtx) {
@Override
protected void decodeNext() {
current.keyLength = ByteBuff.readCompressedInt(currentBuffer);

15
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java
View File

@ -22,10 +22,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
@ -71,7 +68,8 @@ public class RowIndexCodecV1 extends AbstractDataBlockEncoder {
+ "encoding context.");
}
HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
HFileBlockDefaultEncodingContext encodingCtx =
(HFileBlockDefaultEncodingContext) blkEncodingCtx;
encodingCtx.prepareEncoding(out);
RowIndexEncoderV1 encoder = new RowIndexEncoderV1(out, encodingCtx);
@ -115,8 +113,7 @@ public class RowIndexCodecV1 extends AbstractDataBlockEncoder {
dup.limit(sourceAsBuffer.position() + onDiskSize);
return dup.slice();
} else {
RowIndexSeekerV1 seeker = new RowIndexSeekerV1(CellComparatorImpl.COMPARATOR,
decodingCtx);
RowIndexSeekerV1 seeker = new RowIndexSeekerV1(decodingCtx);
seeker.setCurrentBuffer(new SingleByteBuff(sourceAsBuffer));
List<Cell> kvs = new ArrayList<>();
kvs.add(seeker.getCell());
@ -151,9 +148,7 @@ public class RowIndexCodecV1 extends AbstractDataBlockEncoder {
}
@Override
public EncodedSeeker createSeeker(CellComparator comparator,
HFileBlockDecodingContext decodingCtx) {
return new RowIndexSeekerV1(comparator, decodingCtx);
public EncodedSeeker createSeeker(HFileBlockDecodingContext decodingCtx) {
return new RowIndexSeekerV1(decodingCtx);
}
}

6
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java
View File

@ -12,9 +12,7 @@ package org.apache.hadoop.hbase.io.encoding;
import java.io.DataOutputStream;
import java.io.IOException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
import org.apache.yetus.audience.InterfaceAudience;
@ -32,10 +30,12 @@ public class RowIndexEncoderV1 {
private NoneEncoder encoder;
private int startOffset = -1;
private ByteArrayOutputStream rowsOffsetBAOS = new ByteArrayOutputStream(64 * 4);
private final HFileBlockEncodingContext context;
public RowIndexEncoderV1(DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx) {
this.out = out;
this.encoder = new NoneEncoder(out, encodingCtx);
this.context = encodingCtx;
}
public int write(Cell cell) throws IOException {
@ -56,7 +56,7 @@ public class RowIndexEncoderV1 {
throw new IOException("Key cannot be null or empty");
}
if (lastCell != null) {
int keyComp = CellComparatorImpl.COMPARATOR.compareRows(lastCell, cell);
int keyComp = this.context.getHFileContext().getCellComparator().compareRows(lastCell, cell);
if (keyComp > 0) {
throw new IOException("Added a key not lexically larger than"
+ " previous. Current cell = " + cell + ", lastCell = " + lastCell);

30
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java
View File

@ -17,8 +17,6 @@
package org.apache.hadoop.hbase.io.encoding;
import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.ByteBufferExtendedCell;
import org.apache.hadoop.hbase.ByteBufferKeyOnlyKeyValue;
import org.apache.hadoop.hbase.ByteBufferKeyValue;
import org.apache.hadoop.hbase.Cell;
@ -50,10 +48,11 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker {
private int rowNumber;
private ByteBuff rowOffsets = null;
private final CellComparator cellComparator;
public RowIndexSeekerV1(CellComparator comparator,
HFileBlockDecodingContext decodingCtx) {
super(comparator, decodingCtx);
public RowIndexSeekerV1(HFileBlockDecodingContext decodingCtx) {
super(decodingCtx);
this.cellComparator = decodingCtx.getHFileContext().getCellComparator();
}
@Override
@ -131,8 +130,7 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker {
int comp = 0;
while (low <= high) {
mid = low + ((high - low) >> 1);
ByteBuffer row = getRow(mid);
comp = compareRows(row, seekCell);
comp = this.cellComparator.compareRows(getRow(mid), seekCell);
if (comp < 0) {
low = mid + 1;
} else if (comp > 0) {
@ -154,19 +152,6 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker {
}
}
private int compareRows(ByteBuffer row, Cell seekCell) {
if (seekCell instanceof ByteBufferExtendedCell) {
return ByteBufferUtils.compareTo(row, row.position(), row.remaining(),
((ByteBufferExtendedCell) seekCell).getRowByteBuffer(),
((ByteBufferExtendedCell) seekCell).getRowPosition(),
seekCell.getRowLength());
} else {
return ByteBufferUtils.compareTo(row, row.position(), row.remaining(),
seekCell.getRowArray(), seekCell.getRowOffset(),
seekCell.getRowLength());
}
}
private ByteBuffer getRow(int index) {
int offset = rowOffsets.getIntAfterPosition(index * Bytes.SIZEOF_INT);
ByteBuff block = currentBuffer.duplicate();
@ -191,8 +176,8 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker {
}
}
do {
int comp;
comp = PrivateCellUtil.compareKeyIgnoresMvcc(comparator, seekCell, current.currentKey);
int comp =
PrivateCellUtil.compareKeyIgnoresMvcc(this.cellComparator, seekCell, current.currentKey);
if (comp == 0) { // exact match
if (seekBefore) {
if (!previous.isValid()) {
@ -400,5 +385,4 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker {
return ret;
}
}
}

31
hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java
View File

@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.io.hfile;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.compress.Compression;
@ -28,9 +30,10 @@ import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.yetus.audience.InterfaceAudience;
/**
* This carries the information on some of the meta data about the HFile. This
* meta data is used across the HFileWriter/Readers and the HFileBlocks.
* This helps to add new information to the HFile.
* Read-only HFile Context Information. Meta data that is used by HFileWriter/Readers and by
* HFileBlocks. Create one using the {@link HFileContextBuilder} (See HFileInfo and the HFile
* Trailer class).
* @see HFileContextBuilder
*/
@InterfaceAudience.Private
public class HFileContext implements HeapSize, Cloneable {
@ -42,7 +45,7 @@ public class HFileContext implements HeapSize, Cloneable {
//byte[] headers for column family and table name
2 * ClassSize.ARRAY + 2 * ClassSize.REFERENCE);
public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
private static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
/** Whether checksum is enabled or not**/
private boolean usesHBaseChecksum = true;
@ -67,6 +70,7 @@ public class HFileContext implements HeapSize, Cloneable {
private String hfileName;
private byte[] columnFamily;
private byte[] tableName;
private CellComparator cellComparator;
//Empty constructor. Go with setters
public HFileContext() {
@ -74,7 +78,6 @@ public class HFileContext implements HeapSize, Cloneable {
/**
* Copy constructor
* @param context
*/
public HFileContext(HFileContext context) {
this.usesHBaseChecksum = context.usesHBaseChecksum;
@ -91,13 +94,14 @@ public class HFileContext implements HeapSize, Cloneable {
this.hfileName = context.hfileName;
this.columnFamily = context.columnFamily;
this.tableName = context.tableName;
this.cellComparator = context.cellComparator;
}
HFileContext(boolean useHBaseChecksum, boolean includesMvcc, boolean includesTags,
Compression.Algorithm compressAlgo, boolean compressTags, ChecksumType checksumType,
int bytesPerChecksum, int blockSize, DataBlockEncoding encoding,
Encryption.Context cryptoContext, long fileCreateTime, String hfileName,
byte[] columnFamily, byte[] tableName) {
byte[] columnFamily, byte[] tableName, CellComparator cellComparator) {
this.usesHBaseChecksum = useHBaseChecksum;
this.includesMvcc = includesMvcc;
this.includesTags = includesTags;
@ -114,11 +118,14 @@ public class HFileContext implements HeapSize, Cloneable {
this.hfileName = hfileName;
this.columnFamily = columnFamily;
this.tableName = tableName;
// If no cellComparator specified, make a guess based off tablename. If hbase:meta, then should
// be the meta table comparator. Comparators are per table.
this.cellComparator = cellComparator != null ? cellComparator : this.tableName != null ?
CellComparatorImpl.getCellComparator(this.tableName) : CellComparator.getInstance();
}
/**
* @return true when on-disk blocks from this file are compressed, and/or encrypted;
* false otherwise.
* @return true when on-disk blocks are compressed, and/or encrypted; false otherwise.
*/
public boolean isCompressedOrEncrypted() {
Compression.Algorithm compressAlgo = getCompression();
@ -208,6 +215,11 @@ public class HFileContext implements HeapSize, Cloneable {
public byte[] getTableName() {
return this.tableName;
}
public CellComparator getCellComparator() {
return this.cellComparator;
}
/**
* HeapSize implementation. NOTE : The heap size should be altered when new state variable are
* added.
@ -263,8 +275,9 @@ public class HFileContext implements HeapSize, Cloneable {
sb.append(", columnFamily=");
sb.append(Bytes.toStringBinary(columnFamily));
}
sb.append(", cellComparator=");
sb.append(this.cellComparator);
sb.append("]");
return sb.toString();
}
}

10
hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.io.hfile;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.crypto.Encryption;
@ -56,6 +57,7 @@ public class HFileContextBuilder {
private String hfileName = null;
private byte[] columnFamily = null;
private byte[] tableName = null;
private CellComparator cellComparator;
public HFileContextBuilder() {}
@ -77,6 +79,7 @@ public class HFileContextBuilder {
this.hfileName = hfc.getHFileName();
this.columnFamily = hfc.getColumnFamily();
this.tableName = hfc.getTableName();
this.cellComparator = hfc.getCellComparator();
}
public HFileContextBuilder withHBaseCheckSum(boolean useHBaseCheckSum) {
@ -149,9 +152,14 @@ public class HFileContextBuilder {
return this;
}
public HFileContextBuilder withCellComparator(CellComparator cellComparator) {
this.cellComparator = cellComparator;
return this;
}
public HFileContext build() {
return new HFileContext(usesHBaseChecksum, includesMvcc, includesTags, compression,
compressTags, checksumType, bytesPerChecksum, blocksize, encoding, cryptoContext,
fileCreateTime, hfileName, columnFamily, tableName);
fileCreateTime, hfileName, columnFamily, tableName, cellComparator);
}
}

159
hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
View File

@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -21,7 +21,6 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TASK_KEY;
import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TIME_KEY;
import static org.apache.hadoop.hbase.regionserver.HStoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY;
import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.InetSocketAddress;
@ -38,13 +37,11 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
@ -91,7 +88,6 @@ import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
@ -179,18 +175,17 @@ public class HFileOutputFormat2
return combineTableNameSuffix(tableName, family);
}
static <V extends Cell> RecordWriter<ImmutableBytesWritable, V>
createRecordWriter(final TaskAttemptContext context, final OutputCommitter committer)
throws IOException {
static <V extends Cell> RecordWriter<ImmutableBytesWritable, V> createRecordWriter(
final TaskAttemptContext context, final OutputCommitter committer) throws IOException {
// Get the path of the temporary output file
final Path outputDir = ((FileOutputCommitter)committer).getWorkPath();
final Configuration conf = context.getConfiguration();
final boolean writeMultipleTables = conf.getBoolean(MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY, false) ;
final boolean writeMultipleTables =
conf.getBoolean(MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY, false);
final String writeTableNames = conf.get(OUTPUT_TABLE_NAME_CONF_KEY);
if (writeTableNames == null || writeTableNames.isEmpty()) {
throw new IllegalArgumentException("Configuration parameter " + OUTPUT_TABLE_NAME_CONF_KEY
+ " cannot be empty");
throw new IllegalArgumentException("" + OUTPUT_TABLE_NAME_CONF_KEY + " cannot be empty");
}
final FileSystem fs = outputDir.getFileSystem(conf);
// These configs. are from hbase-*.xml
@ -199,19 +194,12 @@ public class HFileOutputFormat2
// Invented config. Add to hbase-*.xml if other than default compression.
final String defaultCompressionStr = conf.get("hfile.compression",
Compression.Algorithm.NONE.getName());
final Algorithm defaultCompression = HFileWriterImpl
.compressionByName(defaultCompressionStr);
final Algorithm defaultCompression = HFileWriterImpl.compressionByName(defaultCompressionStr);
String compressionStr = conf.get(COMPRESSION_OVERRIDE_CONF_KEY);
final Algorithm overriddenCompression;
if (compressionStr != null) {
overriddenCompression = Compression.getCompressionAlgorithmByName(compressionStr);
} else {
overriddenCompression = null;
}
final Algorithm overriddenCompression = compressionStr != null ?
Compression.getCompressionAlgorithmByName(compressionStr): null;
final boolean compactionExclude = conf.getBoolean(
"hbase.mapreduce.hfileoutputformat.compaction.exclude", false);
final Set<String> allTableNames = Arrays.stream(writeTableNames.split(
Bytes.toString(tableSeparator))).collect(Collectors.toSet());
@ -224,24 +212,17 @@ public class HFileOutputFormat2
String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_OVERRIDE_CONF_KEY);
final Map<byte[], DataBlockEncoding> datablockEncodingMap
= createFamilyDataBlockEncodingMap(conf);
final DataBlockEncoding overriddenEncoding;
if (dataBlockEncodingStr != null) {
overriddenEncoding = DataBlockEncoding.valueOf(dataBlockEncodingStr);
} else {
overriddenEncoding = null;
}
final DataBlockEncoding overriddenEncoding = dataBlockEncodingStr != null ?
DataBlockEncoding.valueOf(dataBlockEncodingStr) : null;
return new RecordWriter<ImmutableBytesWritable, V>() {
// Map of families to writers and how much has been output on the writer.
private final Map<byte[], WriterLength> writers =
new TreeMap<>(Bytes.BYTES_COMPARATOR);
private final Map<byte[], byte[]> previousRows =
new TreeMap<>(Bytes.BYTES_COMPARATOR);
private final Map<byte[], WriterLength> writers = new TreeMap<>(Bytes.BYTES_COMPARATOR);
private final Map<byte[], byte[]> previousRows = new TreeMap<>(Bytes.BYTES_COMPARATOR);
private final long now = EnvironmentEdgeManager.currentTime();
@Override
public void write(ImmutableBytesWritable row, V cell)
throws IOException {
public void write(ImmutableBytesWritable row, V cell) throws IOException {
Cell kv = cell;
// null input == user explicitly wants to flush
if (row == null && kv == null) {
@ -255,12 +236,11 @@ public class HFileOutputFormat2
byte[] tableNameBytes = null;
if (writeMultipleTables) {
tableNameBytes = MultiTableHFileOutputFormat.getTableName(row.get());
tableNameBytes =
TableName.valueOf(tableNameBytes).getNameWithNamespaceInclAsString()
tableNameBytes = TableName.valueOf(tableNameBytes).getNameWithNamespaceInclAsString()
.getBytes(Charset.defaultCharset());
if (!allTableNames.contains(Bytes.toString(tableNameBytes))) {
throw new IllegalArgumentException("TableName '" + Bytes.toString(tableNameBytes) +
"' not" + " expected");
throw new IllegalArgumentException("TableName " + Bytes.toString(tableNameBytes) +
" not expected");
}
} else {
tableNameBytes = Bytes.toBytes(writeTableNames);
@ -275,9 +255,7 @@ public class HFileOutputFormat2
if (wl == null) {
Path writerPath = null;
if (writeMultipleTables) {
writerPath = new Path(outputDir,new Path(tableRelPath, Bytes
.toString(family)));
writerPath = new Path(outputDir,new Path(tableRelPath, Bytes.toString(family)));
}
else {
writerPath = new Path(outputDir, Bytes.toString(family));
@ -303,33 +281,23 @@ public class HFileOutputFormat2
connection.getRegionLocator(TableName.valueOf(tableName))) {
loc = locator.getRegionLocation(rowKey);
} catch (Throwable e) {
LOG.warn("There's something wrong when locating rowkey: " +
Bytes.toString(rowKey) + " for tablename: " + tableName, e);
LOG.warn("Something wrong locating rowkey {} in {}",
Bytes.toString(rowKey), tableName, e);
loc = null;
} }
if (null == loc) {
if (LOG.isTraceEnabled()) {
LOG.trace("failed to get region location, so use default writer for rowkey: " +
Bytes.toString(rowKey));
}
LOG.trace("Failed get of location, use default writer {}", Bytes.toString(rowKey));
wl = getNewWriter(tableNameBytes, family, conf, null);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("first rowkey: [" + Bytes.toString(rowKey) + "]");
}
LOG.debug("First rowkey: [{}]", Bytes.toString(rowKey));
InetSocketAddress initialIsa =
new InetSocketAddress(loc.getHostname(), loc.getPort());
if (initialIsa.isUnresolved()) {
if (LOG.isTraceEnabled()) {
LOG.trace("failed to resolve bind address: " + loc.getHostname() + ":"
+ loc.getPort() + ", so use default writer");
}
LOG.trace("Failed resolve address {}, use default writer", loc.getHostnamePort());
wl = getNewWriter(tableNameBytes, family, conf, null);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("use favored nodes writer: " + initialIsa.getHostString());
}
LOG.debug("Use favored nodes writer: {}", initialIsa.getHostString());
wl = getNewWriter(tableNameBytes, family, conf, new InetSocketAddress[] { initialIsa
});
}
@ -369,24 +337,22 @@ public class HFileOutputFormat2
private void closeWriter(WriterLength wl) throws IOException {
if (wl.writer != null) {
LOG.info(
"Writer=" + wl.writer.getPath() + ((wl.written == 0)? "": ", wrote=" + wl.written));
LOG.info("Writer=" + wl.writer.getPath() +
((wl.written == 0)? "": ", wrote=" + wl.written));
close(wl.writer);
}
wl.writer = null;
}
wl.written = 0;
}
/*
* Create a new StoreFile.Writer.
* @param family
* @return A WriterLength, containing a new StoreFile.Writer.
* @throws IOException
*/
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="BX_UNBOXING_IMMEDIATELY_REBOXED",
justification="Not important")
private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration
conf, InetSocketAddress[] favoredNodes) throws IOException {
private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration conf,
InetSocketAddress[] favoredNodes) throws IOException {
byte[] tableAndFamily = getTableNameSuffixedWithFamily(tableName, family);
Path familydir = new Path(outputDir, Bytes.toString(family));
if (writeMultipleTables) {
@ -409,12 +375,9 @@ public class HFileOutputFormat2
encoding = encoding == null ? datablockEncodingMap.get(tableAndFamily) : encoding;
encoding = encoding == null ? DataBlockEncoding.NONE : encoding;
HFileContextBuilder contextBuilder = new HFileContextBuilder()
.withCompression(compression)
.withChecksumType(HStore.getChecksumType(conf))
.withBytesPerCheckSum(HStore.getBytesPerChecksum(conf))
.withBlockSize(blockSize)
.withColumnFamily(family)
.withTableName(tableName);
.withCompression(compression).withChecksumType(HStore.getChecksumType(conf))
.withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blockSize)
.withColumnFamily(family).withTableName(tableName);
if (HFile.getFormatVersion(conf) >= HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
contextBuilder.withIncludesTags(true);
@ -423,16 +386,13 @@ public class HFileOutputFormat2
contextBuilder.withDataBlockEncoding(encoding);
HFileContext hFileContext = contextBuilder.build();
if (null == favoredNodes) {
wl.writer =
new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, fs)
wl.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, fs)
.withOutputDir(familydir).withBloomType(bloomType)
.withComparator(CellComparator.getInstance()).withFileContext(hFileContext).build();
.withFileContext(hFileContext).build();
} else {
wl.writer =
new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, new HFileSystem(fs))
wl.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, new HFileSystem(fs))
.withOutputDir(familydir).withBloomType(bloomType)
.withComparator(CellComparator.getInstance()).withFileContext(hFileContext)
.withFavoredNodes(favoredNodes).build();
.withFileContext(hFileContext).withFavoredNodes(favoredNodes).build();
}
this.writers.put(tableAndFamily, wl);
@ -441,22 +401,17 @@ public class HFileOutputFormat2
private void close(final StoreFileWriter w) throws IOException {
if (w != null) {
w.appendFileInfo(BULKLOAD_TIME_KEY,
Bytes.toBytes(System.currentTimeMillis()));
w.appendFileInfo(BULKLOAD_TASK_KEY,
Bytes.toBytes(context.getTaskAttemptID().toString()));
w.appendFileInfo(MAJOR_COMPACTION_KEY,
Bytes.toBytes(true));
w.appendFileInfo(EXCLUDE_FROM_MINOR_COMPACTION_KEY,
Bytes.toBytes(compactionExclude));
w.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
w.appendFileInfo(BULKLOAD_TASK_KEY, Bytes.toBytes(context.getTaskAttemptID().toString()));
w.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(true));
w.appendFileInfo(EXCLUDE_FROM_MINOR_COMPACTION_KEY, Bytes.toBytes(compactionExclude));
w.appendTrackedTimestampsToMetadata();
w.close();
}
}
@Override
public void close(TaskAttemptContext c)
throws IOException, InterruptedException {
public void close(TaskAttemptContext c) throws IOException, InterruptedException {
for (WriterLength wl: this.writers.values()) {
close(wl.writer);
}
@ -496,21 +451,18 @@ public class HFileOutputFormat2
throws IOException {
ArrayList<ImmutableBytesWritable> ret = new ArrayList<>();
for(RegionLocator regionLocator : regionLocators)
{
for(RegionLocator regionLocator : regionLocators) {
TableName tableName = regionLocator.getName();
LOG.info("Looking up current regions for table " + tableName);
byte[][] byteKeys = regionLocator.getStartKeys();
for (byte[] byteKey : byteKeys) {
byte[] fullKey = byteKey; //HFileOutputFormat2 use case
if (writeMultipleTables)
{
if (writeMultipleTables) {
//MultiTableHFileOutputFormat use case
fullKey = combineTableNameSuffix(tableName.getName(), byteKey);
}
if (LOG.isDebugEnabled()) {
LOG.debug("SplitPoint startkey for table [" + tableName + "]: [" + Bytes.toStringBinary
(fullKey) + "]");
LOG.debug("SplitPoint startkey for " + tableName + ": " + Bytes.toStringBinary(fullKey));
}
ret.add(new ImmutableBytesWritable(fullKey));
}
@ -537,8 +489,8 @@ public class HFileOutputFormat2
TreeSet<ImmutableBytesWritable> sorted = new TreeSet<>(startKeys);
ImmutableBytesWritable first = sorted.first();
if (writeMultipleTables) {
first = new ImmutableBytesWritable(MultiTableHFileOutputFormat.getSuffix(sorted.first
().get()));
first =
new ImmutableBytesWritable(MultiTableHFileOutputFormat.getSuffix(sorted.first().get()));
}
if (!first.equals(HConstants.EMPTY_BYTE_ARRAY)) {
throw new IllegalArgumentException(
@ -644,8 +596,7 @@ public class HFileOutputFormat2
List<RegionLocator> regionLocators = new ArrayList<>(multiTableInfo.size());
List<TableDescriptor> tableDescriptors = new ArrayList<>(multiTableInfo.size());
for( TableInfo tableInfo : multiTableInfo )
{
for(TableInfo tableInfo : multiTableInfo) {
regionLocators.add(tableInfo.getRegionLocator());
String tn = writeMultipleTables?
tableInfo.getRegionLocator().getName().getNameWithNamespaceInclAsString():
@ -653,10 +604,12 @@ public class HFileOutputFormat2
allTableNames.add(tn);
tableDescriptors.add(tableInfo.getTableDescriptor());
}
// Record tablenames for creating writer by favored nodes, and decoding compression, block size and other attributes of columnfamily per table
// Record tablenames for creating writer by favored nodes, and decoding compression,
// block size and other attributes of columnfamily per table
conf.set(OUTPUT_TABLE_NAME_CONF_KEY, StringUtils.join(allTableNames, Bytes
.toString(tableSeparator)));
List<ImmutableBytesWritable> startKeys = getRegionStartKeys(regionLocators, writeMultipleTables);
List<ImmutableBytesWritable> startKeys =
getRegionStartKeys(regionLocators, writeMultipleTables);
// Use table's region boundaries for TOP split points.
LOG.info("Configuring " + startKeys.size() + " reduce partitions " +
"to match current region count for all tables");
@ -852,9 +805,11 @@ public class HFileOutputFormat2
TotalOrderPartitioner.setPartitionFile(conf, partitionsPath);
}
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value =
"RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
@VisibleForTesting
static String serializeColumnFamilyAttribute(Function<ColumnFamilyDescriptor, String> fn, List<TableDescriptor> allTables)
static String serializeColumnFamilyAttribute(Function<ColumnFamilyDescriptor, String> fn,
List<TableDescriptor> allTables)
throws UnsupportedEncodingException {
StringBuilder attributeValue = new StringBuilder();
int i = 0;
@ -869,8 +824,8 @@ public class HFileOutputFormat2
attributeValue.append('&');
}
attributeValue.append(URLEncoder.encode(
Bytes.toString(combineTableNameSuffix(tableDescriptor.getTableName().getName(), familyDescriptor.getName())),
"UTF-8"));
Bytes.toString(combineTableNameSuffix(tableDescriptor.getTableName().getName(),
familyDescriptor.getName())), "UTF-8"));
attributeValue.append('=');
attributeValue.append(URLEncoder.encode(fn.apply(familyDescriptor), "UTF-8"));
}

87
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
View File

@ -26,20 +26,18 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.CellComparatorImpl.MetaCellComparator;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
/**
* The {@link HFile} has a fixed trailer which contains offsets to other
@ -77,16 +75,24 @@ public class FixedFileTrailer {
*/
private long loadOnOpenDataOffset;
/** The number of entries in the root data index. */
/**
* The number of entries in the root data index.
*/
private int dataIndexCount;
/** Total uncompressed size of all blocks of the data index */
/**
* Total uncompressed size of all blocks of the data index
*/
private long uncompressedDataIndexSize;
/** The number of entries in the meta index */
/**
* The number of entries in the meta index
*/
private int metaIndexCount;
/** The total uncompressed size of keys/values stored in the file. */
/**
* The total uncompressed size of keys/values stored in the file.
*/
private long totalUncompressedBytes;
/**
@ -95,7 +101,9 @@ public class FixedFileTrailer {
*/
private long entryCount;
/** The compression codec used for all blocks. */
/**
* The compression codec used for all blocks.
*/
private Compression.Algorithm compressionCodec = Compression.Algorithm.NONE;
/**
@ -104,7 +112,9 @@ public class FixedFileTrailer {
*/
private int numDataIndexLevels;
/** The offset of the first data block. */
/**
* The offset of the first data block.
*/
private long firstDataBlockOffset;
/**
@ -113,17 +123,25 @@ public class FixedFileTrailer {
*/
private long lastDataBlockOffset;
/** Raw key comparator class name in version 3 */
/**
* Raw key comparator class name in version 3
*/
// We could write the actual class name from 2.0 onwards and handle BC
private String comparatorClassName = CellComparator.getInstance().getClass().getName();
/** The encryption key */
/**
* The encryption key
*/
private byte[] encryptionKey;
/** The {@link HFile} format major version. */
/**
* The {@link HFile} format major version.
*/
private final int majorVersion;
/** The {@link HFile} format minor version. */
/**
* The {@link HFile} format minor version.
*/
private final int minorVersion;
FixedFileTrailer(int majorVersion, int minorVersion) {
@ -133,7 +151,7 @@ public class FixedFileTrailer {
}
private static int[] computeTrailerSizeByVersion() {
int versionToSize[] = new int[HFile.MAX_FORMAT_VERSION + 1];
int[] versionToSize = new int[HFile.MAX_FORMAT_VERSION + 1];
// We support only 2 major versions now. ie. V2, V3
versionToSize[2] = 212;
for (int version = 3; version <= HFile.MAX_FORMAT_VERSION; version++) {
@ -147,14 +165,13 @@ public class FixedFileTrailer {
private static int getMaxTrailerSize() {
int maxSize = 0;
for (int version = HFile.MIN_FORMAT_VERSION;
version <= HFile.MAX_FORMAT_VERSION;
++version)
for (int version = HFile.MIN_FORMAT_VERSION; version <= HFile.MAX_FORMAT_VERSION; ++version) {
maxSize = Math.max(getTrailerSize(version), maxSize);
}
return maxSize;
}
private static final int TRAILER_SIZE[] = computeTrailerSizeByVersion();
private static final int[] TRAILER_SIZE = computeTrailerSizeByVersion();
private static final int MAX_TRAILER_SIZE = getMaxTrailerSize();
private static final int NOT_PB_SIZE = BlockType.MAGIC_LENGTH + Bytes.SIZEOF_INT;
@ -171,9 +188,6 @@ public class FixedFileTrailer {
* Write the trailer to a data stream. We support writing version 1 for
* testing and for determining version 1 trailer size. It is also easy to see
* what fields changed in version 2.
*
* @param outputStream
* @throws IOException
*/
void serialize(DataOutputStream outputStream) throws IOException {
HFile.checkFormatVersion(majorVersion);
@ -239,9 +253,6 @@ public class FixedFileTrailer {
* Deserialize the fixed file trailer from the given stream. The version needs
* to already be specified. Make sure this is consistent with
* {@link #serialize(DataOutputStream)}.
*
* @param inputStream
* @throws IOException
*/
void deserialize(DataInputStream inputStream) throws IOException {
HFile.checkFormatVersion(majorVersion);
@ -263,8 +274,6 @@ public class FixedFileTrailer {
/**
* Deserialize the file trailer as protobuf
* @param inputStream
* @throws IOException
*/
void deserializeFromPB(DataInputStream inputStream) throws IOException {
// read PB and skip padding
@ -320,8 +329,6 @@ public class FixedFileTrailer {
/**
* Deserialize the file trailer as writable data
* @param input
* @throws IOException
*/
void deserializeFromWritable(DataInput input) throws IOException {
fileInfoOffset = input.readLong();
@ -343,8 +350,9 @@ public class FixedFileTrailer {
}
private void append(StringBuilder sb, String s) {
if (sb.length() > 0)
if (sb.length() > 0) {
sb.append(", ");
}
sb.append(s);
}
@ -566,7 +574,7 @@ public class FixedFileTrailer {
* than the new name; writing the new name will make it so newly-written hfiles are not parseable
* by hbase-1.x, a facility we'd like to preserve across rolling upgrade and hbase-1.x clusters
* reading hbase-2.x produce.
*
* <p>
* The Comparators in hbase-2.x work the same as they did in hbase-1.x; they compare
* KeyValues. In hbase-2.x they were renamed making use of the more generic 'Cell'
* nomenclature to indicate that we intend to move away from KeyValues post hbase-2. A naming
@ -579,9 +587,9 @@ public class FixedFileTrailer {
*
* @param comparator String class name of the Comparator used in this hfile.
* @return What to store in the trailer as our comparator name.
* @see #getComparatorClass(String)
* @since hbase-2.0.0.
* @deprecated Since hbase-2.0.0. Will be removed in hbase-3.0.0.
* @see #getComparatorClass(String)
*/
@Deprecated
private String getHBase1CompatibleName(final String comparator) {
@ -624,10 +632,14 @@ public class FixedFileTrailer {
return comparatorKlass;
}
public static CellComparator createComparator(
String comparatorClassName) throws IOException {
static CellComparator createComparator(String comparatorClassName) throws IOException {
if (comparatorClassName.equals(CellComparatorImpl.COMPARATOR.getClass().getName())) {
return CellComparatorImpl.COMPARATOR;
} else if (comparatorClassName.equals(
CellComparatorImpl.META_COMPARATOR.getClass().getName())) {
return CellComparatorImpl.META_COMPARATOR;
}
try {
Class<? extends CellComparator> comparatorClass = getComparatorClass(comparatorClassName);
if (comparatorClass != null) {
return comparatorClass.getDeclaredConstructor().newInstance();
@ -635,8 +647,7 @@ public class FixedFileTrailer {
LOG.warn("No Comparator class for " + comparatorClassName + ". Returning Null.");
return null;
} catch (Exception e) {
throw new IOException("Comparator class " + comparatorClassName +
" is not instantiable", e);
throw new IOException("Comparator class " + comparatorClassName + " is not instantiable", e);
}
}

32
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
View File

@ -1,4 +1,4 @@
/**
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@ -26,7 +26,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.LongAdder;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@ -52,7 +51,6 @@ import org.apache.hadoop.io.Writable;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
@ -121,7 +119,7 @@ import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
* an HFile instance?
*/
@InterfaceAudience.Private
public class HFile {
public final class HFile {
// LOG is being used in HFileBlock and CheckSumUtil
static final Logger LOG = LoggerFactory.getLogger(HFile.class);
@ -177,6 +175,11 @@ public class HFile {
/** Static instance for the metrics so that HFileReaders access the same instance */
static final MetricsIO metrics = new MetricsIO(new MetricsIOWrapperImpl());
/**
* Shutdown constructor.
*/
private HFile() {}
/**
* Number of checksum verification failures. It also
* clears the counter.
@ -222,10 +225,11 @@ public class HFile {
*/
void addInlineBlockWriter(InlineBlockWriter bloomWriter);
// The below three methods take Writables. We'd like to undo Writables but undoing the below would be pretty
// painful. Could take a byte [] or a Message but we want to be backward compatible around hfiles so would need
// to map between Message and Writable or byte [] and current Writable serialization. This would be a bit of work
// to little gain. Thats my thinking at moment. St.Ack 20121129
// The below three methods take Writables. We'd like to undo Writables but undoing the below
// would be pretty painful. Could take a byte [] or a Message but we want to be backward
// compatible around hfiles so would need to map between Message and Writable or byte [] and
// current Writable serialization. This would be a bit of work to little gain. Thats my
// thinking at moment. St.Ack 20121129
void appendMetaBlock(String bloomFilterMetaKey, Writable metaWriter);
@ -258,7 +262,6 @@ public class HFile {
protected FileSystem fs;
protected Path path;
protected FSDataOutputStream ostream;
protected CellComparator comparator = CellComparator.getInstance();
protected InetSocketAddress[] favoredNodes;
private HFileContext fileContext;
protected boolean shouldDropBehind = false;
@ -282,12 +285,6 @@ public class HFile {
return this;
}
public WriterFactory withComparator(CellComparator comparator) {
Preconditions.checkNotNull(comparator);
this.comparator = comparator;
return this;
}
public WriterFactory withFavoredNodes(InetSocketAddress[] favoredNodes) {
// Deliberately not checking for null here.
this.favoredNodes = favoredNodes;
@ -319,7 +316,7 @@ public class HFile {
LOG.debug("Unable to set drop behind on {}", path.getName());
}
}
return new HFileWriterImpl(conf, cacheConf, path, ostream, comparator, fileContext);
return new HFileWriterImpl(conf, cacheConf, path, ostream, fileContext);
}
}
@ -372,8 +369,6 @@ public class HFile {
* Read in a file block.
* @param offset offset to read.
* @param onDiskBlockSize size of the block
* @param cacheBlock
* @param pread
* @param isCompaction is this block being read as part of a compaction
* @param expectedBlockType the block type we are expecting to read with this read operation,
* or null to read whatever block type is available and avoid checking (that might reduce
@ -383,7 +378,6 @@ public class HFile {
* encoding. This check only applies to data blocks and can be set to null when the caller is
* expecting to read a non-data block and has set expectedBlockType accordingly.
* @return Block wrapped in a ByteBuffer.
* @throws IOException
*/
HFileBlock readBlock(long offset, long onDiskBlockSize,
boolean cacheBlock, final boolean pread, final boolean isCompaction,

46
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.io.hfile;
import static org.apache.hadoop.hbase.io.ByteBuffAllocator.HEAP;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
@ -29,18 +28,13 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.io.ByteBuffAllocator;
import org.apache.hadoop.hbase.io.util.BlockIOUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
import org.apache.hadoop.hbase.io.ByteBuffAllocator;
import org.apache.hadoop.hbase.io.ByteBuffInputStream;
import org.apache.hadoop.hbase.io.ByteBufferWriterDataOutputStream;
import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
@ -50,6 +44,7 @@ import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;
import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
import org.apache.hadoop.hbase.io.util.BlockIOUtils;
import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.nio.MultiByteBuff;
import org.apache.hadoop.hbase.nio.SingleByteBuff;
@ -57,7 +52,9 @@ import org.apache.hadoop.hbase.regionserver.ShipperListener;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
@ -360,8 +357,8 @@ public class HFileBlock implements Cacheable {
final long prevBlockOffset = buf.getLong(Header.PREV_BLOCK_OFFSET_INDEX);
// This constructor is called when we deserialize a block from cache and when we read a block in
// from the fs. fileCache is null when deserialized from cache so need to make up one.
HFileContextBuilder fileContextBuilder =
fileContext != null ? new HFileContextBuilder(fileContext) : new HFileContextBuilder();
HFileContextBuilder fileContextBuilder = fileContext != null ?
new HFileContextBuilder(fileContext) : new HFileContextBuilder();
fileContextBuilder.withHBaseCheckSum(usesHBaseChecksum);
int onDiskDataSizeWithHeader;
if (usesHBaseChecksum) {
@ -705,8 +702,8 @@ public class HFileBlock implements Cacheable {
}
/**
* Cannot be {@link #UNSET}. Must be a legitimate value. Used re-making the {@link BlockCacheKey} when
* block is returned to the cache.
* Cannot be {@link #UNSET}. Must be a legitimate value. Used re-making the {@link BlockCacheKey}
* when block is returned to the cache.
* @return the offset of this block in the file it was read from
*/
long getOffset() {
@ -892,7 +889,6 @@ public class HFileBlock implements Cacheable {
* Starts writing into the block. The previous block's data is discarded.
*
* @return the stream the user can write their data into
* @throws IOException
*/
DataOutputStream startWriting(BlockType newBlockType)
throws IOException {
@ -922,8 +918,6 @@ public class HFileBlock implements Cacheable {
/**
* Writes the Cell to this block
* @param cell
* @throws IOException
*/
void write(Cell cell) throws IOException{
expectState(State.WRITING);
@ -1041,9 +1035,6 @@ public class HFileBlock implements Cacheable {
* Similar to {@link #writeHeaderAndData(FSDataOutputStream)}, but records
* the offset of this block so that it can be referenced in the next block
* of the same type.
*
* @param out
* @throws IOException
*/
void writeHeaderAndData(FSDataOutputStream out) throws IOException {
long offset = out.getPos();
@ -1061,9 +1052,7 @@ public class HFileBlock implements Cacheable {
* data when not using compression) into the given stream. Can be called in
* the "writing" state or in the "block ready" state. If called in the
* "writing" state, transitions the writer to the "block ready" state.
*
* @param out the output stream to write the
* @throws IOException
*/
protected void finishBlockAndWriteHeaderAndData(DataOutputStream out)
throws IOException {
@ -1082,7 +1071,6 @@ public class HFileBlock implements Cacheable {
* the header + data + checksums stored on disk.
*
* @return header and data as they would be stored on disk in a byte array
* @throws IOException
*/
byte[] getHeaderAndDataForTest() throws IOException {
ensureBlockReady();
@ -1167,9 +1155,7 @@ public class HFileBlock implements Cacheable {
* @return the number of bytes written
*/
public int encodedBlockSizeWritten() {
if (state != State.WRITING)
return 0;
return this.encodedDataSizeWritten;
return state != State.WRITING ? 0 : this.encodedDataSizeWritten;
}
/**
@ -1180,8 +1166,7 @@ public class HFileBlock implements Cacheable {
* @return the number of bytes written
*/
int blockSizeWritten() {
if (state != State.WRITING) return 0;
return this.unencodedDataSizeWritten;
return state != State.WRITING ? 0 : this.unencodedDataSizeWritten;
}
/**
@ -1234,7 +1219,6 @@ public class HFileBlock implements Cacheable {
*
* @param bw the block-writable object to write as a block
* @param out the file system output stream
* @throws IOException
*/
void writeBlock(BlockWritable bw, FSDataOutputStream out)
throws IOException {
@ -1431,7 +1415,8 @@ public class HFileBlock implements Cacheable {
* next blocks header seems unnecessary given we usually get the block size
* from the hfile index. Review!
*/
private AtomicReference<PrefetchedHeader> prefetchedHeader = new AtomicReference<>(new PrefetchedHeader());
private AtomicReference<PrefetchedHeader> prefetchedHeader =
new AtomicReference<>(new PrefetchedHeader());
/** The size of the file we are reading from, or -1 if unknown. */
private long fileSize;
@ -1643,7 +1628,6 @@ public class HFileBlock implements Cacheable {
/**
* @return Check <code>onDiskSizeWithHeaderL</code> size is healthy and then return it as an int
* @throws IOException
*/
private static int checkAndGetSizeAsInt(final long onDiskSizeWithHeaderL, final int hdrSize)
throws IOException {
@ -1659,7 +1643,6 @@ public class HFileBlock implements Cacheable {
/**
* Verify the passed in onDiskSizeWithHeader aligns with what is in the header else something
* is not right.
* @throws IOException
*/
private void verifyOnDiskSizeMatchesHeader(final int passedIn, final ByteBuff headerBuf,
final long offset, boolean verifyChecksum)
@ -1827,7 +1810,8 @@ public class HFileBlock implements Cacheable {
@Override
public void setIncludesMemStoreTS(boolean includesMemstoreTS) {
this.fileContext.setIncludesMvcc(includesMemstoreTS);
this.fileContext = new HFileContextBuilder(this.fileContext)
.withIncludesMvcc(includesMemstoreTS).build();
}
@Override

16
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
View File

@ -1,4 +1,4 @@
/**
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@ -33,7 +33,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@ -52,7 +51,14 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.BytesBytesP
import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
/**
* Metadata for HFile. Conjured by the writer. Read in by the reader.
* Metadata Map of attributes for HFile written out as HFile Trailer. Created by the Writer and
* added to the tail of the file just before close. Metadata includes core attributes such as last
* key seen, comparator used writing the file, etc. Clients can add their own attributes via
* {@link #append(byte[], byte[], boolean)} and they'll be persisted and available at read time.
* Reader creates the HFileInfo on open by reading the tail of the HFile. The parse of the HFile
* trailer also creates a {@link HFileContext}, a read-only data structure that includes bulk of
* the HFileInfo and extras that is safe to pass around when working on HFiles.
* @see HFileContext
*/
@InterfaceAudience.Private
public class HFileInfo implements SortedMap<byte[], byte[]> {
@ -62,7 +68,6 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
static final byte [] AVG_KEY_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN");
static final byte [] AVG_VALUE_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN");
static final byte [] CREATE_TIME_TS = Bytes.toBytes(RESERVED_PREFIX + "CREATE_TIME_TS");
static final byte [] COMPARATOR = Bytes.toBytes(RESERVED_PREFIX + "COMPARATOR");
static final byte [] TAGS_COMPRESSED = Bytes.toBytes(RESERVED_PREFIX + "TAGS_COMPRESSED");
public static final byte [] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX + "MAX_TAGS_LEN");
private final SortedMap<byte [], byte []> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
@ -380,7 +385,8 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
HFileContextBuilder builder = new HFileContextBuilder()
.withHBaseCheckSum(true)
.withHFileName(path.getName())
.withCompression(trailer.getCompressionCodec());
.withCompression(trailer.getCompressionCodec())
.withCellComparator(trailer.createComparator(trailer.getComparatorClassName()));
// Check for any key material available
byte[] keyBytes = trailer.getEncryptionKey();
if (keyBytes != null) {

51
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
View File

@ -22,7 +22,6 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Optional;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@ -49,12 +48,10 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.IdLock;
import org.apache.hadoop.hbase.util.ObjectIntPair;
import org.apache.hadoop.io.WritableUtils;
import org.apache.htrace.core.TraceScope;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
@ -85,9 +82,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
*/
protected HFileDataBlockEncoder dataBlockEncoder = NoOpDataBlockEncoder.INSTANCE;
/** Key comparator */
protected CellComparator comparator = CellComparator.getInstance();
/** Block cache configuration. */
protected final CacheConfig cacheConf;
@ -145,8 +139,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
this.primaryReplicaReader = context.isPrimaryReplicaReader();
this.fileInfo = fileInfo;
this.trailer = fileInfo.getTrailer();
// Comparator class name is stored in the trailer in version 2.
this.comparator = trailer.createComparator();
this.hfileContext = fileInfo.getHFileContext();
this.fsBlockReader = new HFileBlock.FSReaderImpl(context, hfileContext,
cacheConf.getByteBuffAllocator());
@ -238,7 +230,7 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
/** @return comparator */
@Override
public CellComparator getComparator() {
return comparator;
return this.hfileContext.getCellComparator();
}
@VisibleForTesting
@ -398,9 +390,10 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
}
protected void assertSeeked() {
if (!isSeeked())
if (!isSeeked()) {
throw new NotSeekedException(reader.getPath());
}
}
@Override
public HFile.Reader getReader() {
@ -476,7 +469,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
/**
* Read mvcc. Does checks to see if we even need to read the mvcc at all.
* @param offsetFromPos
*/
protected void readMvccVersion(final int offsetFromPos) {
// See if we even need to decode mvcc.
@ -493,7 +485,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
/**
* Actually do the mvcc read. Does no checks.
* @param offsetFromPos
*/
private void _readMvccVersion(int offsetFromPos) {
// This is Bytes#bytesToVint inlined so can save a few instructions in this hot method; i.e.
@ -681,7 +672,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
* 0 if we are at the given key, 1 if we are past the given key
* -2 if the key is earlier than the first key of the file while
* using a faked index key
* @throws IOException
*/
public int seekTo(Cell key, boolean rewind) throws IOException {
HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
@ -747,7 +737,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
* data block is found.
*
* @return the next block, or null if there are no more data blocks
* @throws IOException
*/
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH",
justification="Yeah, unnecessary null check; could do w/ clean up")
@ -764,7 +753,7 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
}
if (block.getOffset() < 0) {
releaseIfNotCurBlock(block);
throw new IOException("Invalid block file offset: " + block + ", path=" + reader.getPath());
throw new IOException("Invalid block offset: " + block + ", path=" + reader.getPath());
}
// We are reading the next block without block type validation, because
// it might turn out to be a non-data block.
@ -786,8 +775,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
@Override
public Cell getCell() {
if (!isSeeked())
if (!isSeeked()) {
return null;
}
Cell ret;
int cellBufSize = getKVBufSize();
@ -886,7 +876,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
/**
* Set our selves up for the next 'next' invocation, set up next block.
* @return True is more to read else false if at the end.
* @throws IOException
*/
private boolean positionForNextBlock() throws IOException {
// Methods are small so they get inlined because they are 'hot'.
@ -942,7 +931,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
*
* @return false if empty file; i.e. a call to next would return false and
* the current key and value are undefined.
* @throws IOException
*/
@Override
public boolean seekTo() throws IOException {
@ -975,7 +963,8 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
if (newBlock.getOffset() < 0) {
releaseIfNotCurBlock(newBlock);
throw new IOException("Invalid block offset: " + newBlock.getOffset() + ", path=" + reader.getPath());
throw new IOException("Invalid offset=" + newBlock.getOffset() +
", path=" + reader.getPath());
}
updateCurrentBlock(newBlock);
}
@ -993,7 +982,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
}
/**
* @param v
* @return True if v &lt;= 0 or v &gt; current block buffer limit.
*/
protected final boolean checkKeyLen(final int v) {
@ -1001,7 +989,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
}
/**
* @param v
* @return True if v &lt; 0 or v &gt; current block buffer limit.
*/
protected final boolean checkLen(final int v) {
@ -1172,9 +1159,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
// so blocks with the old encoding still linger in cache for some
// period of time. This event should be rare as it only happens on
// schema definition change.
LOG.info("Evicting cached block with key " + cacheKey
+ " because of a data block encoding mismatch" + "; expected: "
+ expectedDataBlockEncoding + ", actual: " + actualDataBlockEncoding + ", path=" + path);
LOG.info("Evicting cached block with key {} because data block encoding mismatch; " +
"expected {}, actual {}, path={}", cacheKey, actualDataBlockEncoding,
expectedDataBlockEncoding, path);
// This is an error scenario. so here we need to release the block.
returnAndEvictBlock(cache, cacheKey, cachedBlock);
}
@ -1192,10 +1179,8 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
}
/**
* @param metaBlockName
* @param cacheBlock Add block to cache, if found
* @return block wrapped in a ByteBuffer, with header skipped
* @throws IOException
*/
@Override
public HFileBlock getMetaBlock(String metaBlockName, boolean cacheBlock)
@ -1210,8 +1195,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
byte[] mbname = Bytes.toBytes(metaBlockName);
int block = metaBlockIndexReader.rootBlockContainingKey(mbname,
0, mbname.length);
if (block == -1)
if (block == -1) {
return null;
}
long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
// Per meta key from any given file, synchronize reads for said block. This
@ -1418,7 +1404,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
/**
* @return Midkey for this file. We work with block boundaries only so
* returned midkey is an approximation only.
* @throws IOException
*/
@Override
public Optional<Cell> midKey() throws IOException {
@ -1455,8 +1440,7 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
DataBlockEncoding encoding = reader.getDataBlockEncoding();
dataBlockEncoder = encoding.getEncoder();
decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
seeker = dataBlockEncoder.createSeeker(
reader.getComparator(), decodingCtx);
seeker = dataBlockEncoder.createSeeker(decodingCtx);
}
@Override
@ -1475,7 +1459,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
* @param newBlock the block to make current, and read by {@link HFileReaderImpl#readBlock},
* it's a totally new block with new allocated {@link ByteBuff}, so if no further
* reference to this block, we should release it carefully.
* @throws CorruptHFileException
*/
@Override
protected void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
@ -1487,9 +1470,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
short dataBlockEncoderId = newBlock.getDataBlockEncodingId();
if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
String encoderCls = dataBlockEncoder.getClass().getName();
throw new CorruptHFileException(
"Encoder " + encoderCls + " doesn't support data block encoding "
+ DataBlockEncoding.getNameFromId(dataBlockEncoderId) + ",path=" + reader.getPath());
throw new CorruptHFileException("Encoder " + encoderCls +
" doesn't support data block encoding " +
DataBlockEncoding.getNameFromId(dataBlockEncoderId) + ",path=" + reader.getPath());
}
updateCurrBlockRef(newBlock);
ByteBuff encodedBuffer = getEncodedBuffer(newBlock);

68
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java
View File

@ -25,7 +25,6 @@ import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@ -65,7 +64,8 @@ public class HFileWriterImpl implements HFile.Writer {
private static final long UNSET = -1;
/** if this feature is enabled, preCalculate encoded data size before real encoding happens*/
public static final String UNIFIED_ENCODED_BLOCKSIZE_RATIO = "hbase.writer.unified.encoded.blocksize.ratio";
public static final String UNIFIED_ENCODED_BLOCKSIZE_RATIO =
"hbase.writer.unified.encoded.blocksize.ratio";
/** Block size limit after encoding, used to unify encoded block Cache entry size*/
private final int encodedBlockSizeLimit;
@ -94,9 +94,6 @@ public class HFileWriterImpl implements HFile.Writer {
/** Total uncompressed bytes, maybe calculate a compression ratio later. */
protected long totalUncompressedBytes = 0;
/** Key comparator. Used to ensure we write in order. */
protected final CellComparator comparator;
/** Meta block names. */
protected List<byte[]> metaNames = new ArrayList<>();
@ -165,8 +162,7 @@ public class HFileWriterImpl implements HFile.Writer {
protected long maxMemstoreTS = 0;
public HFileWriterImpl(final Configuration conf, CacheConfig cacheConf, Path path,
FSDataOutputStream outputStream,
CellComparator comparator, HFileContext fileContext) {
FSDataOutputStream outputStream, HFileContext fileContext) {
this.outputStream = outputStream;
this.path = path;
this.name = path != null ? path.getName() : outputStream.toString();
@ -177,8 +173,6 @@ public class HFileWriterImpl implements HFile.Writer {
} else {
this.blockEncoder = NoOpDataBlockEncoder.INSTANCE;
}
this.comparator = comparator != null ? comparator : CellComparator.getInstance();
closeOutputStream = path != null;
this.cacheConf = cacheConf;
float encodeBlockSizeRatio = conf.getFloat(UNIFIED_ENCODED_BLOCKSIZE_RATIO, 1f);
@ -187,7 +181,6 @@ public class HFileWriterImpl implements HFile.Writer {
if (LOG.isTraceEnabled()) {
LOG.trace("Writer" + (path != null ? " for " + path : "") +
" initialized with cacheConf: " + cacheConf +
" comparator: " + comparator.getClass().getSimpleName() +
" fileContext: " + fileContext);
}
}
@ -214,7 +207,6 @@ public class HFileWriterImpl implements HFile.Writer {
*
* @param trailer fixed file trailer
* @param out the data output to write the file info to
* @throws IOException
*/
protected final void writeFileInfo(FixedFileTrailer trailer, DataOutputStream out)
throws IOException {
@ -239,7 +231,8 @@ public class HFileWriterImpl implements HFile.Writer {
throw new IOException("Key cannot be null or empty");
}
if (lastCell != null) {
int keyComp = PrivateCellUtil.compareKeyIgnoresMvcc(comparator, lastCell, cell);
int keyComp = PrivateCellUtil.compareKeyIgnoresMvcc(this.hFileContext.getCellComparator(),
lastCell, cell);
if (keyComp > 0) {
String message = getLexicalErrorMessage(cell);
throw new IOException(message);
@ -285,8 +278,9 @@ public class HFileWriterImpl implements HFile.Writer {
}
public static Compression.Algorithm compressionByName(String algoName) {
if (algoName == null)
if (algoName == null) {
return HFile.DEFAULT_COMPRESSION_ALGORITHM;
}
return Compression.getCompressionAlgorithmByName(algoName);
}
@ -318,17 +312,15 @@ public class HFileWriterImpl implements HFile.Writer {
// Meta data block index writer
metaBlockIndexWriter = new HFileBlockIndex.BlockIndexWriter();
if (LOG.isTraceEnabled()) LOG.trace("Initialized with " + cacheConf);
LOG.trace("Initialized with {}", cacheConf);
}
/**
* At a block boundary, write all the inline blocks and opens new block.
*
* @throws IOException
*/
protected void checkBlockBoundary() throws IOException {
//for encoder like prefixTree, encoded size is not available, so we have to compare both encoded size
//and unencoded size to blocksize limit.
// For encoder like prefixTree, encoded size is not available, so we have to compare both
// encoded size and unencoded size to blocksize limit.
if (blockWriter.encodedBlockSizeWritten() >= encodedBlockSizeLimit
|| blockWriter.blockSizeWritten() >= hFileContext.getBlocksize()) {
finishBlock();
@ -339,7 +331,9 @@ public class HFileWriterImpl implements HFile.Writer {
/** Clean up the data block that is currently being written.*/
private void finishBlock() throws IOException {
if (!blockWriter.isWriting() || blockWriter.blockSizeWritten() == 0) return;
if (!blockWriter.isWriting() || blockWriter.blockSizeWritten() == 0) {
return;
}
// Update the first data block offset if UNSET; used scanning.
if (firstDataBlockOffset == UNSET) {
@ -350,7 +344,7 @@ public class HFileWriterImpl implements HFile.Writer {
blockWriter.writeHeaderAndData(outputStream);
int onDiskSize = blockWriter.getOnDiskSizeWithHeader();
Cell indexEntry =
getMidpoint(this.comparator, lastCellOfPreviousBlock, firstCellInBlock);
getMidpoint(this.hFileContext.getCellComparator(), lastCellOfPreviousBlock, firstCellInBlock);
dataBlockIndexWriter.addEntry(PrivateCellUtil.getCellKeySerializedAsKeyValueKey(indexEntry),
lastDataBlockOffset, onDiskSize);
totalUncompressedBytes += blockWriter.getUncompressedSizeWithHeader();
@ -364,11 +358,6 @@ public class HFileWriterImpl implements HFile.Writer {
* <code>right</code> but that is shorter; i.e. takes up less space. This
* trick is used building HFile block index. Its an optimization. It does not
* always work. In this case we'll just return the <code>right</code> cell.
*
* @param comparator
* Comparator to use.
* @param left
* @param right
* @return A cell that sorts between <code>left</code> and <code>right</code>.
*/
public static Cell getMidpoint(final CellComparator comparator, final Cell left,
@ -407,7 +396,9 @@ public class HFileWriterImpl implements HFile.Writer {
left.getRowLength(), right.getRowArray(), right.getRowOffset(), right.getRowLength());
}
// If midRow is null, just return 'right'. Can't do optimization.
if (midRow == null) return right;
if (midRow == null) {
return right;
}
return PrivateCellUtil.createFirstOnRow(midRow);
}
// Rows are same. Compare on families.
@ -428,7 +419,9 @@ public class HFileWriterImpl implements HFile.Writer {
right.getFamilyLength());
}
// If midRow is null, just return 'right'. Can't do optimization.
if (midRow == null) return right;
if (midRow == null) {
return right;
}
// Return new Cell where we use right row and then a mid sort family.
return PrivateCellUtil.createFirstOnRowFamily(right, midRow, 0, midRow.length);
}
@ -450,7 +443,9 @@ public class HFileWriterImpl implements HFile.Writer {
right.getQualifierLength());
}
// If midRow is null, just return 'right'. Can't do optimization.
if (midRow == null) return right;
if (midRow == null) {
return right;
}
// Return new Cell where we use right row and family and then a mid sort qualifier.
return PrivateCellUtil.createFirstOnRowCol(right, midRow, 0, midRow.length);
}
@ -459,12 +454,6 @@ public class HFileWriterImpl implements HFile.Writer {
}
/**
* @param leftArray
* @param leftOffset
* @param leftLength
* @param rightArray
* @param rightOffset
* @param rightLength
* @return Return a new array that is between left and right and minimally
* sized else just return null as indicator that we could not create a
* mid point.
@ -566,8 +555,6 @@ public class HFileWriterImpl implements HFile.Writer {
/**
* Ready a new block for writing.
*
* @throws IOException
*/
protected void newBlock() throws IOException {
// This is where the next block begins.
@ -678,7 +665,7 @@ public class HFileWriterImpl implements HFile.Writer {
dataBlockIndexWriter.getTotalUncompressedSize());
trailer.setFirstDataBlockOffset(firstDataBlockOffset);
trailer.setLastDataBlockOffset(lastDataBlockOffset);
trailer.setComparatorClass(comparator.getClass());
trailer.setComparatorClass(this.hFileContext.getCellComparator().getClass());
trailer.setDataIndexCount(dataBlockIndexWriter.getNumRootEntries());
@ -704,8 +691,9 @@ public class HFileWriterImpl implements HFile.Writer {
private void addBloomFilter(final BloomFilterWriter bfw,
final BlockType blockType) {
if (bfw.getKeyCount() <= 0)
if (bfw.getKeyCount() <= 0) {
return;
}
if (blockType != BlockType.GENERAL_BLOOM_META &&
blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
@ -722,9 +710,10 @@ public class HFileWriterImpl implements HFile.Writer {
public void writeToBlock(DataOutput out) throws IOException {
bfw.getMetaWriter().write(out);
Writable dataWriter = bfw.getDataWriter();
if (dataWriter != null)
if (dataWriter != null) {
dataWriter.write(out);
}
}
});
}
@ -739,7 +728,6 @@ public class HFileWriterImpl implements HFile.Writer {
*
* @param cell
* Cell to add. Cannot be empty nor null.
* @throws IOException
*/
@Override
public void append(final Cell cell) throws IOException {

24
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java
View File

@ -1,4 +1,4 @@
/**
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@ -35,13 +35,11 @@ import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.TableName;
@ -124,7 +122,6 @@ public final class MobUtils {
* Parses the string to a date.
* @param dateString The string format of a date, it's yyyymmdd.
* @return A date.
* @throws ParseException
*/
public static Date parseDate(String dateString) throws ParseException {
return LOCAL_FORMAT.get().parse(dateString);
@ -288,7 +285,6 @@ public final class MobUtils {
* @param columnDescriptor The descriptor of the current column family.
* @param cacheConfig The cacheConfig that disables the block cache.
* @param current The current time.
* @throws IOException
*/
public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName,
ColumnFamilyDescriptor columnDescriptor, CacheConfig cacheConfig, long current)
@ -384,7 +380,6 @@ public final class MobUtils {
* Gets the qualified root dir of the mob files.
* @param conf The current configuration.
* @return The qualified root dir.
* @throws IOException
*/
public static Path getQualifiedMobRootDir(Configuration conf) throws IOException {
Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
@ -505,7 +500,6 @@ public final class MobUtils {
* @param tableDir The table directory.
* @param family The name of the column family.
* @param storeFiles The files to be deleted.
* @throws IOException
*/
public static void removeMobFiles(Configuration conf, FileSystem fs, TableName tableName,
Path tableDir, byte[] family, Collection<HStoreFile> storeFiles) throws IOException {
@ -556,7 +550,6 @@ public final class MobUtils {
* @param cryptoContext The encryption context.
* @param isCompaction If the writer is used in compaction.
* @return The writer for the mob file.
* @throws IOException
*/
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount,
@ -580,7 +573,6 @@ public final class MobUtils {
* @param cryptoContext The encryption context.
* @param isCompaction If the writer is used in compaction.
* @return The writer for the mob file.
* @throws IOException
*/
public static StoreFileWriter createRefFileWriter(Configuration conf, FileSystem fs,
ColumnFamilyDescriptor family, Path basePath, long maxKeyCount, CacheConfig cacheConfig,
@ -607,7 +599,6 @@ public final class MobUtils {
* @param cryptoContext The encryption context.
* @param isCompaction If the writer is used in compaction.
* @return The writer for the mob file.
* @throws IOException
*/
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount,
@ -633,7 +624,6 @@ public final class MobUtils {
* @param cacheConfig The current cache config.
* @param cryptoContext The encryption context.
* @return The writer for the del file.
* @throws IOException
*/
public static StoreFileWriter createDelFileWriter(Configuration conf, FileSystem fs,
ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount,
@ -660,7 +650,6 @@ public final class MobUtils {
* @param cryptoContext The encryption context.
* @param isCompaction If the writer is used in compaction.
* @return The writer for the mob file.
* @throws IOException
*/
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
ColumnFamilyDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount,
@ -689,7 +678,6 @@ public final class MobUtils {
* @param bloomType The bloom filter type.
* @param isCompaction If the writer is used in compaction.
* @return The writer for the mob file.
* @throws IOException
*/
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
ColumnFamilyDescriptor family, Path path, long maxKeyCount,
@ -718,8 +706,7 @@ public final class MobUtils {
.withCreateTime(EnvironmentEdgeManager.currentTime()).build();
StoreFileWriter w = new StoreFileWriter.Builder(conf, writerCacheConf, fs)
.withFilePath(path)
.withComparator(CellComparator.getInstance()).withBloomType(bloomType)
.withFilePath(path).withBloomType(bloomType)
.withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build();
return w;
}
@ -732,7 +719,6 @@ public final class MobUtils {
* @param targetPath The directory path where the source file is renamed to.
* @param cacheConfig The current cache config.
* @return The target file path the source file is renamed to.
* @throws IOException
*/
public static Path commitFile(Configuration conf, FileSystem fs, final Path sourceFile,
Path targetPath, CacheConfig cacheConfig) throws IOException {
@ -837,7 +823,8 @@ public final class MobUtils {
* @param allFiles Whether add all mob files into the compaction.
*/
public static void doMobCompaction(Configuration conf, FileSystem fs, TableName tableName,
ColumnFamilyDescriptor hcd, ExecutorService pool, boolean allFiles, LockManager.MasterLock lock)
ColumnFamilyDescriptor hcd, ExecutorService pool, boolean allFiles,
LockManager.MasterLock lock)
throws IOException {
String className = conf.get(MobConstants.MOB_COMPACTOR_CLASS_KEY,
PartitionedMobCompactor.class.getName());
@ -943,7 +930,8 @@ public final class MobUtils {
* @param fileDate The date string parsed from the mob file name.
* @return True if the mob file is expired.
*/
public static boolean isMobFileExpired(ColumnFamilyDescriptor column, long current, String fileDate) {
public static boolean isMobFileExpired(ColumnFamilyDescriptor column, long current,
String fileDate) {
if (column.getMinVersions() > 0) {
return false;
}

88
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
View File

@ -52,7 +52,6 @@ import java.util.function.Predicate;
import java.util.function.ToLongFunction;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@ -132,7 +131,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDes
* not be called directly but by an HRegion manager.
*/
@InterfaceAudience.Private
public class HStore implements Store, HeapSize, StoreConfigInformation, PropagatingConfigurationObserver {
public class HStore implements Store, HeapSize, StoreConfigInformation,
PropagatingConfigurationObserver {
public static final String MEMSTORE_CLASS_NAME = "hbase.regionserver.memstore.class";
public static final String COMPACTCHECKER_INTERVAL_MULTIPLIER_KEY =
"hbase.server.compactchecker.interval.multiplier";
@ -237,11 +237,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
/**
* Constructor
* @param region
* @param family HColumnDescriptor for this column
* @param confParam configuration object
* failed. Can be null.
* @throws IOException
* @param confParam configuration object failed. Can be null.
*/
protected HStore(final HRegion region, final ColumnFamilyDescriptor family,
final Configuration confParam, boolean warmup) throws IOException {
@ -399,7 +396,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
}
/**
* @param family
* @return TTL in seconds of the specified family
*/
public static long determineTTLFromFamily(final ColumnFamilyDescriptor family) {
@ -521,6 +517,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
* @param hri {@link RegionInfo} for the region.
* @param family {@link ColumnFamilyDescriptor} describing the column family
* @return Path to family/Store home directory.
* @deprecated Since 05/05/2013, HBase-7808, hbase-1.0.0
*/
@Deprecated
public static Path getStoreHomedir(final Path tabledir,
@ -533,6 +530,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
* @param encodedName Encoded region name.
* @param family {@link ColumnFamilyDescriptor} describing the column family
* @return Path to family/Store home directory.
* @deprecated Since 05/05/2013, HBase-7808, hbase-1.0.0
*/
@Deprecated
public static Path getStoreHomedir(final Path tabledir,
@ -558,7 +556,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
/**
* Creates an unsorted list of StoreFile loaded in parallel
* from the given directory.
* @throws IOException
*/
private List<HStoreFile> loadStoreFiles(boolean warmup) throws IOException {
Collection<StoreFileInfo> files = fs.getStoreFiles(getColumnFamilyName());
@ -574,7 +571,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
ThreadPoolExecutor storeFileOpenerThreadPool =
this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-"
+ this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName());
CompletionService<HStoreFile> completionService = new ExecutorCompletionService<>(storeFileOpenerThreadPool);
CompletionService<HStoreFile> completionService =
new ExecutorCompletionService<>(storeFileOpenerThreadPool);
int totalValidStoreFile = 0;
for (StoreFileInfo storeFileInfo : files) {
@ -596,9 +594,13 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
compactedStoreFiles.addAll(storeFile.getCompactedStoreFiles());
}
} catch (InterruptedException e) {
if (ioe == null) ioe = new InterruptedIOException(e.getMessage());
if (ioe == null) {
ioe = new InterruptedIOException(e.getMessage());
}
} catch (ExecutionException e) {
if (ioe == null) ioe = new IOException(e.getCause());
if (ioe == null) {
ioe = new IOException(e.getCause());
}
}
}
} finally {
@ -650,7 +652,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
/**
* Replaces the store files that the store has with the given files. Mainly used by secondary
* region replicas to keep up to date with the primary region files.
* @throws IOException
*/
public void refreshStoreFiles(Collection<String> newFiles) throws IOException {
List<StoreFileInfo> storeFiles = new ArrayList<>(newFiles.size());
@ -665,15 +666,20 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
* been opened, and removes the store file readers for store files no longer
* available. Mainly used by secondary region replicas to keep up to date with
* the primary region files.
* @throws IOException
*/
private void refreshStoreFilesInternal(Collection<StoreFileInfo> newFiles) throws IOException {
StoreFileManager sfm = storeEngine.getStoreFileManager();
Collection<HStoreFile> currentFiles = sfm.getStorefiles();
Collection<HStoreFile> compactedFiles = sfm.getCompactedfiles();
if (currentFiles == null) currentFiles = Collections.emptySet();
if (newFiles == null) newFiles = Collections.emptySet();
if (compactedFiles == null) compactedFiles = Collections.emptySet();
if (currentFiles == null) {
currentFiles = Collections.emptySet();
}
if (newFiles == null) {
newFiles = Collections.emptySet();
}
if (compactedFiles == null) {
compactedFiles = Collections.emptySet();
}
HashMap<StoreFileInfo, HStoreFile> currentFilesSet = new HashMap<>(currentFiles.size());
for (HStoreFile sf : currentFiles) {
@ -868,12 +874,13 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
}
prevCell = cell;
} while (scanner.next());
LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString()
+ " took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime)
+ " ms");
LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString() +
" took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime) + " ms");
}
} finally {
if (reader != null) reader.close();
if (reader != null) {
reader.close();
}
}
}
@ -881,7 +888,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
* This method should only be called from Region. It is assumed that the ranges of values in the
* HFile fit within the stores assigned region. (assertBulkLoadHFileOk checks this)
*
* @param srcPathStr
* @param seqNum sequence Id associated with the HFile
*/
public Pair<Path, Path> preBulkLoadHFile(String srcPathStr, long seqNum) throws IOException {
@ -993,13 +999,17 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
ioe.initCause(e);
}
} catch (ExecutionException e) {
if (ioe == null) ioe = new IOException(e.getCause());
if (ioe == null) {
ioe = new IOException(e.getCause());
}
}
}
} finally {
storeFileCloserThreadPool.shutdownNow();
}
if (ioe != null) throw ioe;
if (ioe != null) {
throw ioe;
}
}
LOG.trace("Closed {}", this);
return result;
@ -1027,9 +1037,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
/**
* Write out current snapshot. Presumes {@link #snapshot()} has been called previously.
* @param logCacheFlushId flush sequence number
* @param snapshot
* @param status
* @param throughputController
* @return The path name of the tmp file to which the store was flushed
* @throws IOException if exception occurs during process
*/
@ -1081,10 +1088,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
/**
* @param path The pathname of the tmp file into which the store was flushed
* @param logCacheFlushId
* @param status
* @return store file created.
* @throws IOException
*/
private HStoreFile commitFile(Path path, long logCacheFlushId, MonitoredTask status)
throws IOException {
@ -1107,7 +1111,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
}
/**
* @param maxKeyCount
* @param compression Compression algorithm to use
* @param isCompaction whether we are creating a new file in a compaction
* @param includeMVCCReadpoint - whether to include MVCC or not
@ -1162,7 +1165,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf,
this.getFileSystem())
.withOutputDir(familyTempDir)
.withComparator(comparator)
.withBloomType(family.getBloomFilterType())
.withMaxKeyCount(maxKeyCount)
.withFavoredNodes(favoredNodes)
@ -1192,6 +1194,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
.withColumnFamily(family.getName())
.withTableName(region.getTableDescriptor()
.getTableName().getName())
.withCellComparator(this.comparator)
.build();
return hFileContext;
}
@ -1204,8 +1207,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
/**
* Change storeFiles adding into place the Reader produced by this new flush.
* @param sfs Store files
* @param snapshotId
* @throws IOException
* @return Whether compaction is required.
*/
private boolean updateStorefiles(List<HStoreFile> sfs, long snapshotId) throws IOException {
@ -1237,7 +1238,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
/**
* Notify all observers that set of Readers has changed.
* @throws IOException
*/
private void notifyChangedReadersObservers(List<HStoreFile> sfs) throws IOException {
for (ChangedReadersObserver o : this.changedReaderObservers) {
@ -1454,7 +1454,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
* See HBASE-2231 for details.
*
* @param compaction compaction details obtained from requestCompaction()
* @throws IOException
* @return Storefile we compacted into or null if we failed or opted out early.
*/
public List<HStoreFile> compact(CompactionContext compaction,
@ -1521,8 +1520,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
return sfs;
}
private List<HStoreFile> moveCompactedFilesIntoPlace(CompactionRequestImpl cr, List<Path> newFiles,
User user) throws IOException {
private List<HStoreFile> moveCompactedFilesIntoPlace(CompactionRequestImpl cr,
List<Path> newFiles, User user) throws IOException {
List<HStoreFile> sfs = new ArrayList<>(newFiles.size());
for (Path newFile : newFiles) {
assert newFile != null;
@ -1560,7 +1559,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
newFiles.stream().map(HStoreFile::getPath).collect(Collectors.toList());
RegionInfo info = this.region.getRegionInfo();
CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
family.getName(), inputPaths, outputPaths, fs.getStoreDir(getColumnFamilyDescriptor().getNameAsString()));
family.getName(), inputPaths, outputPaths,
fs.getStoreDir(getColumnFamilyDescriptor().getNameAsString()));
// Fix reaching into Region to get the maxWaitForSeqId.
// Does this method belong in Region altogether given it is making so many references up there?
// Could be Region#writeCompactionMarker(compactionDescriptor);
@ -1664,7 +1664,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
* Call to complete a compaction. Its for the case where we find in the WAL a compaction
* that was not finished. We could find one recovering a WAL after a regionserver crash.
* See HBASE-2231.
* @param compaction
*/
public void replayCompactionMarker(CompactionDescriptor compaction, boolean pickCompactionFiles,
boolean removeFiles) throws IOException {
@ -1917,7 +1916,9 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
}
private void removeUnneededFiles() throws IOException {
if (!conf.getBoolean("hbase.store.delete.expired.storefile", true)) return;
if (!conf.getBoolean("hbase.store.delete.expired.storefile", true)) {
return;
}
if (getColumnFamilyDescriptor().getMinVersions() > 0) {
LOG.debug("Skipping expired store file removal due to min version being {}",
getColumnFamilyDescriptor().getMinVersions());
@ -2120,7 +2121,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
* @param readPt the read point of the current scane
* @param includeMemstoreScanner whether the current scanner should include memstorescanner
* @return list of scanners recreated on the current Scanners
* @throws IOException
*/
public List<KeyValueScanner> recreateScanners(List<KeyValueScanner> currentFileScanners,
boolean cacheBlocks, boolean usePread, boolean isCompaction, ScanQueryMatcher matcher,
@ -2319,7 +2319,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
* This operation is atomic on each KeyValue (row/family/qualifier) but not necessarily atomic
* across all of them.
* @param readpoint readpoint below which we can safely remove duplicate KVs
* @throws IOException
*/
public void upsert(Iterable<Cell> cells, long readpoint, MemStoreSizing memstoreSizing)
throws IOException {
@ -2433,7 +2432,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
* snapshot depending on dropMemstoreSnapshot argument.
* @param fileNames names of the flushed files
* @param dropMemstoreSnapshot whether to drop the prepared memstore snapshot
* @throws IOException
*/
@Override
public void replayFlush(List<String> fileNames, boolean dropMemstoreSnapshot)
@ -2465,7 +2463,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
/**
* Abort the snapshot preparation. Drops the snapshot if any.
* @throws IOException
*/
@Override
public void abort() throws IOException {
@ -2720,7 +2717,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
LOG.debug("Moving the files {} to archive", filesToRemove);
// Only if this is successful it has to be removed
try {
this.fs.removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(), filesToRemove);
this.fs.removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(),
filesToRemove);
} catch (FailedArchiveException fae) {
// Even if archiving some files failed, we still need to clear out any of the
// files which were successfully archived. Otherwise we will receive a

42
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
View File

@ -26,7 +26,6 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_K
import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAX_SEQ_ID_KEY;
import static org.apache.hadoop.hbase.regionserver.HStoreFile.MOB_CELLS_COUNT;
import static org.apache.hadoop.hbase.regionserver.HStoreFile.TIMERANGE_KEY;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Collection;
@ -37,15 +36,13 @@ import java.util.UUID;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
@ -62,9 +59,7 @@ import org.apache.hadoop.hbase.util.RowPrefixFixedLengthBloomContext;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
/**
@ -94,7 +89,6 @@ public class StoreFileWriter implements CellSink, ShipperListener {
* @param fs file system to write to
* @param path file name to create
* @param conf user configuration
* @param comparator key comparator
* @param bloomType bloom filter setting
* @param maxKeys the expected maximum number of keys to be added. Was used
* for Bloom filter size in {@link HFile} format version 1.
@ -105,15 +99,14 @@ public class StoreFileWriter implements CellSink, ShipperListener {
* @throws IOException problem writing to FS
*/
private StoreFileWriter(FileSystem fs, Path path, final Configuration conf, CacheConfig cacheConf,
final CellComparator comparator, BloomType bloomType, long maxKeys,
InetSocketAddress[] favoredNodes, HFileContext fileContext, boolean shouldDropCacheBehind,
Supplier<Collection<HStoreFile>> compactedFilesSupplier) throws IOException {
BloomType bloomType, long maxKeys, InetSocketAddress[] favoredNodes, HFileContext fileContext,
boolean shouldDropCacheBehind, Supplier<Collection<HStoreFile>> compactedFilesSupplier)
throws IOException {
this.compactedFilesSupplier = compactedFilesSupplier;
this.timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC);
// TODO : Change all writers to be specifically created for compaction context
writer = HFile.getWriterFactory(conf, cacheConf)
.withPath(fs, path)
.withComparator(comparator)
.withFavoredNodes(favoredNodes)
.withFileContext(fileContext)
.withShouldDropCacheBehind(shouldDropCacheBehind)
@ -135,14 +128,16 @@ public class StoreFileWriter implements CellSink, ShipperListener {
// init bloom context
switch (bloomType) {
case ROW:
bloomContext = new RowBloomContext(generalBloomFilterWriter, comparator);
bloomContext =
new RowBloomContext(generalBloomFilterWriter, fileContext.getCellComparator());
break;
case ROWCOL:
bloomContext = new RowColBloomContext(generalBloomFilterWriter, comparator);
bloomContext =
new RowColBloomContext(generalBloomFilterWriter, fileContext.getCellComparator());
break;
case ROWPREFIX_FIXED_LENGTH:
bloomContext = new RowPrefixFixedLengthBloomContext(generalBloomFilterWriter, comparator,
Bytes.toInt(bloomParam));
bloomContext = new RowPrefixFixedLengthBloomContext(generalBloomFilterWriter,
fileContext.getCellComparator(), Bytes.toInt(bloomParam));
break;
default:
throw new IOException(
@ -159,7 +154,8 @@ public class StoreFileWriter implements CellSink, ShipperListener {
this.deleteFamilyBloomFilterWriter = BloomFilterFactory
.createDeleteBloomAtWrite(conf, cacheConf,
(int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
deleteFamilyBloomContext = new RowBloomContext(deleteFamilyBloomFilterWriter, comparator);
deleteFamilyBloomContext =
new RowBloomContext(deleteFamilyBloomFilterWriter, fileContext.getCellComparator());
} else {
deleteFamilyBloomFilterWriter = null;
}
@ -396,7 +392,6 @@ public class StoreFileWriter implements CellSink, ShipperListener {
}
/**
* @param fs
* @param dir Directory to create file in.
* @return random filename inside passed <code>dir</code>
*/
@ -414,7 +409,6 @@ public class StoreFileWriter implements CellSink, ShipperListener {
private final CacheConfig cacheConf;
private final FileSystem fs;
private CellComparator comparator = CellComparator.getInstance();
private BloomType bloomType = BloomType.NONE;
private long maxKeyCount = 0;
private Path dir;
@ -473,12 +467,6 @@ public class StoreFileWriter implements CellSink, ShipperListener {
return this;
}
public Builder withComparator(CellComparator comparator) {
Preconditions.checkNotNull(comparator);
this.comparator = comparator;
return this;
}
public Builder withBloomType(BloomType bloomType) {
Preconditions.checkNotNull(bloomType);
this.bloomType = bloomType;
@ -545,11 +533,7 @@ public class StoreFileWriter implements CellSink, ShipperListener {
}
}
if (comparator == null) {
comparator = CellComparator.getInstance();
}
return new StoreFileWriter(fs, filePath, conf, cacheConf, comparator, bloomType, maxKeyCount,
return new StoreFileWriter(fs, filePath, conf, cacheConf, bloomType, maxKeyCount,
favoredNodes, fileContext, shouldDropCacheBehind, compactedFilesSupplier);
}
}

10
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.wal;
import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.HashMap;
@ -29,7 +28,6 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@ -208,7 +206,7 @@ public class BoundedRecoveredHFilesOutputSink extends OutputSink {
.withFilePath(outputFile);
HFileContextBuilder hFileContextBuilder = new HFileContextBuilder();
if (isMetaTable) {
writerBuilder.withComparator(CellComparatorImpl.META_COMPARATOR);
hFileContextBuilder.withCellComparator(CellComparatorImpl.META_COMPARATOR);
} else {
configContextForNonMetaWriter(tableName, familyName, hFileContextBuilder, writerBuilder);
}
@ -224,9 +222,9 @@ public class BoundedRecoveredHFilesOutputSink extends OutputSink {
TableDescriptor tableDesc = tableDescCache.get(tableName);
ColumnFamilyDescriptor cfd = tableDesc.getColumnFamily(Bytes.toBytesBinary(familyName));
hFileContextBuilder.withCompression(cfd.getCompressionType()).withBlockSize(cfd.getBlocksize())
.withCompressTags(cfd.isCompressTags()).withDataBlockEncoding(cfd.getDataBlockEncoding());
writerBuilder.withBloomType(cfd.getBloomFilterType())
.withComparator(CellComparatorImpl.COMPARATOR);
.withCompressTags(cfd.isCompressTags()).withDataBlockEncoding(cfd.getDataBlockEncoding())
.withCellComparator(CellComparatorImpl.COMPARATOR);
writerBuilder.withBloomType(cfd.getBloomFilterType());
}
private void checkPathValid(Path outputFile) throws IOException {

1
hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
View File

@ -380,7 +380,6 @@ public class HFilePerformanceEvaluation {
writer = HFile.getWriterFactoryNoCache(conf)
.withPath(fs, mf)
.withFileContext(hFileContext)
.withComparator(CellComparator.getInstance())
.create();
}

14
hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
View File

@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -206,8 +206,8 @@ public class TestDataBlockEncoders {
.withIncludesTags(includesTags)
.withCompression(Compression.Algorithm.NONE)
.build();
DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR,
encoder.newDataBlockDecodingContext(meta));
DataBlockEncoder.EncodedSeeker seeker =
encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
encodedSeekers.add(seeker);
}
@ -279,8 +279,8 @@ public class TestDataBlockEncoders {
.withIncludesTags(includesTags)
.withCompression(Compression.Algorithm.NONE)
.build();
DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR,
encoder.newDataBlockDecodingContext(meta));
DataBlockEncoder.EncodedSeeker seeker =
encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
int i = 0;
do {
@ -341,8 +341,8 @@ public class TestDataBlockEncoders {
HFileContext meta =
new HFileContextBuilder().withHBaseCheckSum(false).withIncludesMvcc(includesMemstoreTS)
.withIncludesTags(includesTags).withCompression(Compression.Algorithm.NONE).build();
DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR,
encoder.newDataBlockDecodingContext(meta));
DataBlockEncoder.EncodedSeeker seeker =
encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
Cell cell = seeker.getCell();
Assert.assertEquals(expectedKV.getLength(), ((KeyValue) cell).getLength());

5
hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekToBlockWithEncoders.java
View File

@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.HConstants;
@ -290,8 +289,8 @@ public class TestSeekToBlockWithEncoders {
HFILEBLOCK_DUMMY_HEADER, meta);
ByteBuffer encodedBuffer = TestDataBlockEncoders.encodeKeyValues(encoding, kvs,
encodingContext, this.useOffheapData);
DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR,
encoder.newDataBlockDecodingContext(meta));
DataBlockEncoder.EncodedSeeker seeker =
encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
encodedSeekers.add(seeker);
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
View File

@ -37,7 +37,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@ -394,7 +393,7 @@ public class TestCacheOnWrite {
.withDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding())
.withIncludesTags(useTags).build();
StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs)
.withOutputDir(storeFileParentDir).withComparator(CellComparatorImpl.COMPARATOR)
.withOutputDir(storeFileParentDir)
.withFileContext(meta)
.withBloomType(BLOOM_TYPE).withMaxKeyCount(NUM_KV).build();
byte[] cf = Bytes.toBytes("fam");

4
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
View File

@ -131,6 +131,10 @@ public class TestFixedFileTrailer {
t.createComparator(KeyValue.META_COMPARATOR.getClass().getName()).getClass());
assertEquals(CellComparatorImpl.MetaCellComparator.class, t.createComparator(
CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
assertEquals(CellComparatorImpl.META_COMPARATOR.getClass(), t.createComparator(
CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
assertEquals(CellComparatorImpl.COMPARATOR.getClass(), t.createComparator(
CellComparatorImpl.MetaCellComparator.COMPARATOR.getClass().getName()).getClass());
assertNull(t.createComparator(Bytes.BYTES_RAWCOMPARATOR.getClass().getName()));
assertNull(t.createComparator("org.apache.hadoop.hbase.KeyValue$RawBytesComparator"));
} catch (IOException e) {

6
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java
View File

@ -50,7 +50,6 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellBuilder;
import org.apache.hadoop.hbase.CellBuilderFactory;
import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
@ -277,7 +276,7 @@ public class TestHFile {
HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
StoreFileWriter sfw =
new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir)
.withComparator(CellComparatorImpl.COMPARATOR).withFileContext(meta).build();
.withFileContext(meta).build();
final int rowLen = 32;
Random RNG = new Random();
@ -360,7 +359,7 @@ public class TestHFile {
.withCompressTags(false)
.build();
HFileWriterImpl writer = new HFileWriterImpl(conf, cacheConf, path, mockedOutputStream,
CellComparator.getInstance(), fileContext);
fileContext);
CellBuilder cellBuilder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY);
byte[] row = Bytes.toBytes("foo");
byte[] qualifier = Bytes.toBytes("qualifier");
@ -516,7 +515,6 @@ public class TestHFile {
Writer writer = HFile.getWriterFactory(conf, cacheConf)
.withOutputStream(fout)
.withFileContext(meta)
.withComparator(CellComparatorImpl.COMPARATOR)
.create();
LOG.info(Objects.toString(writer));
writeRecords(writer, useTags);

6
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java
View File

@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.KeyValue;
@ -66,9 +65,8 @@ public class TestHFileReaderImpl {
HFileContext context =
new HFileContextBuilder().withBlockSize(blocksize).withIncludesTags(true).build();
Configuration conf = TEST_UTIL.getConfiguration();
HFile.Writer writer =
HFile.getWriterFactoryNoCache(conf).withOutputStream(fout).withFileContext(context)
.withComparator(CellComparatorImpl.COMPARATOR).create();
HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
.withOutputStream(fout).withFileContext(context).create();
// 4 bytes * 3 * 2 for each key/value +
// 3 for keys, 15 for values = 42 (woot)
writer.append(toKV("c"));

3
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileScannerImplReferenceCount.java
View File

@ -37,7 +37,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
@ -165,7 +164,7 @@ public class TestHFileScannerImplReferenceCount {
.withCompression(compression).withDataBlockEncoding(encoding).build();
try (HFile.Writer writer =
new HFile.WriterFactory(conf, new CacheConfig(conf)).withPath(fs, hfilePath)
.withFileContext(context).withComparator(CellComparatorImpl.COMPARATOR).create()) {
.withFileContext(context).create()) {
Random rand = new Random(9713312); // Just a fixed seed.
for (int i = 0; i < cellCount; ++i) {
byte[] keyBytes = Bytes.add(Bytes.toBytes(i), SUFFIX);

2
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
View File

@ -27,7 +27,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.KeyValue;
@ -142,7 +141,6 @@ public class TestHFileSeek extends TestCase {
Writer writer = HFile.getWriterFactoryNoCache(conf)
.withOutputStream(fout)
.withFileContext(context)
.withComparator(CellComparatorImpl.COMPARATOR)
.create();
try {
BytesWritable key = new BytesWritable();

2
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java
View File

@ -36,7 +36,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@ -132,7 +131,6 @@ public class TestHFileWriterV3 {
HFile.Writer writer = new HFile.WriterFactory(conf, cacheConfig)
.withPath(fs, hfilePath)
.withFileContext(context)
.withComparator(CellComparatorImpl.COMPARATOR)
.create();
Random rand = new Random(9713312); // Just a fixed seed.

2
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
View File

@ -26,7 +26,6 @@ import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@ -145,7 +144,6 @@ public class TestPrefetch {
.build();
StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs)
.withOutputDir(storeFileParentDir)
.withComparator(CellComparatorImpl.COMPARATOR)
.withFileContext(meta)
.build();

3
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java
View File

@ -25,7 +25,6 @@ import java.util.List;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
@ -69,8 +68,6 @@ public class TestReseekTo {
TEST_UTIL.getConfiguration(), cacheConf)
.withOutputStream(fout)
.withFileContext(context)
// NOTE: This test is dependent on this deprecated nonstandard comparator
.withComparator(CellComparatorImpl.COMPARATOR)
.create();
int numberOfKeys = 1000;

4
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java
View File

@ -33,7 +33,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.ByteBufferKeyValue;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
@ -123,8 +122,7 @@ public class TestSeekTo {
.withIncludesTags(true).build();
Configuration conf = TEST_UTIL.getConfiguration();
HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout)
.withFileContext(context)
.withComparator(CellComparatorImpl.COMPARATOR).create();
.withFileContext(context).create();
// 4 bytes * 3 * 2 for each key/value +
// 3 for keys, 15 for values = 42 (woot)
writer.append(toKV("c", tagUsage));

4
hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestIgnoreUnknownFamily.java
View File

@ -23,7 +23,7 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@ -84,7 +84,7 @@ public class TestIgnoreUnknownFamily {
Path familyDir = new Path(regionDir, Bytes.toString(UNKNOWN_FAMILY));
StoreFileWriter writer =
new StoreFileWriter.Builder(mfs.getConfiguration(), mfs.getFileSystem())
.withOutputDir(familyDir).withFileContext(new HFileContext()).build();
.withOutputDir(familyDir).withFileContext(new HFileContextBuilder().build()).build();
writer.close();
}

4
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java
View File

@ -52,7 +52,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@ -281,7 +281,7 @@ public class TestBulkLoad {
FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation), null);
try {
hFileFactory.withOutputStream(out);
hFileFactory.withFileContext(new HFileContext());
hFileFactory.withFileContext(new HFileContextBuilder().build());
HFile.Writer writer = hFileFactory.create();
try {
writer.append(new KeyValue(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)

2
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java
View File

@ -335,7 +335,7 @@ public class TestBulkLoadReplication extends TestReplicationBase {
new FSDataOutputStream(new FileOutputStream(hFileLocation), null);
try {
hFileFactory.withOutputStream(out);
hFileFactory.withFileContext(new HFileContext());
hFileFactory.withFileContext(new HFileContextBuilder().build());
HFile.Writer writer = hFileFactory.create();
try {
writer.append(new KeyValue(cellBuilder.build()));

24
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
View File

@ -24,9 +24,7 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
@ -41,6 +39,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptor;
@ -61,7 +61,8 @@ import org.slf4j.LoggerFactory;
/**
* TestGet is a medley of tests of get all done up as a single test.
* This class
* It was originally written to test a method since removed, getClosestAtOrBefore
* but the test is retained because it runs some interesting exercises.
*/
@Category({RegionServerTests.class, MediumTests.class})
public class TestGetClosestAtOrBefore {
@ -88,7 +89,6 @@ public class TestGetClosestAtOrBefore {
@Test
public void testUsingMetaAndBinary() throws IOException {
FileSystem filesystem = FileSystem.get(conf);
Path rootdir = UTIL.getDataTestDirOnTestFS();
// Up flush size else we bind up when we use default catalog flush of 16k.
TableDescriptors tds = new FSTableDescriptors(UTIL.getConfiguration());
@ -103,13 +103,14 @@ public class TestGetClosestAtOrBefore {
final int last = 128;
final int interval = 2;
for (int i = 0; i <= last; i += interval) {
HRegionInfo hri = new HRegionInfo(htd.getTableName(),
i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i),
i == last ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i + interval));
RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
.setStartKey(i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte)i))
.setEndKey(i == last ? HConstants.EMPTY_BYTE_ARRAY :
Bytes.toBytes((byte)i + interval)).build();
Put put =
MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
put.setDurability(Durability.SKIP_WAL);
LOG.info("Put {}", put);
mr.put(put);
}
}
@ -117,7 +118,7 @@ public class TestGetClosestAtOrBefore {
try {
List<Cell> keys = new ArrayList<>();
while (s.next(keys)) {
LOG.info(Objects.toString(keys));
LOG.info("Scan {}", keys);
keys.clear();
}
} finally {
@ -133,13 +134,14 @@ public class TestGetClosestAtOrBefore {
findRow(mr, 'C', 46, 46);
findRow(mr, 'C', 43, 42);
// Now delete 'C' and make sure I don't get entries from 'B'.
byte[] firstRowInC = HRegionInfo.createRegionName(TableName.valueOf("" + 'C'),
byte[] firstRowInC = RegionInfo.createRegionName(TableName.valueOf("" + 'C'),
HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false);
Scan scan = new Scan(firstRowInC);
Scan scan = new Scan().withStartRow(firstRowInC);
s = mr.getScanner(scan);
try {
List<Cell> keys = new ArrayList<>();
while (s.next(keys)) {
LOG.info("Delete {}", keys);
mr.delete(new Delete(CellUtil.cloneRow(keys.get(0))));
keys.clear();
}

4
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
View File

@ -63,7 +63,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.regionserver.HRegion.FlushResultImpl;
import org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult;
import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
@ -1657,7 +1657,7 @@ public class TestHRegionReplayEvents {
FSDataOutputStream out = TEST_UTIL.getTestFileSystem().create(testFile);
try {
hFileFactory.withOutputStream(out);
hFileFactory.withFileContext(new HFileContext());
hFileFactory.withFileContext(new HFileContextBuilder().build());
HFile.Writer writer = hFileFactory.create();
try {
writer.append(new KeyValue(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)

3
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java
View File

@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.tool.BulkLoadHFiles;
@ -161,7 +162,7 @@ public class TestScannerWithBulkload {
Path path = new Path(pathStr);
HFile.WriterFactory wf = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration());
Assert.assertNotNull(wf);
HFileContext context = new HFileContext();
HFileContext context = new HFileContextBuilder().build();
HFile.Writer writer = wf.withPath(fs, path).withFileContext(context).create();
KeyValue kv = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l,
Bytes.toBytes("version2"));

3
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java
View File

@ -35,7 +35,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@ -184,7 +183,7 @@ public class TestStoreScannerClosure {
Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile");
HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
StoreFileWriter sfw = new StoreFileWriter.Builder(CONF, fs).withOutputDir(storeFileParentDir)
.withComparator(CellComparatorImpl.COMPARATOR).withFileContext(meta).build();
.withFileContext(meta).build();
final int rowLen = 32;
Random RNG = new Random();