HBASE-23705 Add CellComparator to HFileContext (#1062)
Codecs don't have access to what CellComparator to use. Backfill. M hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java Adds a new compareRows with default implementation that takes a ByteBuffer. Needed by the index in a block encoder implementation. M hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java Adds implementation for meta of new compareRows method. Adds utility method for figuring comparator based off tablename. M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java Comparator is in context. Remove redundant handling. M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java Comparator is in context. Remove redundant handling. Clean javadoc. M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java Clean javadoc. M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java Cache context so can use it to get comparator to use later. M hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java Cache cellcomparator to use. Javdoc on diff between HFileContext and HFileInfo. M hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java Add CellComparator M hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java Remove comparator caching. Get from context instead. M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java Skip a reflection if we can. M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java Javadoc. Removed unused filed. Signed-off-by: Anoop Sam John <anoopsamjohn@apacher.org> Signed-off-by: Ramkrishna <ramkrishna.s.vasudevan@intel.com> Signed-off-by: Jan Hentschel <janh@apache.org>
This commit is contained in:
parent
a58f2a4a55
commit
6cdc4b1f05
|
@ -17,8 +17,9 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Comparator;
|
||||
|
||||
import org.apache.hadoop.hbase.util.ByteBufferUtils;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.apache.yetus.audience.InterfaceStability;
|
||||
|
||||
|
@ -31,11 +32,12 @@ import org.apache.yetus.audience.InterfaceStability;
|
|||
public interface CellComparator extends Comparator<Cell> {
|
||||
/**
|
||||
* A comparator for ordering cells in user-space tables. Useful when writing cells in sorted
|
||||
* order as necessary for bulk import (i.e. via MapReduce)
|
||||
* order as necessary for bulk import (i.e. via MapReduce).
|
||||
* <p>
|
||||
* CAUTION: This comparator may provide inaccurate ordering for cells from system tables,
|
||||
* and should not be relied upon in that case.
|
||||
*/
|
||||
// For internal use, see CellComparatorImpl utility methods.
|
||||
static CellComparator getInstance() {
|
||||
return CellComparatorImpl.COMPARATOR;
|
||||
}
|
||||
|
@ -80,6 +82,24 @@ public interface CellComparator extends Comparator<Cell> {
|
|||
*/
|
||||
int compareRows(Cell cell, byte[] bytes, int offset, int length);
|
||||
|
||||
/**
|
||||
* @param row ByteBuffer that wraps a row; will read from current position and will reading all
|
||||
* remaining; will not disturb the ByteBuffer internal state.
|
||||
* @return greater than 0 if leftCell is bigger, less than 0 if rightCell is bigger, 0 if both
|
||||
* cells are equal
|
||||
*/
|
||||
default int compareRows(ByteBuffer row, Cell cell) {
|
||||
if (cell instanceof ByteBufferExtendedCell) {
|
||||
return ByteBufferUtils.compareTo(row, row.position(), row.remaining(),
|
||||
((ByteBufferExtendedCell) cell).getRowByteBuffer(),
|
||||
((ByteBufferExtendedCell) cell).getRowPosition(),
|
||||
cell.getRowLength());
|
||||
}
|
||||
return ByteBufferUtils.compareTo(row, row.position(), row.remaining(),
|
||||
cell.getRowArray(), cell.getRowOffset(),
|
||||
cell.getRowLength());
|
||||
}
|
||||
|
||||
/**
|
||||
* Lexographically compares the two cells excluding the row part. It compares family, qualifier,
|
||||
* timestamp and the type
|
||||
|
|
|
@ -17,8 +17,8 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Comparator;
|
||||
|
||||
import org.apache.hadoop.hbase.KeyValue.Type;
|
||||
import org.apache.hadoop.hbase.util.ByteBufferUtils;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
@ -26,7 +26,6 @@ import org.apache.yetus.audience.InterfaceAudience;
|
|||
import org.apache.yetus.audience.InterfaceStability;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hbase.thirdparty.com.google.common.primitives.Longs;
|
||||
|
||||
/**
|
||||
|
@ -377,6 +376,26 @@ public class CellComparatorImpl implements CellComparator {
|
|||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareRows(ByteBuffer row, Cell cell) {
|
||||
byte [] array;
|
||||
int offset;
|
||||
int len = row.remaining();
|
||||
if (row.hasArray()) {
|
||||
array = row.array();
|
||||
offset = row.position() + row.arrayOffset();
|
||||
} else {
|
||||
// We copy the row array if offheap just so we can do a compare. We do this elsewhere too
|
||||
// in BBUtils when Cell is backed by an offheap ByteBuffer. Needs fixing so no copy. TODO.
|
||||
array = new byte[len];
|
||||
offset = 0;
|
||||
ByteBufferUtils.copyFromBufferToArray(array, row, row.position(),
|
||||
0, len);
|
||||
}
|
||||
// Reverse result since we swap the order of the params we pass below.
|
||||
return -compareRows(cell, array, offset, len);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator getSimpleComparator() {
|
||||
return this;
|
||||
|
@ -387,4 +406,24 @@ public class CellComparatorImpl implements CellComparator {
|
|||
public Comparator getSimpleComparator() {
|
||||
return new BBKVComparator(this);
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility method that makes a guess at comparator to use based off passed tableName.
|
||||
* Use in extreme when no comparator specified.
|
||||
* @return CellComparator to use going off the {@code tableName} passed.
|
||||
*/
|
||||
public static CellComparator getCellComparator(TableName tableName) {
|
||||
return getCellComparator(tableName.toBytes());
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility method that makes a guess at comparator to use based off passed tableName.
|
||||
* Use in extreme when no comparator specified.
|
||||
* @return CellComparator to use going off the {@code tableName} passed.
|
||||
*/
|
||||
public static CellComparator getCellComparator(byte [] tableName) {
|
||||
// FYI, TableName.toBytes does not create an array; just returns existing array pointer.
|
||||
return Bytes.equals(tableName, TableName.META_TABLE_NAME.toBytes())?
|
||||
CellComparatorImpl.META_COMPARATOR: CellComparatorImpl.COMPARATOR;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -292,6 +292,9 @@ public final class TableName implements Comparable<TableName> {
|
|||
return qualifierAsString;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return A pointer to TableName as String bytes.
|
||||
*/
|
||||
public byte[] toBytes() {
|
||||
return name;
|
||||
}
|
||||
|
|
|
@ -18,10 +18,8 @@ package org.apache.hadoop.hbase.io.encoding;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.hadoop.hbase.ByteBufferKeyOnlyKeyValue;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.io.hfile.BlockType;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileContext;
|
||||
|
@ -59,14 +57,13 @@ public abstract class AbstractDataBlockEncoder implements DataBlockEncoder {
|
|||
}
|
||||
}
|
||||
|
||||
protected abstract static class AbstractEncodedSeeker implements
|
||||
EncodedSeeker {
|
||||
/**
|
||||
* Decorates EncodedSeeker with a {@link HFileBlockDecodingContext}
|
||||
*/
|
||||
protected abstract static class AbstractEncodedSeeker implements EncodedSeeker {
|
||||
protected HFileBlockDecodingContext decodingCtx;
|
||||
protected final CellComparator comparator;
|
||||
|
||||
public AbstractEncodedSeeker(CellComparator comparator,
|
||||
HFileBlockDecodingContext decodingCtx) {
|
||||
this.comparator = comparator;
|
||||
public AbstractEncodedSeeker(HFileBlockDecodingContext decodingCtx) {
|
||||
this.decodingCtx = decodingCtx;
|
||||
}
|
||||
|
||||
|
@ -77,7 +74,5 @@ public abstract class AbstractDataBlockEncoder implements DataBlockEncoder {
|
|||
protected boolean includesTags() {
|
||||
return this.decodingCtx.getHFileContext().isIncludesTags();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -732,9 +732,8 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder {
|
|||
protected final ObjectIntPair<ByteBuffer> tmpPair = new ObjectIntPair<>();
|
||||
protected STATE current, previous;
|
||||
|
||||
public BufferedEncodedSeeker(CellComparator comparator,
|
||||
HFileBlockDecodingContext decodingCtx) {
|
||||
super(comparator, decodingCtx);
|
||||
public BufferedEncodedSeeker(HFileBlockDecodingContext decodingCtx) {
|
||||
super(decodingCtx);
|
||||
if (decodingCtx.getHFileContext().isCompressTags()) {
|
||||
try {
|
||||
tagCompressionContext = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE);
|
||||
|
@ -1008,11 +1007,7 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder {
|
|||
}
|
||||
|
||||
/**
|
||||
* @param cell
|
||||
* @param out
|
||||
* @param encodingCtx
|
||||
* @return unencoded size added
|
||||
* @throws IOException
|
||||
*/
|
||||
protected final int afterEncodingKeyValue(Cell cell, DataOutputStream out,
|
||||
HFileBlockDefaultEncodingContext encodingCtx) throws IOException {
|
||||
|
@ -1102,7 +1097,7 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder {
|
|||
public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out)
|
||||
throws IOException {
|
||||
if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
|
||||
throw new IOException (this.getClass().getName() + " only accepts "
|
||||
throw new IOException(this.getClass().getName() + " only accepts "
|
||||
+ HFileBlockDefaultEncodingContext.class.getName() + " as the " +
|
||||
"encoding context.");
|
||||
}
|
||||
|
@ -1154,8 +1149,8 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder {
|
|||
.getEncodingState();
|
||||
// Write the unencodedDataSizeWritten (with header size)
|
||||
Bytes.putInt(uncompressedBytesWithHeader,
|
||||
HConstants.HFILEBLOCK_HEADER_SIZE + DataBlockEncoding.ID_SIZE, state.unencodedDataSizeWritten
|
||||
);
|
||||
HConstants.HFILEBLOCK_HEADER_SIZE + DataBlockEncoding.ID_SIZE,
|
||||
state.unencodedDataSizeWritten);
|
||||
postEncoding(encodingCtx);
|
||||
}
|
||||
|
||||
|
|
|
@ -20,9 +20,7 @@ import java.io.DataInputStream;
|
|||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.nio.ByteBuff;
|
||||
import org.apache.hadoop.hbase.util.ByteBufferUtils;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
@ -48,7 +46,8 @@ public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder {
|
|||
+ "encoding context.");
|
||||
}
|
||||
|
||||
HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
|
||||
HFileBlockDefaultEncodingContext encodingCtx =
|
||||
(HFileBlockDefaultEncodingContext) blkEncodingCtx;
|
||||
encodingCtx.prepareEncoding(out);
|
||||
|
||||
NoneEncoder encoder = new NoneEncoder(out, encodingCtx);
|
||||
|
@ -81,9 +80,8 @@ public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder {
|
|||
}
|
||||
|
||||
@Override
|
||||
public EncodedSeeker createSeeker(CellComparator comparator,
|
||||
final HFileBlockDecodingContext decodingCtx) {
|
||||
return new BufferedEncodedSeeker<SeekerState>(comparator, decodingCtx) {
|
||||
public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) {
|
||||
return new BufferedEncodedSeeker<SeekerState>(decodingCtx) {
|
||||
@Override
|
||||
protected void decodeNext() {
|
||||
current.keyLength = currentBuffer.getInt();
|
||||
|
|
|
@ -20,7 +20,6 @@ import java.io.DataInputStream;
|
|||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileContext;
|
||||
|
@ -45,20 +44,13 @@ public interface DataBlockEncoder {
|
|||
* Starts encoding for a block of KeyValues. Call
|
||||
* {@link #endBlockEncoding(HFileBlockEncodingContext, DataOutputStream, byte[])} to finish
|
||||
* encoding of a block.
|
||||
* @param encodingCtx
|
||||
* @param out
|
||||
* @throws IOException
|
||||
*/
|
||||
void startBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out)
|
||||
throws IOException;
|
||||
|
||||
/**
|
||||
* Encodes a KeyValue.
|
||||
* @param cell
|
||||
* @param encodingCtx
|
||||
* @param out
|
||||
* @return unencoded kv size written
|
||||
* @throws IOException
|
||||
*/
|
||||
int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out)
|
||||
throws IOException;
|
||||
|
@ -66,10 +58,6 @@ public interface DataBlockEncoder {
|
|||
/**
|
||||
* Ends encoding for a block of KeyValues. Gives a chance for the encoder to do the finishing
|
||||
* stuff for the encoded block. It must be called at the end of block encoding.
|
||||
* @param encodingCtx
|
||||
* @param out
|
||||
* @param uncompressedBytesWithHeader
|
||||
* @throws IOException
|
||||
*/
|
||||
void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out,
|
||||
byte[] uncompressedBytesWithHeader) throws IOException;
|
||||
|
@ -77,7 +65,6 @@ public interface DataBlockEncoder {
|
|||
/**
|
||||
* Decode.
|
||||
* @param source Compressed stream of KeyValues.
|
||||
* @param decodingCtx
|
||||
* @return Uncompressed block of KeyValues.
|
||||
* @throws IOException If there is an error in source.
|
||||
*/
|
||||
|
@ -96,11 +83,9 @@ public interface DataBlockEncoder {
|
|||
|
||||
/**
|
||||
* Create a HFileBlock seeker which find KeyValues within a block.
|
||||
* @param comparator what kind of comparison should be used
|
||||
* @param decodingCtx
|
||||
* @return A newly created seeker.
|
||||
*/
|
||||
EncodedSeeker createSeeker(CellComparator comparator, HFileBlockDecodingContext decodingCtx);
|
||||
EncodedSeeker createSeeker(HFileBlockDecodingContext decodingCtx);
|
||||
|
||||
/**
|
||||
* Creates a encoder specific encoding context
|
||||
|
@ -188,8 +173,6 @@ public interface DataBlockEncoder {
|
|||
|
||||
/**
|
||||
* Compare the given key against the current key
|
||||
* @param comparator
|
||||
* @param key
|
||||
* @return -1 is the passed key is smaller than the current key, 0 if equal and 1 if greater
|
||||
*/
|
||||
public int compareKey(CellComparator comparator, Cell key);
|
||||
|
|
|
@ -20,9 +20,7 @@ import java.io.DataInputStream;
|
|||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.KeyValueUtil;
|
||||
import org.apache.hadoop.hbase.PrivateCellUtil;
|
||||
|
@ -382,9 +380,8 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
|
|||
}
|
||||
|
||||
@Override
|
||||
public EncodedSeeker createSeeker(CellComparator comparator,
|
||||
HFileBlockDecodingContext decodingCtx) {
|
||||
return new BufferedEncodedSeeker<DiffSeekerState>(comparator, decodingCtx) {
|
||||
public EncodedSeeker createSeeker(HFileBlockDecodingContext decodingCtx) {
|
||||
return new BufferedEncodedSeeker<DiffSeekerState>(decodingCtx) {
|
||||
private byte[] familyNameWithSize;
|
||||
private static final int TIMESTAMP_WITH_TYPE_LENGTH =
|
||||
Bytes.SIZEOF_LONG + Bytes.SIZEOF_BYTE;
|
||||
|
|
|
@ -21,9 +21,7 @@ import java.io.DataOutput;
|
|||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.KeyValueUtil;
|
||||
import org.apache.hadoop.hbase.PrivateCellUtil;
|
||||
|
@ -397,9 +395,8 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
|
|||
}
|
||||
|
||||
@Override
|
||||
public EncodedSeeker createSeeker(CellComparator comparator,
|
||||
final HFileBlockDecodingContext decodingCtx) {
|
||||
return new BufferedEncodedSeeker<FastDiffSeekerState>(comparator, decodingCtx) {
|
||||
public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) {
|
||||
return new BufferedEncodedSeeker<FastDiffSeekerState>(decodingCtx) {
|
||||
private void decode(boolean isFirst) {
|
||||
byte flag = currentBuffer.get();
|
||||
if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
|
||||
|
|
|
@ -17,20 +17,18 @@
|
|||
package org.apache.hadoop.hbase.io.encoding;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileContext;
|
||||
import org.apache.hadoop.hbase.nio.ByteBuff;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
|
||||
/**
|
||||
* A decoding context that is created by a reader's encoder, and is shared
|
||||
* across the reader's all read operations.
|
||||
* across all of the reader's read operations.
|
||||
*
|
||||
* @see HFileBlockEncodingContext for encoding
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public interface HFileBlockDecodingContext {
|
||||
|
||||
/**
|
||||
* Perform all actions that need to be done before the encoder's real decoding
|
||||
* process. Decompression needs to be done if
|
||||
|
@ -46,7 +44,6 @@ public interface HFileBlockDecodingContext {
|
|||
* ByteBuffer pointed after the header but before the data
|
||||
* @param onDiskBlock
|
||||
* on disk data to be decoded
|
||||
* @throws IOException
|
||||
*/
|
||||
void prepareDecoding(
|
||||
int onDiskSizeWithoutHeader,
|
||||
|
|
|
@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.io.encoding;
|
|||
import java.io.DataInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.hadoop.hbase.io.ByteBuffInputStream;
|
||||
import org.apache.hadoop.hbase.io.TagCompressionContext;
|
||||
|
@ -41,11 +40,10 @@ import org.apache.yetus.audience.InterfaceAudience;
|
|||
*
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class HFileBlockDefaultDecodingContext implements
|
||||
HFileBlockDecodingContext {
|
||||
public class HFileBlockDefaultDecodingContext implements HFileBlockDecodingContext {
|
||||
private final HFileContext fileContext;
|
||||
private TagCompressionContext tagCompressionContext;
|
||||
|
||||
|
||||
public HFileBlockDefaultDecodingContext(HFileContext fileContext) {
|
||||
this.fileContext = fileContext;
|
||||
}
|
||||
|
|
|
@ -17,13 +17,11 @@
|
|||
package org.apache.hadoop.hbase.io.encoding;
|
||||
|
||||
import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.NONE;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.security.SecureRandom;
|
||||
|
||||
import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
|
||||
import org.apache.hadoop.hbase.io.TagCompressionContext;
|
||||
import org.apache.hadoop.hbase.io.compress.Compression;
|
||||
|
@ -36,7 +34,6 @@ import org.apache.hadoop.hbase.util.Bytes;
|
|||
import org.apache.hadoop.io.compress.CompressionOutputStream;
|
||||
import org.apache.hadoop.io.compress.Compressor;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
|
||||
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
|
||||
|
||||
/**
|
||||
|
@ -47,8 +44,7 @@ import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
|
|||
*
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class HFileBlockDefaultEncodingContext implements
|
||||
HFileBlockEncodingContext {
|
||||
public class HFileBlockDefaultEncodingContext implements HFileBlockEncodingContext {
|
||||
private BlockType blockType;
|
||||
private final DataBlockEncoding encodingAlgo;
|
||||
|
||||
|
@ -113,7 +109,6 @@ public class HFileBlockDefaultEncodingContext implements
|
|||
|
||||
/**
|
||||
* prepare to start a new encoding.
|
||||
* @throws IOException
|
||||
*/
|
||||
public void prepareEncoding(DataOutputStream out) throws IOException {
|
||||
if (encodingAlgo != null && encodingAlgo != DataBlockEncoding.NONE) {
|
||||
|
@ -133,7 +128,8 @@ public class HFileBlockDefaultEncodingContext implements
|
|||
}
|
||||
|
||||
private Bytes compressAfterEncoding(byte[] uncompressedBytesWithHeaderBuffer,
|
||||
int uncompressedBytesWithHeaderOffset, int uncompressedBytesWithHeaderLength, byte[] headerBytes)
|
||||
int uncompressedBytesWithHeaderOffset, int uncompressedBytesWithHeaderLength,
|
||||
byte[] headerBytes)
|
||||
throws IOException {
|
||||
Encryption.Context cryptoContext = fileContext.getEncryptionContext();
|
||||
if (cryptoContext != Encryption.Context.NONE) {
|
||||
|
@ -158,7 +154,8 @@ public class HFileBlockDefaultEncodingContext implements
|
|||
compressedByteStream.reset();
|
||||
compressionStream.resetState();
|
||||
compressionStream.write(uncompressedBytesWithHeaderBuffer,
|
||||
headerBytes.length + uncompressedBytesWithHeaderOffset, uncompressedBytesWithHeaderLength - headerBytes.length);
|
||||
headerBytes.length + uncompressedBytesWithHeaderOffset,
|
||||
uncompressedBytesWithHeaderLength - headerBytes.length);
|
||||
compressionStream.flush();
|
||||
compressionStream.finish();
|
||||
byte[] plaintext = compressedByteStream.toByteArray();
|
||||
|
|
|
@ -17,7 +17,6 @@
|
|||
package org.apache.hadoop.hbase.io.encoding;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.hadoop.hbase.io.hfile.BlockType;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileContext;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
@ -46,9 +45,6 @@ public interface HFileBlockEncodingContext {
|
|||
/**
|
||||
* Do any action that needs to be performed after the encoding.
|
||||
* Compression is also included if a non-null compression algorithm is used
|
||||
*
|
||||
* @param blockType
|
||||
* @throws IOException
|
||||
*/
|
||||
void postEncoding(BlockType blockType) throws IOException;
|
||||
|
||||
|
@ -64,7 +60,6 @@ public interface HFileBlockEncodingContext {
|
|||
|
||||
/**
|
||||
* Sets the encoding state.
|
||||
* @param state
|
||||
*/
|
||||
void setEncodingState(EncodingState state);
|
||||
|
||||
|
|
|
@ -21,9 +21,7 @@ import java.io.DataOutput;
|
|||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.KeyValueUtil;
|
||||
import org.apache.hadoop.hbase.PrivateCellUtil;
|
||||
|
@ -195,9 +193,8 @@ public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder {
|
|||
}
|
||||
|
||||
@Override
|
||||
public EncodedSeeker createSeeker(CellComparator comparator,
|
||||
final HFileBlockDecodingContext decodingCtx) {
|
||||
return new BufferedEncodedSeeker<SeekerState>(comparator, decodingCtx) {
|
||||
public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) {
|
||||
return new BufferedEncodedSeeker<SeekerState>(decodingCtx) {
|
||||
@Override
|
||||
protected void decodeNext() {
|
||||
current.keyLength = ByteBuff.readCompressedInt(currentBuffer);
|
||||
|
|
|
@ -22,10 +22,7 @@ import java.io.IOException;
|
|||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.KeyValueUtil;
|
||||
import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
|
||||
|
@ -71,7 +68,8 @@ public class RowIndexCodecV1 extends AbstractDataBlockEncoder {
|
|||
+ "encoding context.");
|
||||
}
|
||||
|
||||
HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
|
||||
HFileBlockDefaultEncodingContext encodingCtx =
|
||||
(HFileBlockDefaultEncodingContext) blkEncodingCtx;
|
||||
encodingCtx.prepareEncoding(out);
|
||||
|
||||
RowIndexEncoderV1 encoder = new RowIndexEncoderV1(out, encodingCtx);
|
||||
|
@ -115,8 +113,7 @@ public class RowIndexCodecV1 extends AbstractDataBlockEncoder {
|
|||
dup.limit(sourceAsBuffer.position() + onDiskSize);
|
||||
return dup.slice();
|
||||
} else {
|
||||
RowIndexSeekerV1 seeker = new RowIndexSeekerV1(CellComparatorImpl.COMPARATOR,
|
||||
decodingCtx);
|
||||
RowIndexSeekerV1 seeker = new RowIndexSeekerV1(decodingCtx);
|
||||
seeker.setCurrentBuffer(new SingleByteBuff(sourceAsBuffer));
|
||||
List<Cell> kvs = new ArrayList<>();
|
||||
kvs.add(seeker.getCell());
|
||||
|
@ -151,9 +148,7 @@ public class RowIndexCodecV1 extends AbstractDataBlockEncoder {
|
|||
}
|
||||
|
||||
@Override
|
||||
public EncodedSeeker createSeeker(CellComparator comparator,
|
||||
HFileBlockDecodingContext decodingCtx) {
|
||||
return new RowIndexSeekerV1(comparator, decodingCtx);
|
||||
public EncodedSeeker createSeeker(HFileBlockDecodingContext decodingCtx) {
|
||||
return new RowIndexSeekerV1(decodingCtx);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -12,9 +12,7 @@ package org.apache.hadoop.hbase.io.encoding;
|
|||
|
||||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.KeyValueUtil;
|
||||
import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
|
@ -32,10 +30,12 @@ public class RowIndexEncoderV1 {
|
|||
private NoneEncoder encoder;
|
||||
private int startOffset = -1;
|
||||
private ByteArrayOutputStream rowsOffsetBAOS = new ByteArrayOutputStream(64 * 4);
|
||||
private final HFileBlockEncodingContext context;
|
||||
|
||||
public RowIndexEncoderV1(DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx) {
|
||||
this.out = out;
|
||||
this.encoder = new NoneEncoder(out, encodingCtx);
|
||||
this.context = encodingCtx;
|
||||
}
|
||||
|
||||
public int write(Cell cell) throws IOException {
|
||||
|
@ -56,7 +56,7 @@ public class RowIndexEncoderV1 {
|
|||
throw new IOException("Key cannot be null or empty");
|
||||
}
|
||||
if (lastCell != null) {
|
||||
int keyComp = CellComparatorImpl.COMPARATOR.compareRows(lastCell, cell);
|
||||
int keyComp = this.context.getHFileContext().getCellComparator().compareRows(lastCell, cell);
|
||||
if (keyComp > 0) {
|
||||
throw new IOException("Added a key not lexically larger than"
|
||||
+ " previous. Current cell = " + cell + ", lastCell = " + lastCell);
|
||||
|
|
|
@ -17,8 +17,6 @@
|
|||
package org.apache.hadoop.hbase.io.encoding;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.hadoop.hbase.ByteBufferExtendedCell;
|
||||
import org.apache.hadoop.hbase.ByteBufferKeyOnlyKeyValue;
|
||||
import org.apache.hadoop.hbase.ByteBufferKeyValue;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
|
@ -50,10 +48,11 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker {
|
|||
|
||||
private int rowNumber;
|
||||
private ByteBuff rowOffsets = null;
|
||||
private final CellComparator cellComparator;
|
||||
|
||||
public RowIndexSeekerV1(CellComparator comparator,
|
||||
HFileBlockDecodingContext decodingCtx) {
|
||||
super(comparator, decodingCtx);
|
||||
public RowIndexSeekerV1(HFileBlockDecodingContext decodingCtx) {
|
||||
super(decodingCtx);
|
||||
this.cellComparator = decodingCtx.getHFileContext().getCellComparator();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -131,8 +130,7 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker {
|
|||
int comp = 0;
|
||||
while (low <= high) {
|
||||
mid = low + ((high - low) >> 1);
|
||||
ByteBuffer row = getRow(mid);
|
||||
comp = compareRows(row, seekCell);
|
||||
comp = this.cellComparator.compareRows(getRow(mid), seekCell);
|
||||
if (comp < 0) {
|
||||
low = mid + 1;
|
||||
} else if (comp > 0) {
|
||||
|
@ -154,19 +152,6 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker {
|
|||
}
|
||||
}
|
||||
|
||||
private int compareRows(ByteBuffer row, Cell seekCell) {
|
||||
if (seekCell instanceof ByteBufferExtendedCell) {
|
||||
return ByteBufferUtils.compareTo(row, row.position(), row.remaining(),
|
||||
((ByteBufferExtendedCell) seekCell).getRowByteBuffer(),
|
||||
((ByteBufferExtendedCell) seekCell).getRowPosition(),
|
||||
seekCell.getRowLength());
|
||||
} else {
|
||||
return ByteBufferUtils.compareTo(row, row.position(), row.remaining(),
|
||||
seekCell.getRowArray(), seekCell.getRowOffset(),
|
||||
seekCell.getRowLength());
|
||||
}
|
||||
}
|
||||
|
||||
private ByteBuffer getRow(int index) {
|
||||
int offset = rowOffsets.getIntAfterPosition(index * Bytes.SIZEOF_INT);
|
||||
ByteBuff block = currentBuffer.duplicate();
|
||||
|
@ -191,8 +176,8 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker {
|
|||
}
|
||||
}
|
||||
do {
|
||||
int comp;
|
||||
comp = PrivateCellUtil.compareKeyIgnoresMvcc(comparator, seekCell, current.currentKey);
|
||||
int comp =
|
||||
PrivateCellUtil.compareKeyIgnoresMvcc(this.cellComparator, seekCell, current.currentKey);
|
||||
if (comp == 0) { // exact match
|
||||
if (seekBefore) {
|
||||
if (!previous.isValid()) {
|
||||
|
@ -400,5 +385,4 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker {
|
|||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -17,6 +17,8 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase.io.hfile;
|
||||
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.io.HeapSize;
|
||||
import org.apache.hadoop.hbase.io.compress.Compression;
|
||||
|
@ -28,9 +30,10 @@ import org.apache.hadoop.hbase.util.ClassSize;
|
|||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
|
||||
/**
|
||||
* This carries the information on some of the meta data about the HFile. This
|
||||
* meta data is used across the HFileWriter/Readers and the HFileBlocks.
|
||||
* This helps to add new information to the HFile.
|
||||
* Read-only HFile Context Information. Meta data that is used by HFileWriter/Readers and by
|
||||
* HFileBlocks. Create one using the {@link HFileContextBuilder} (See HFileInfo and the HFile
|
||||
* Trailer class).
|
||||
* @see HFileContextBuilder
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class HFileContext implements HeapSize, Cloneable {
|
||||
|
@ -42,7 +45,7 @@ public class HFileContext implements HeapSize, Cloneable {
|
|||
//byte[] headers for column family and table name
|
||||
2 * ClassSize.ARRAY + 2 * ClassSize.REFERENCE);
|
||||
|
||||
public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
|
||||
private static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
|
||||
|
||||
/** Whether checksum is enabled or not**/
|
||||
private boolean usesHBaseChecksum = true;
|
||||
|
@ -67,6 +70,7 @@ public class HFileContext implements HeapSize, Cloneable {
|
|||
private String hfileName;
|
||||
private byte[] columnFamily;
|
||||
private byte[] tableName;
|
||||
private CellComparator cellComparator;
|
||||
|
||||
//Empty constructor. Go with setters
|
||||
public HFileContext() {
|
||||
|
@ -74,7 +78,6 @@ public class HFileContext implements HeapSize, Cloneable {
|
|||
|
||||
/**
|
||||
* Copy constructor
|
||||
* @param context
|
||||
*/
|
||||
public HFileContext(HFileContext context) {
|
||||
this.usesHBaseChecksum = context.usesHBaseChecksum;
|
||||
|
@ -91,13 +94,14 @@ public class HFileContext implements HeapSize, Cloneable {
|
|||
this.hfileName = context.hfileName;
|
||||
this.columnFamily = context.columnFamily;
|
||||
this.tableName = context.tableName;
|
||||
this.cellComparator = context.cellComparator;
|
||||
}
|
||||
|
||||
HFileContext(boolean useHBaseChecksum, boolean includesMvcc, boolean includesTags,
|
||||
Compression.Algorithm compressAlgo, boolean compressTags, ChecksumType checksumType,
|
||||
int bytesPerChecksum, int blockSize, DataBlockEncoding encoding,
|
||||
Encryption.Context cryptoContext, long fileCreateTime, String hfileName,
|
||||
byte[] columnFamily, byte[] tableName) {
|
||||
byte[] columnFamily, byte[] tableName, CellComparator cellComparator) {
|
||||
this.usesHBaseChecksum = useHBaseChecksum;
|
||||
this.includesMvcc = includesMvcc;
|
||||
this.includesTags = includesTags;
|
||||
|
@ -114,11 +118,14 @@ public class HFileContext implements HeapSize, Cloneable {
|
|||
this.hfileName = hfileName;
|
||||
this.columnFamily = columnFamily;
|
||||
this.tableName = tableName;
|
||||
// If no cellComparator specified, make a guess based off tablename. If hbase:meta, then should
|
||||
// be the meta table comparator. Comparators are per table.
|
||||
this.cellComparator = cellComparator != null ? cellComparator : this.tableName != null ?
|
||||
CellComparatorImpl.getCellComparator(this.tableName) : CellComparator.getInstance();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true when on-disk blocks from this file are compressed, and/or encrypted;
|
||||
* false otherwise.
|
||||
* @return true when on-disk blocks are compressed, and/or encrypted; false otherwise.
|
||||
*/
|
||||
public boolean isCompressedOrEncrypted() {
|
||||
Compression.Algorithm compressAlgo = getCompression();
|
||||
|
@ -208,6 +215,11 @@ public class HFileContext implements HeapSize, Cloneable {
|
|||
public byte[] getTableName() {
|
||||
return this.tableName;
|
||||
}
|
||||
|
||||
public CellComparator getCellComparator() {
|
||||
return this.cellComparator;
|
||||
}
|
||||
|
||||
/**
|
||||
* HeapSize implementation. NOTE : The heap size should be altered when new state variable are
|
||||
* added.
|
||||
|
@ -263,8 +275,9 @@ public class HFileContext implements HeapSize, Cloneable {
|
|||
sb.append(", columnFamily=");
|
||||
sb.append(Bytes.toStringBinary(columnFamily));
|
||||
}
|
||||
sb.append(", cellComparator=");
|
||||
sb.append(this.cellComparator);
|
||||
sb.append("]");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase.io.hfile;
|
||||
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
|
||||
import org.apache.hadoop.hbase.io.crypto.Encryption;
|
||||
|
@ -56,6 +57,7 @@ public class HFileContextBuilder {
|
|||
private String hfileName = null;
|
||||
private byte[] columnFamily = null;
|
||||
private byte[] tableName = null;
|
||||
private CellComparator cellComparator;
|
||||
|
||||
public HFileContextBuilder() {}
|
||||
|
||||
|
@ -77,6 +79,7 @@ public class HFileContextBuilder {
|
|||
this.hfileName = hfc.getHFileName();
|
||||
this.columnFamily = hfc.getColumnFamily();
|
||||
this.tableName = hfc.getTableName();
|
||||
this.cellComparator = hfc.getCellComparator();
|
||||
}
|
||||
|
||||
public HFileContextBuilder withHBaseCheckSum(boolean useHBaseCheckSum) {
|
||||
|
@ -149,9 +152,14 @@ public class HFileContextBuilder {
|
|||
return this;
|
||||
}
|
||||
|
||||
public HFileContextBuilder withCellComparator(CellComparator cellComparator) {
|
||||
this.cellComparator = cellComparator;
|
||||
return this;
|
||||
}
|
||||
|
||||
public HFileContext build() {
|
||||
return new HFileContext(usesHBaseChecksum, includesMvcc, includesTags, compression,
|
||||
compressTags, checksumType, bytesPerChecksum, blocksize, encoding, cryptoContext,
|
||||
fileCreateTime, hfileName, columnFamily, tableName);
|
||||
fileCreateTime, hfileName, columnFamily, tableName, cellComparator);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
/**
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
|
@ -21,7 +21,6 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TASK_KEY;
|
|||
import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TIME_KEY;
|
||||
import static org.apache.hadoop.hbase.regionserver.HStoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY;
|
||||
import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.InetSocketAddress;
|
||||
|
@ -38,13 +37,11 @@ import java.util.TreeSet;
|
|||
import java.util.UUID;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.CellUtil;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.HRegionLocation;
|
||||
|
@ -91,7 +88,6 @@ import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner;
|
|||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
/**
|
||||
|
@ -179,18 +175,17 @@ public class HFileOutputFormat2
|
|||
return combineTableNameSuffix(tableName, family);
|
||||
}
|
||||
|
||||
static <V extends Cell> RecordWriter<ImmutableBytesWritable, V>
|
||||
createRecordWriter(final TaskAttemptContext context, final OutputCommitter committer)
|
||||
throws IOException {
|
||||
static <V extends Cell> RecordWriter<ImmutableBytesWritable, V> createRecordWriter(
|
||||
final TaskAttemptContext context, final OutputCommitter committer) throws IOException {
|
||||
|
||||
// Get the path of the temporary output file
|
||||
final Path outputDir = ((FileOutputCommitter)committer).getWorkPath();
|
||||
final Configuration conf = context.getConfiguration();
|
||||
final boolean writeMultipleTables = conf.getBoolean(MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY, false) ;
|
||||
final boolean writeMultipleTables =
|
||||
conf.getBoolean(MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY, false);
|
||||
final String writeTableNames = conf.get(OUTPUT_TABLE_NAME_CONF_KEY);
|
||||
if (writeTableNames==null || writeTableNames.isEmpty()) {
|
||||
throw new IllegalArgumentException("Configuration parameter " + OUTPUT_TABLE_NAME_CONF_KEY
|
||||
+ " cannot be empty");
|
||||
if (writeTableNames == null || writeTableNames.isEmpty()) {
|
||||
throw new IllegalArgumentException("" + OUTPUT_TABLE_NAME_CONF_KEY + " cannot be empty");
|
||||
}
|
||||
final FileSystem fs = outputDir.getFileSystem(conf);
|
||||
// These configs. are from hbase-*.xml
|
||||
|
@ -199,19 +194,12 @@ public class HFileOutputFormat2
|
|||
// Invented config. Add to hbase-*.xml if other than default compression.
|
||||
final String defaultCompressionStr = conf.get("hfile.compression",
|
||||
Compression.Algorithm.NONE.getName());
|
||||
final Algorithm defaultCompression = HFileWriterImpl
|
||||
.compressionByName(defaultCompressionStr);
|
||||
final Algorithm defaultCompression = HFileWriterImpl.compressionByName(defaultCompressionStr);
|
||||
String compressionStr = conf.get(COMPRESSION_OVERRIDE_CONF_KEY);
|
||||
final Algorithm overriddenCompression;
|
||||
if (compressionStr != null) {
|
||||
overriddenCompression = Compression.getCompressionAlgorithmByName(compressionStr);
|
||||
} else {
|
||||
overriddenCompression = null;
|
||||
}
|
||||
|
||||
final Algorithm overriddenCompression = compressionStr != null ?
|
||||
Compression.getCompressionAlgorithmByName(compressionStr): null;
|
||||
final boolean compactionExclude = conf.getBoolean(
|
||||
"hbase.mapreduce.hfileoutputformat.compaction.exclude", false);
|
||||
|
||||
final Set<String> allTableNames = Arrays.stream(writeTableNames.split(
|
||||
Bytes.toString(tableSeparator))).collect(Collectors.toSet());
|
||||
|
||||
|
@ -224,24 +212,17 @@ public class HFileOutputFormat2
|
|||
String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_OVERRIDE_CONF_KEY);
|
||||
final Map<byte[], DataBlockEncoding> datablockEncodingMap
|
||||
= createFamilyDataBlockEncodingMap(conf);
|
||||
final DataBlockEncoding overriddenEncoding;
|
||||
if (dataBlockEncodingStr != null) {
|
||||
overriddenEncoding = DataBlockEncoding.valueOf(dataBlockEncodingStr);
|
||||
} else {
|
||||
overriddenEncoding = null;
|
||||
}
|
||||
final DataBlockEncoding overriddenEncoding = dataBlockEncodingStr != null ?
|
||||
DataBlockEncoding.valueOf(dataBlockEncodingStr) : null;
|
||||
|
||||
return new RecordWriter<ImmutableBytesWritable, V>() {
|
||||
// Map of families to writers and how much has been output on the writer.
|
||||
private final Map<byte[], WriterLength> writers =
|
||||
new TreeMap<>(Bytes.BYTES_COMPARATOR);
|
||||
private final Map<byte[], byte[]> previousRows =
|
||||
new TreeMap<>(Bytes.BYTES_COMPARATOR);
|
||||
private final Map<byte[], WriterLength> writers = new TreeMap<>(Bytes.BYTES_COMPARATOR);
|
||||
private final Map<byte[], byte[]> previousRows = new TreeMap<>(Bytes.BYTES_COMPARATOR);
|
||||
private final long now = EnvironmentEdgeManager.currentTime();
|
||||
|
||||
@Override
|
||||
public void write(ImmutableBytesWritable row, V cell)
|
||||
throws IOException {
|
||||
public void write(ImmutableBytesWritable row, V cell) throws IOException {
|
||||
Cell kv = cell;
|
||||
// null input == user explicitly wants to flush
|
||||
if (row == null && kv == null) {
|
||||
|
@ -255,12 +236,11 @@ public class HFileOutputFormat2
|
|||
byte[] tableNameBytes = null;
|
||||
if (writeMultipleTables) {
|
||||
tableNameBytes = MultiTableHFileOutputFormat.getTableName(row.get());
|
||||
tableNameBytes =
|
||||
TableName.valueOf(tableNameBytes).getNameWithNamespaceInclAsString()
|
||||
tableNameBytes = TableName.valueOf(tableNameBytes).getNameWithNamespaceInclAsString()
|
||||
.getBytes(Charset.defaultCharset());
|
||||
if (!allTableNames.contains(Bytes.toString(tableNameBytes))) {
|
||||
throw new IllegalArgumentException("TableName '" + Bytes.toString(tableNameBytes) +
|
||||
"' not" + " expected");
|
||||
throw new IllegalArgumentException("TableName " + Bytes.toString(tableNameBytes) +
|
||||
" not expected");
|
||||
}
|
||||
} else {
|
||||
tableNameBytes = Bytes.toBytes(writeTableNames);
|
||||
|
@ -275,9 +255,7 @@ public class HFileOutputFormat2
|
|||
if (wl == null) {
|
||||
Path writerPath = null;
|
||||
if (writeMultipleTables) {
|
||||
writerPath = new Path(outputDir,new Path(tableRelPath, Bytes
|
||||
.toString(family)));
|
||||
|
||||
writerPath = new Path(outputDir,new Path(tableRelPath, Bytes.toString(family)));
|
||||
}
|
||||
else {
|
||||
writerPath = new Path(outputDir, Bytes.toString(family));
|
||||
|
@ -303,33 +281,23 @@ public class HFileOutputFormat2
|
|||
connection.getRegionLocator(TableName.valueOf(tableName))) {
|
||||
loc = locator.getRegionLocation(rowKey);
|
||||
} catch (Throwable e) {
|
||||
LOG.warn("There's something wrong when locating rowkey: " +
|
||||
Bytes.toString(rowKey) + " for tablename: " + tableName, e);
|
||||
LOG.warn("Something wrong locating rowkey {} in {}",
|
||||
Bytes.toString(rowKey), tableName, e);
|
||||
loc = null;
|
||||
} }
|
||||
|
||||
if (null == loc) {
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("failed to get region location, so use default writer for rowkey: " +
|
||||
Bytes.toString(rowKey));
|
||||
}
|
||||
LOG.trace("Failed get of location, use default writer {}", Bytes.toString(rowKey));
|
||||
wl = getNewWriter(tableNameBytes, family, conf, null);
|
||||
} else {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("first rowkey: [" + Bytes.toString(rowKey) + "]");
|
||||
}
|
||||
LOG.debug("First rowkey: [{}]", Bytes.toString(rowKey));
|
||||
InetSocketAddress initialIsa =
|
||||
new InetSocketAddress(loc.getHostname(), loc.getPort());
|
||||
if (initialIsa.isUnresolved()) {
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("failed to resolve bind address: " + loc.getHostname() + ":"
|
||||
+ loc.getPort() + ", so use default writer");
|
||||
}
|
||||
LOG.trace("Failed resolve address {}, use default writer", loc.getHostnamePort());
|
||||
wl = getNewWriter(tableNameBytes, family, conf, null);
|
||||
} else {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("use favored nodes writer: " + initialIsa.getHostString());
|
||||
}
|
||||
LOG.debug("Use favored nodes writer: {}", initialIsa.getHostString());
|
||||
wl = getNewWriter(tableNameBytes, family, conf, new InetSocketAddress[] { initialIsa
|
||||
});
|
||||
}
|
||||
|
@ -369,29 +337,27 @@ public class HFileOutputFormat2
|
|||
|
||||
private void closeWriter(WriterLength wl) throws IOException {
|
||||
if (wl.writer != null) {
|
||||
LOG.info(
|
||||
"Writer=" + wl.writer.getPath() + ((wl.written == 0)? "": ", wrote=" + wl.written));
|
||||
LOG.info("Writer=" + wl.writer.getPath() +
|
||||
((wl.written == 0)? "": ", wrote=" + wl.written));
|
||||
close(wl.writer);
|
||||
wl.writer = null;
|
||||
}
|
||||
wl.writer = null;
|
||||
wl.written = 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Create a new StoreFile.Writer.
|
||||
* @param family
|
||||
* @return A WriterLength, containing a new StoreFile.Writer.
|
||||
* @throws IOException
|
||||
*/
|
||||
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="BX_UNBOXING_IMMEDIATELY_REBOXED",
|
||||
justification="Not important")
|
||||
private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration
|
||||
conf, InetSocketAddress[] favoredNodes) throws IOException {
|
||||
private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration conf,
|
||||
InetSocketAddress[] favoredNodes) throws IOException {
|
||||
byte[] tableAndFamily = getTableNameSuffixedWithFamily(tableName, family);
|
||||
Path familydir = new Path(outputDir, Bytes.toString(family));
|
||||
if (writeMultipleTables) {
|
||||
familydir = new Path(outputDir,
|
||||
new Path(getTableRelativePath(tableName), Bytes.toString(family)));
|
||||
new Path(getTableRelativePath(tableName), Bytes.toString(family)));
|
||||
}
|
||||
WriterLength wl = new WriterLength();
|
||||
Algorithm compression = overriddenCompression;
|
||||
|
@ -409,12 +375,9 @@ public class HFileOutputFormat2
|
|||
encoding = encoding == null ? datablockEncodingMap.get(tableAndFamily) : encoding;
|
||||
encoding = encoding == null ? DataBlockEncoding.NONE : encoding;
|
||||
HFileContextBuilder contextBuilder = new HFileContextBuilder()
|
||||
.withCompression(compression)
|
||||
.withChecksumType(HStore.getChecksumType(conf))
|
||||
.withBytesPerCheckSum(HStore.getBytesPerChecksum(conf))
|
||||
.withBlockSize(blockSize)
|
||||
.withColumnFamily(family)
|
||||
.withTableName(tableName);
|
||||
.withCompression(compression).withChecksumType(HStore.getChecksumType(conf))
|
||||
.withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blockSize)
|
||||
.withColumnFamily(family).withTableName(tableName);
|
||||
|
||||
if (HFile.getFormatVersion(conf) >= HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
|
||||
contextBuilder.withIncludesTags(true);
|
||||
|
@ -423,16 +386,13 @@ public class HFileOutputFormat2
|
|||
contextBuilder.withDataBlockEncoding(encoding);
|
||||
HFileContext hFileContext = contextBuilder.build();
|
||||
if (null == favoredNodes) {
|
||||
wl.writer =
|
||||
new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, fs)
|
||||
.withOutputDir(familydir).withBloomType(bloomType)
|
||||
.withComparator(CellComparator.getInstance()).withFileContext(hFileContext).build();
|
||||
wl.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, fs)
|
||||
.withOutputDir(familydir).withBloomType(bloomType)
|
||||
.withFileContext(hFileContext).build();
|
||||
} else {
|
||||
wl.writer =
|
||||
new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, new HFileSystem(fs))
|
||||
.withOutputDir(familydir).withBloomType(bloomType)
|
||||
.withComparator(CellComparator.getInstance()).withFileContext(hFileContext)
|
||||
.withFavoredNodes(favoredNodes).build();
|
||||
wl.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, new HFileSystem(fs))
|
||||
.withOutputDir(familydir).withBloomType(bloomType)
|
||||
.withFileContext(hFileContext).withFavoredNodes(favoredNodes).build();
|
||||
}
|
||||
|
||||
this.writers.put(tableAndFamily, wl);
|
||||
|
@ -441,22 +401,17 @@ public class HFileOutputFormat2
|
|||
|
||||
private void close(final StoreFileWriter w) throws IOException {
|
||||
if (w != null) {
|
||||
w.appendFileInfo(BULKLOAD_TIME_KEY,
|
||||
Bytes.toBytes(System.currentTimeMillis()));
|
||||
w.appendFileInfo(BULKLOAD_TASK_KEY,
|
||||
Bytes.toBytes(context.getTaskAttemptID().toString()));
|
||||
w.appendFileInfo(MAJOR_COMPACTION_KEY,
|
||||
Bytes.toBytes(true));
|
||||
w.appendFileInfo(EXCLUDE_FROM_MINOR_COMPACTION_KEY,
|
||||
Bytes.toBytes(compactionExclude));
|
||||
w.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
|
||||
w.appendFileInfo(BULKLOAD_TASK_KEY, Bytes.toBytes(context.getTaskAttemptID().toString()));
|
||||
w.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(true));
|
||||
w.appendFileInfo(EXCLUDE_FROM_MINOR_COMPACTION_KEY, Bytes.toBytes(compactionExclude));
|
||||
w.appendTrackedTimestampsToMetadata();
|
||||
w.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close(TaskAttemptContext c)
|
||||
throws IOException, InterruptedException {
|
||||
public void close(TaskAttemptContext c) throws IOException, InterruptedException {
|
||||
for (WriterLength wl: this.writers.values()) {
|
||||
close(wl.writer);
|
||||
}
|
||||
|
@ -496,21 +451,18 @@ public class HFileOutputFormat2
|
|||
throws IOException {
|
||||
|
||||
ArrayList<ImmutableBytesWritable> ret = new ArrayList<>();
|
||||
for(RegionLocator regionLocator : regionLocators)
|
||||
{
|
||||
for(RegionLocator regionLocator : regionLocators) {
|
||||
TableName tableName = regionLocator.getName();
|
||||
LOG.info("Looking up current regions for table " + tableName);
|
||||
byte[][] byteKeys = regionLocator.getStartKeys();
|
||||
for (byte[] byteKey : byteKeys) {
|
||||
byte[] fullKey = byteKey; //HFileOutputFormat2 use case
|
||||
if (writeMultipleTables)
|
||||
{
|
||||
if (writeMultipleTables) {
|
||||
//MultiTableHFileOutputFormat use case
|
||||
fullKey = combineTableNameSuffix(tableName.getName(), byteKey);
|
||||
}
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("SplitPoint startkey for table [" + tableName + "]: [" + Bytes.toStringBinary
|
||||
(fullKey) + "]");
|
||||
LOG.debug("SplitPoint startkey for " + tableName + ": " + Bytes.toStringBinary(fullKey));
|
||||
}
|
||||
ret.add(new ImmutableBytesWritable(fullKey));
|
||||
}
|
||||
|
@ -537,8 +489,8 @@ public class HFileOutputFormat2
|
|||
TreeSet<ImmutableBytesWritable> sorted = new TreeSet<>(startKeys);
|
||||
ImmutableBytesWritable first = sorted.first();
|
||||
if (writeMultipleTables) {
|
||||
first = new ImmutableBytesWritable(MultiTableHFileOutputFormat.getSuffix(sorted.first
|
||||
().get()));
|
||||
first =
|
||||
new ImmutableBytesWritable(MultiTableHFileOutputFormat.getSuffix(sorted.first().get()));
|
||||
}
|
||||
if (!first.equals(HConstants.EMPTY_BYTE_ARRAY)) {
|
||||
throw new IllegalArgumentException(
|
||||
|
@ -641,11 +593,10 @@ public class HFileOutputFormat2
|
|||
|
||||
/* Now get the region start keys for every table required */
|
||||
List<String> allTableNames = new ArrayList<>(multiTableInfo.size());
|
||||
List<RegionLocator> regionLocators = new ArrayList<>( multiTableInfo.size());
|
||||
List<TableDescriptor> tableDescriptors = new ArrayList<>( multiTableInfo.size());
|
||||
List<RegionLocator> regionLocators = new ArrayList<>(multiTableInfo.size());
|
||||
List<TableDescriptor> tableDescriptors = new ArrayList<>(multiTableInfo.size());
|
||||
|
||||
for( TableInfo tableInfo : multiTableInfo )
|
||||
{
|
||||
for(TableInfo tableInfo : multiTableInfo) {
|
||||
regionLocators.add(tableInfo.getRegionLocator());
|
||||
String tn = writeMultipleTables?
|
||||
tableInfo.getRegionLocator().getName().getNameWithNamespaceInclAsString():
|
||||
|
@ -653,10 +604,12 @@ public class HFileOutputFormat2
|
|||
allTableNames.add(tn);
|
||||
tableDescriptors.add(tableInfo.getTableDescriptor());
|
||||
}
|
||||
// Record tablenames for creating writer by favored nodes, and decoding compression, block size and other attributes of columnfamily per table
|
||||
// Record tablenames for creating writer by favored nodes, and decoding compression,
|
||||
// block size and other attributes of columnfamily per table
|
||||
conf.set(OUTPUT_TABLE_NAME_CONF_KEY, StringUtils.join(allTableNames, Bytes
|
||||
.toString(tableSeparator)));
|
||||
List<ImmutableBytesWritable> startKeys = getRegionStartKeys(regionLocators, writeMultipleTables);
|
||||
List<ImmutableBytesWritable> startKeys =
|
||||
getRegionStartKeys(regionLocators, writeMultipleTables);
|
||||
// Use table's region boundaries for TOP split points.
|
||||
LOG.info("Configuring " + startKeys.size() + " reduce partitions " +
|
||||
"to match current region count for all tables");
|
||||
|
@ -852,9 +805,11 @@ public class HFileOutputFormat2
|
|||
TotalOrderPartitioner.setPartitionFile(conf, partitionsPath);
|
||||
}
|
||||
|
||||
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
|
||||
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value =
|
||||
"RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
|
||||
@VisibleForTesting
|
||||
static String serializeColumnFamilyAttribute(Function<ColumnFamilyDescriptor, String> fn, List<TableDescriptor> allTables)
|
||||
static String serializeColumnFamilyAttribute(Function<ColumnFamilyDescriptor, String> fn,
|
||||
List<TableDescriptor> allTables)
|
||||
throws UnsupportedEncodingException {
|
||||
StringBuilder attributeValue = new StringBuilder();
|
||||
int i = 0;
|
||||
|
@ -869,8 +824,8 @@ public class HFileOutputFormat2
|
|||
attributeValue.append('&');
|
||||
}
|
||||
attributeValue.append(URLEncoder.encode(
|
||||
Bytes.toString(combineTableNameSuffix(tableDescriptor.getTableName().getName(), familyDescriptor.getName())),
|
||||
"UTF-8"));
|
||||
Bytes.toString(combineTableNameSuffix(tableDescriptor.getTableName().getName(),
|
||||
familyDescriptor.getName())), "UTF-8"));
|
||||
attributeValue.append('=');
|
||||
attributeValue.append(URLEncoder.encode(fn.apply(familyDescriptor), "UTF-8"));
|
||||
}
|
||||
|
|
|
@ -26,27 +26,25 @@ import java.io.DataInputStream;
|
|||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.hadoop.fs.FSDataInputStream;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl.MetaCellComparator;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.apache.hadoop.hbase.io.compress.Compression;
|
||||
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
|
||||
|
||||
/**
|
||||
* The {@link HFile} has a fixed trailer which contains offsets to other
|
||||
* variable parts of the file. Also includes basic metadata on this file. The
|
||||
* trailer size is fixed within a given {@link HFile} format version only, but
|
||||
* we always store the version number as the last four-byte integer of the file.
|
||||
* The version number itself is split into two portions, a major
|
||||
* The version number itself is split into two portions, a major
|
||||
* version and a minor version. The last three bytes of a file are the major
|
||||
* version and a single preceding byte is the minor number. The major version
|
||||
* determines which readers/writers to use to read/write a hfile while a minor
|
||||
|
@ -77,16 +75,24 @@ public class FixedFileTrailer {
|
|||
*/
|
||||
private long loadOnOpenDataOffset;
|
||||
|
||||
/** The number of entries in the root data index. */
|
||||
/**
|
||||
* The number of entries in the root data index.
|
||||
*/
|
||||
private int dataIndexCount;
|
||||
|
||||
/** Total uncompressed size of all blocks of the data index */
|
||||
/**
|
||||
* Total uncompressed size of all blocks of the data index
|
||||
*/
|
||||
private long uncompressedDataIndexSize;
|
||||
|
||||
/** The number of entries in the meta index */
|
||||
/**
|
||||
* The number of entries in the meta index
|
||||
*/
|
||||
private int metaIndexCount;
|
||||
|
||||
/** The total uncompressed size of keys/values stored in the file. */
|
||||
/**
|
||||
* The total uncompressed size of keys/values stored in the file.
|
||||
*/
|
||||
private long totalUncompressedBytes;
|
||||
|
||||
/**
|
||||
|
@ -95,7 +101,9 @@ public class FixedFileTrailer {
|
|||
*/
|
||||
private long entryCount;
|
||||
|
||||
/** The compression codec used for all blocks. */
|
||||
/**
|
||||
* The compression codec used for all blocks.
|
||||
*/
|
||||
private Compression.Algorithm compressionCodec = Compression.Algorithm.NONE;
|
||||
|
||||
/**
|
||||
|
@ -104,7 +112,9 @@ public class FixedFileTrailer {
|
|||
*/
|
||||
private int numDataIndexLevels;
|
||||
|
||||
/** The offset of the first data block. */
|
||||
/**
|
||||
* The offset of the first data block.
|
||||
*/
|
||||
private long firstDataBlockOffset;
|
||||
|
||||
/**
|
||||
|
@ -113,17 +123,25 @@ public class FixedFileTrailer {
|
|||
*/
|
||||
private long lastDataBlockOffset;
|
||||
|
||||
/** Raw key comparator class name in version 3 */
|
||||
/**
|
||||
* Raw key comparator class name in version 3
|
||||
*/
|
||||
// We could write the actual class name from 2.0 onwards and handle BC
|
||||
private String comparatorClassName = CellComparator.getInstance().getClass().getName();
|
||||
|
||||
/** The encryption key */
|
||||
/**
|
||||
* The encryption key
|
||||
*/
|
||||
private byte[] encryptionKey;
|
||||
|
||||
/** The {@link HFile} format major version. */
|
||||
/**
|
||||
* The {@link HFile} format major version.
|
||||
*/
|
||||
private final int majorVersion;
|
||||
|
||||
/** The {@link HFile} format minor version. */
|
||||
/**
|
||||
* The {@link HFile} format minor version.
|
||||
*/
|
||||
private final int minorVersion;
|
||||
|
||||
FixedFileTrailer(int majorVersion, int minorVersion) {
|
||||
|
@ -133,7 +151,7 @@ public class FixedFileTrailer {
|
|||
}
|
||||
|
||||
private static int[] computeTrailerSizeByVersion() {
|
||||
int versionToSize[] = new int[HFile.MAX_FORMAT_VERSION + 1];
|
||||
int[] versionToSize = new int[HFile.MAX_FORMAT_VERSION + 1];
|
||||
// We support only 2 major versions now. ie. V2, V3
|
||||
versionToSize[2] = 212;
|
||||
for (int version = 3; version <= HFile.MAX_FORMAT_VERSION; version++) {
|
||||
|
@ -147,14 +165,13 @@ public class FixedFileTrailer {
|
|||
|
||||
private static int getMaxTrailerSize() {
|
||||
int maxSize = 0;
|
||||
for (int version = HFile.MIN_FORMAT_VERSION;
|
||||
version <= HFile.MAX_FORMAT_VERSION;
|
||||
++version)
|
||||
for (int version = HFile.MIN_FORMAT_VERSION; version <= HFile.MAX_FORMAT_VERSION; ++version) {
|
||||
maxSize = Math.max(getTrailerSize(version), maxSize);
|
||||
}
|
||||
return maxSize;
|
||||
}
|
||||
|
||||
private static final int TRAILER_SIZE[] = computeTrailerSizeByVersion();
|
||||
private static final int[] TRAILER_SIZE = computeTrailerSizeByVersion();
|
||||
private static final int MAX_TRAILER_SIZE = getMaxTrailerSize();
|
||||
|
||||
private static final int NOT_PB_SIZE = BlockType.MAGIC_LENGTH + Bytes.SIZEOF_INT;
|
||||
|
@ -171,9 +188,6 @@ public class FixedFileTrailer {
|
|||
* Write the trailer to a data stream. We support writing version 1 for
|
||||
* testing and for determining version 1 trailer size. It is also easy to see
|
||||
* what fields changed in version 2.
|
||||
*
|
||||
* @param outputStream
|
||||
* @throws IOException
|
||||
*/
|
||||
void serialize(DataOutputStream outputStream) throws IOException {
|
||||
HFile.checkFormatVersion(majorVersion);
|
||||
|
@ -193,18 +207,18 @@ public class FixedFileTrailer {
|
|||
@org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting
|
||||
HFileProtos.FileTrailerProto toProtobuf() {
|
||||
HFileProtos.FileTrailerProto.Builder builder = HFileProtos.FileTrailerProto.newBuilder()
|
||||
.setFileInfoOffset(fileInfoOffset)
|
||||
.setLoadOnOpenDataOffset(loadOnOpenDataOffset)
|
||||
.setUncompressedDataIndexSize(uncompressedDataIndexSize)
|
||||
.setTotalUncompressedBytes(totalUncompressedBytes)
|
||||
.setDataIndexCount(dataIndexCount)
|
||||
.setMetaIndexCount(metaIndexCount)
|
||||
.setEntryCount(entryCount)
|
||||
.setNumDataIndexLevels(numDataIndexLevels)
|
||||
.setFirstDataBlockOffset(firstDataBlockOffset)
|
||||
.setLastDataBlockOffset(lastDataBlockOffset)
|
||||
.setComparatorClassName(getHBase1CompatibleName(comparatorClassName))
|
||||
.setCompressionCodec(compressionCodec.ordinal());
|
||||
.setFileInfoOffset(fileInfoOffset)
|
||||
.setLoadOnOpenDataOffset(loadOnOpenDataOffset)
|
||||
.setUncompressedDataIndexSize(uncompressedDataIndexSize)
|
||||
.setTotalUncompressedBytes(totalUncompressedBytes)
|
||||
.setDataIndexCount(dataIndexCount)
|
||||
.setMetaIndexCount(metaIndexCount)
|
||||
.setEntryCount(entryCount)
|
||||
.setNumDataIndexLevels(numDataIndexLevels)
|
||||
.setFirstDataBlockOffset(firstDataBlockOffset)
|
||||
.setLastDataBlockOffset(lastDataBlockOffset)
|
||||
.setComparatorClassName(getHBase1CompatibleName(comparatorClassName))
|
||||
.setCompressionCodec(compressionCodec.ordinal());
|
||||
if (encryptionKey != null) {
|
||||
builder.setEncryptionKey(UnsafeByteOperations.unsafeWrap(encryptionKey));
|
||||
}
|
||||
|
@ -239,9 +253,6 @@ public class FixedFileTrailer {
|
|||
* Deserialize the fixed file trailer from the given stream. The version needs
|
||||
* to already be specified. Make sure this is consistent with
|
||||
* {@link #serialize(DataOutputStream)}.
|
||||
*
|
||||
* @param inputStream
|
||||
* @throws IOException
|
||||
*/
|
||||
void deserialize(DataInputStream inputStream) throws IOException {
|
||||
HFile.checkFormatVersion(majorVersion);
|
||||
|
@ -249,7 +260,7 @@ public class FixedFileTrailer {
|
|||
BlockType.TRAILER.readAndCheck(inputStream);
|
||||
|
||||
if (majorVersion > 2
|
||||
|| (majorVersion == 2 && minorVersion >= HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION)) {
|
||||
|| (majorVersion == 2 && minorVersion >= HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION)) {
|
||||
deserializeFromPB(inputStream);
|
||||
} else {
|
||||
deserializeFromWritable(inputStream);
|
||||
|
@ -263,14 +274,12 @@ public class FixedFileTrailer {
|
|||
|
||||
/**
|
||||
* Deserialize the file trailer as protobuf
|
||||
* @param inputStream
|
||||
* @throws IOException
|
||||
*/
|
||||
void deserializeFromPB(DataInputStream inputStream) throws IOException {
|
||||
// read PB and skip padding
|
||||
int start = inputStream.available();
|
||||
HFileProtos.FileTrailerProto trailerProto =
|
||||
HFileProtos.FileTrailerProto.PARSER.parseDelimitedFrom(inputStream);
|
||||
HFileProtos.FileTrailerProto.PARSER.parseDelimitedFrom(inputStream);
|
||||
int size = start - inputStream.available();
|
||||
inputStream.skip(getTrailerSize() - NOT_PB_SIZE - size);
|
||||
|
||||
|
@ -320,8 +329,6 @@ public class FixedFileTrailer {
|
|||
|
||||
/**
|
||||
* Deserialize the file trailer as writable data
|
||||
* @param input
|
||||
* @throws IOException
|
||||
*/
|
||||
void deserializeFromWritable(DataInput input) throws IOException {
|
||||
fileInfoOffset = input.readLong();
|
||||
|
@ -339,12 +346,13 @@ public class FixedFileTrailer {
|
|||
// TODO this is a classname encoded into an HFile's trailer. We are going to need to have
|
||||
// some compat code here.
|
||||
setComparatorClass(getComparatorClass(Bytes.readStringFixedSize(input,
|
||||
MAX_COMPARATOR_NAME_LENGTH)));
|
||||
MAX_COMPARATOR_NAME_LENGTH)));
|
||||
}
|
||||
|
||||
|
||||
private void append(StringBuilder sb, String s) {
|
||||
if (sb.length() > 0)
|
||||
if (sb.length() > 0) {
|
||||
sb.append(", ");
|
||||
}
|
||||
sb.append(s);
|
||||
}
|
||||
|
||||
|
@ -375,18 +383,18 @@ public class FixedFileTrailer {
|
|||
/**
|
||||
* Reads a file trailer from the given file.
|
||||
*
|
||||
* @param istream the input stream with the ability to seek. Does not have to
|
||||
* be buffered, as only one read operation is made.
|
||||
* @param istream the input stream with the ability to seek. Does not have to
|
||||
* be buffered, as only one read operation is made.
|
||||
* @param fileSize the file size. Can be obtained using
|
||||
* {@link org.apache.hadoop.fs.FileSystem#getFileStatus(
|
||||
* org.apache.hadoop.fs.Path)}.
|
||||
* {@link org.apache.hadoop.fs.FileSystem#getFileStatus(
|
||||
*org.apache.hadoop.fs.Path)}.
|
||||
* @return the fixed file trailer read
|
||||
* @throws IOException if failed to read from the underlying stream, or the
|
||||
* trailer is corrupted, or the version of the trailer is
|
||||
* unsupported
|
||||
* trailer is corrupted, or the version of the trailer is
|
||||
* unsupported
|
||||
*/
|
||||
public static FixedFileTrailer readFromStream(FSDataInputStream istream,
|
||||
long fileSize) throws IOException {
|
||||
long fileSize) throws IOException {
|
||||
int bufferSize = MAX_TRAILER_SIZE;
|
||||
long seekPoint = fileSize - bufferSize;
|
||||
if (seekPoint < 0) {
|
||||
|
@ -399,7 +407,7 @@ public class FixedFileTrailer {
|
|||
|
||||
ByteBuffer buf = ByteBuffer.allocate(bufferSize);
|
||||
istream.readFully(buf.array(), buf.arrayOffset(),
|
||||
buf.arrayOffset() + buf.limit());
|
||||
buf.arrayOffset() + buf.limit());
|
||||
|
||||
// Read the version from the last int of the file.
|
||||
buf.position(buf.limit() - Bytes.SIZEOF_INT);
|
||||
|
@ -415,30 +423,30 @@ public class FixedFileTrailer {
|
|||
|
||||
FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion);
|
||||
fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(),
|
||||
buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
|
||||
buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
|
||||
return fft;
|
||||
}
|
||||
|
||||
public void expectMajorVersion(int expected) {
|
||||
if (majorVersion != expected) {
|
||||
throw new IllegalArgumentException("Invalid HFile major version: "
|
||||
+ majorVersion
|
||||
+ " (expected: " + expected + ")");
|
||||
+ majorVersion
|
||||
+ " (expected: " + expected + ")");
|
||||
}
|
||||
}
|
||||
|
||||
public void expectMinorVersion(int expected) {
|
||||
if (minorVersion != expected) {
|
||||
throw new IllegalArgumentException("Invalid HFile minor version: "
|
||||
+ minorVersion + " (expected: " + expected + ")");
|
||||
+ minorVersion + " (expected: " + expected + ")");
|
||||
}
|
||||
}
|
||||
|
||||
public void expectAtLeastMajorVersion(int lowerBound) {
|
||||
if (majorVersion < lowerBound) {
|
||||
throw new IllegalArgumentException("Invalid HFile major version: "
|
||||
+ majorVersion
|
||||
+ " (expected: " + lowerBound + " or higher).");
|
||||
+ majorVersion
|
||||
+ " (expected: " + lowerBound + " or higher).");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -566,7 +574,7 @@ public class FixedFileTrailer {
|
|||
* than the new name; writing the new name will make it so newly-written hfiles are not parseable
|
||||
* by hbase-1.x, a facility we'd like to preserve across rolling upgrade and hbase-1.x clusters
|
||||
* reading hbase-2.x produce.
|
||||
*
|
||||
* <p>
|
||||
* The Comparators in hbase-2.x work the same as they did in hbase-1.x; they compare
|
||||
* KeyValues. In hbase-2.x they were renamed making use of the more generic 'Cell'
|
||||
* nomenclature to indicate that we intend to move away from KeyValues post hbase-2. A naming
|
||||
|
@ -579,9 +587,9 @@ public class FixedFileTrailer {
|
|||
*
|
||||
* @param comparator String class name of the Comparator used in this hfile.
|
||||
* @return What to store in the trailer as our comparator name.
|
||||
* @see #getComparatorClass(String)
|
||||
* @since hbase-2.0.0.
|
||||
* @deprecated Since hbase-2.0.0. Will be removed in hbase-3.0.0.
|
||||
* @see #getComparatorClass(String)
|
||||
*/
|
||||
@Deprecated
|
||||
private String getHBase1CompatibleName(final String comparator) {
|
||||
|
@ -596,20 +604,20 @@ public class FixedFileTrailer {
|
|||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static Class<? extends CellComparator> getComparatorClass(String comparatorClassName)
|
||||
throws IOException {
|
||||
throws IOException {
|
||||
Class<? extends CellComparator> comparatorKlass;
|
||||
// for BC
|
||||
if (comparatorClassName.equals(KeyValue.COMPARATOR.getLegacyKeyComparatorName())
|
||||
|| comparatorClassName.equals(KeyValue.COMPARATOR.getClass().getName())
|
||||
|| (comparatorClassName.equals("org.apache.hadoop.hbase.CellComparator"))) {
|
||||
|| comparatorClassName.equals(KeyValue.COMPARATOR.getClass().getName())
|
||||
|| (comparatorClassName.equals("org.apache.hadoop.hbase.CellComparator"))) {
|
||||
comparatorKlass = CellComparatorImpl.class;
|
||||
} else if (comparatorClassName.equals(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName())
|
||||
|| comparatorClassName.equals(KeyValue.META_COMPARATOR.getClass().getName())
|
||||
|| (comparatorClassName
|
||||
.equals("org.apache.hadoop.hbase.CellComparator$MetaCellComparator"))) {
|
||||
|| comparatorClassName.equals(KeyValue.META_COMPARATOR.getClass().getName())
|
||||
|| (comparatorClassName
|
||||
.equals("org.apache.hadoop.hbase.CellComparator$MetaCellComparator"))) {
|
||||
comparatorKlass = MetaCellComparator.class;
|
||||
} else if (comparatorClassName.equals("org.apache.hadoop.hbase.KeyValue$RawBytesComparator")
|
||||
|| comparatorClassName.equals("org.apache.hadoop.hbase.util.Bytes$ByteArrayComparator")) {
|
||||
|| comparatorClassName.equals("org.apache.hadoop.hbase.util.Bytes$ByteArrayComparator")) {
|
||||
// When the comparator to be used is Bytes.BYTES_RAWCOMPARATOR, we just return null from here
|
||||
// Bytes.BYTES_RAWCOMPARATOR is not a CellComparator
|
||||
comparatorKlass = null;
|
||||
|
@ -624,19 +632,22 @@ public class FixedFileTrailer {
|
|||
return comparatorKlass;
|
||||
}
|
||||
|
||||
public static CellComparator createComparator(
|
||||
String comparatorClassName) throws IOException {
|
||||
static CellComparator createComparator(String comparatorClassName) throws IOException {
|
||||
if (comparatorClassName.equals(CellComparatorImpl.COMPARATOR.getClass().getName())) {
|
||||
return CellComparatorImpl.COMPARATOR;
|
||||
} else if (comparatorClassName.equals(
|
||||
CellComparatorImpl.META_COMPARATOR.getClass().getName())) {
|
||||
return CellComparatorImpl.META_COMPARATOR;
|
||||
}
|
||||
try {
|
||||
|
||||
Class<? extends CellComparator> comparatorClass = getComparatorClass(comparatorClassName);
|
||||
if(comparatorClass != null){
|
||||
if (comparatorClass != null) {
|
||||
return comparatorClass.getDeclaredConstructor().newInstance();
|
||||
}
|
||||
LOG.warn("No Comparator class for " + comparatorClassName + ". Returning Null.");
|
||||
return null;
|
||||
} catch (Exception e) {
|
||||
throw new IOException("Comparator class " + comparatorClassName +
|
||||
" is not instantiable", e);
|
||||
throw new IOException("Comparator class " + comparatorClassName + " is not instantiable", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -650,7 +661,7 @@ public class FixedFileTrailer {
|
|||
}
|
||||
|
||||
public void setUncompressedDataIndexSize(
|
||||
long uncompressedDataIndexSize) {
|
||||
long uncompressedDataIndexSize) {
|
||||
expectAtLeastMajorVersion(2);
|
||||
this.uncompressedDataIndexSize = uncompressedDataIndexSize;
|
||||
}
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
/**
|
||||
/*
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
|
@ -26,7 +26,6 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.LongAdder;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
|
@ -52,7 +51,6 @@ import org.apache.hadoop.io.Writable;
|
|||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
|
||||
|
||||
|
@ -121,7 +119,7 @@ import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
|
|||
* an HFile instance?
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class HFile {
|
||||
public final class HFile {
|
||||
// LOG is being used in HFileBlock and CheckSumUtil
|
||||
static final Logger LOG = LoggerFactory.getLogger(HFile.class);
|
||||
|
||||
|
@ -177,6 +175,11 @@ public class HFile {
|
|||
/** Static instance for the metrics so that HFileReaders access the same instance */
|
||||
static final MetricsIO metrics = new MetricsIO(new MetricsIOWrapperImpl());
|
||||
|
||||
/**
|
||||
* Shutdown constructor.
|
||||
*/
|
||||
private HFile() {}
|
||||
|
||||
/**
|
||||
* Number of checksum verification failures. It also
|
||||
* clears the counter.
|
||||
|
@ -222,10 +225,11 @@ public class HFile {
|
|||
*/
|
||||
void addInlineBlockWriter(InlineBlockWriter bloomWriter);
|
||||
|
||||
// The below three methods take Writables. We'd like to undo Writables but undoing the below would be pretty
|
||||
// painful. Could take a byte [] or a Message but we want to be backward compatible around hfiles so would need
|
||||
// to map between Message and Writable or byte [] and current Writable serialization. This would be a bit of work
|
||||
// to little gain. Thats my thinking at moment. St.Ack 20121129
|
||||
// The below three methods take Writables. We'd like to undo Writables but undoing the below
|
||||
// would be pretty painful. Could take a byte [] or a Message but we want to be backward
|
||||
// compatible around hfiles so would need to map between Message and Writable or byte [] and
|
||||
// current Writable serialization. This would be a bit of work to little gain. Thats my
|
||||
// thinking at moment. St.Ack 20121129
|
||||
|
||||
void appendMetaBlock(String bloomFilterMetaKey, Writable metaWriter);
|
||||
|
||||
|
@ -258,7 +262,6 @@ public class HFile {
|
|||
protected FileSystem fs;
|
||||
protected Path path;
|
||||
protected FSDataOutputStream ostream;
|
||||
protected CellComparator comparator = CellComparator.getInstance();
|
||||
protected InetSocketAddress[] favoredNodes;
|
||||
private HFileContext fileContext;
|
||||
protected boolean shouldDropBehind = false;
|
||||
|
@ -282,12 +285,6 @@ public class HFile {
|
|||
return this;
|
||||
}
|
||||
|
||||
public WriterFactory withComparator(CellComparator comparator) {
|
||||
Preconditions.checkNotNull(comparator);
|
||||
this.comparator = comparator;
|
||||
return this;
|
||||
}
|
||||
|
||||
public WriterFactory withFavoredNodes(InetSocketAddress[] favoredNodes) {
|
||||
// Deliberately not checking for null here.
|
||||
this.favoredNodes = favoredNodes;
|
||||
|
@ -319,7 +316,7 @@ public class HFile {
|
|||
LOG.debug("Unable to set drop behind on {}", path.getName());
|
||||
}
|
||||
}
|
||||
return new HFileWriterImpl(conf, cacheConf, path, ostream, comparator, fileContext);
|
||||
return new HFileWriterImpl(conf, cacheConf, path, ostream, fileContext);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -349,16 +346,16 @@ public class HFile {
|
|||
CacheConfig cacheConf) {
|
||||
int version = getFormatVersion(conf);
|
||||
switch (version) {
|
||||
case 2:
|
||||
throw new IllegalArgumentException("This should never happen. " +
|
||||
"Did you change hfile.format.version to read v2? This version of the software writes v3" +
|
||||
" hfiles only (but it can read v2 files without having to update hfile.format.version " +
|
||||
"in hbase-site.xml)");
|
||||
case 3:
|
||||
return new HFile.WriterFactory(conf, cacheConf);
|
||||
default:
|
||||
throw new IllegalArgumentException("Cannot create writer for HFile " +
|
||||
"format version " + version);
|
||||
case 2:
|
||||
throw new IllegalArgumentException("This should never happen. " +
|
||||
"Did you change hfile.format.version to read v2? This version of the software writes v3" +
|
||||
" hfiles only (but it can read v2 files without having to update hfile.format.version " +
|
||||
"in hbase-site.xml)");
|
||||
case 3:
|
||||
return new HFile.WriterFactory(conf, cacheConf);
|
||||
default:
|
||||
throw new IllegalArgumentException("Cannot create writer for HFile " +
|
||||
"format version " + version);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -372,18 +369,15 @@ public class HFile {
|
|||
* Read in a file block.
|
||||
* @param offset offset to read.
|
||||
* @param onDiskBlockSize size of the block
|
||||
* @param cacheBlock
|
||||
* @param pread
|
||||
* @param isCompaction is this block being read as part of a compaction
|
||||
* @param expectedBlockType the block type we are expecting to read with this read operation,
|
||||
* or null to read whatever block type is available and avoid checking (that might reduce
|
||||
* caching efficiency of encoded data blocks)
|
||||
* or null to read whatever block type is available and avoid checking (that might reduce
|
||||
* caching efficiency of encoded data blocks)
|
||||
* @param expectedDataBlockEncoding the data block encoding the caller is expecting data blocks
|
||||
* to be in, or null to not perform this check and return the block irrespective of the
|
||||
* encoding. This check only applies to data blocks and can be set to null when the caller is
|
||||
* expecting to read a non-data block and has set expectedBlockType accordingly.
|
||||
* to be in, or null to not perform this check and return the block irrespective of the
|
||||
* encoding. This check only applies to data blocks and can be set to null when the caller is
|
||||
* expecting to read a non-data block and has set expectedBlockType accordingly.
|
||||
* @return Block wrapped in a ByteBuffer.
|
||||
* @throws IOException
|
||||
*/
|
||||
HFileBlock readBlock(long offset, long onDiskBlockSize,
|
||||
boolean cacheBlock, final boolean pread, final boolean isCompaction,
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
package org.apache.hadoop.hbase.io.hfile;
|
||||
|
||||
import static org.apache.hadoop.hbase.io.ByteBuffAllocator.HEAP;
|
||||
|
||||
import java.io.DataInputStream;
|
||||
import java.io.DataOutput;
|
||||
import java.io.DataOutputStream;
|
||||
|
@ -29,18 +28,13 @@ import java.util.List;
|
|||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.concurrent.locks.Lock;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
import org.apache.hadoop.fs.FSDataInputStream;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.io.ByteBuffAllocator;
|
||||
import org.apache.hadoop.hbase.io.util.BlockIOUtils;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.hadoop.hbase.fs.HFileSystem;
|
||||
import org.apache.hadoop.hbase.io.ByteArrayOutputStream;
|
||||
import org.apache.hadoop.hbase.io.ByteBuffAllocator;
|
||||
import org.apache.hadoop.hbase.io.ByteBuffInputStream;
|
||||
import org.apache.hadoop.hbase.io.ByteBufferWriterDataOutputStream;
|
||||
import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
|
||||
|
@ -50,6 +44,7 @@ import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext;
|
|||
import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext;
|
||||
import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
|
||||
import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
|
||||
import org.apache.hadoop.hbase.io.util.BlockIOUtils;
|
||||
import org.apache.hadoop.hbase.nio.ByteBuff;
|
||||
import org.apache.hadoop.hbase.nio.MultiByteBuff;
|
||||
import org.apache.hadoop.hbase.nio.SingleByteBuff;
|
||||
|
@ -57,7 +52,9 @@ import org.apache.hadoop.hbase.regionserver.ShipperListener;
|
|||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.ChecksumType;
|
||||
import org.apache.hadoop.hbase.util.ClassSize;
|
||||
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
|
||||
|
||||
|
@ -360,8 +357,8 @@ public class HFileBlock implements Cacheable {
|
|||
final long prevBlockOffset = buf.getLong(Header.PREV_BLOCK_OFFSET_INDEX);
|
||||
// This constructor is called when we deserialize a block from cache and when we read a block in
|
||||
// from the fs. fileCache is null when deserialized from cache so need to make up one.
|
||||
HFileContextBuilder fileContextBuilder =
|
||||
fileContext != null ? new HFileContextBuilder(fileContext) : new HFileContextBuilder();
|
||||
HFileContextBuilder fileContextBuilder = fileContext != null ?
|
||||
new HFileContextBuilder(fileContext) : new HFileContextBuilder();
|
||||
fileContextBuilder.withHBaseCheckSum(usesHBaseChecksum);
|
||||
int onDiskDataSizeWithHeader;
|
||||
if (usesHBaseChecksum) {
|
||||
|
@ -407,8 +404,8 @@ public class HFileBlock implements Cacheable {
|
|||
|
||||
/**
|
||||
* @return the on-disk size of the next block (including the header size and any checksums if
|
||||
* present) read by peeking into the next block's header; use as a hint when doing
|
||||
* a read of the next block when scanning or running over a file.
|
||||
* present) read by peeking into the next block's header; use as a hint when doing
|
||||
* a read of the next block when scanning or running over a file.
|
||||
*/
|
||||
int getNextBlockOnDiskSize() {
|
||||
return nextBlockOnDiskSize;
|
||||
|
@ -465,7 +462,7 @@ public class HFileBlock implements Cacheable {
|
|||
/**
|
||||
* @return the uncompressed size of data part (header and checksum excluded).
|
||||
*/
|
||||
int getUncompressedSizeWithoutHeader() {
|
||||
int getUncompressedSizeWithoutHeader() {
|
||||
return uncompressedSizeWithoutHeader;
|
||||
}
|
||||
|
||||
|
@ -705,8 +702,8 @@ public class HFileBlock implements Cacheable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Cannot be {@link #UNSET}. Must be a legitimate value. Used re-making the {@link BlockCacheKey} when
|
||||
* block is returned to the cache.
|
||||
* Cannot be {@link #UNSET}. Must be a legitimate value. Used re-making the {@link BlockCacheKey}
|
||||
* when block is returned to the cache.
|
||||
* @return the offset of this block in the file it was read from
|
||||
*/
|
||||
long getOffset() {
|
||||
|
@ -892,7 +889,6 @@ public class HFileBlock implements Cacheable {
|
|||
* Starts writing into the block. The previous block's data is discarded.
|
||||
*
|
||||
* @return the stream the user can write their data into
|
||||
* @throws IOException
|
||||
*/
|
||||
DataOutputStream startWriting(BlockType newBlockType)
|
||||
throws IOException {
|
||||
|
@ -922,8 +918,6 @@ public class HFileBlock implements Cacheable {
|
|||
|
||||
/**
|
||||
* Writes the Cell to this block
|
||||
* @param cell
|
||||
* @throws IOException
|
||||
*/
|
||||
void write(Cell cell) throws IOException{
|
||||
expectState(State.WRITING);
|
||||
|
@ -1041,9 +1035,6 @@ public class HFileBlock implements Cacheable {
|
|||
* Similar to {@link #writeHeaderAndData(FSDataOutputStream)}, but records
|
||||
* the offset of this block so that it can be referenced in the next block
|
||||
* of the same type.
|
||||
*
|
||||
* @param out
|
||||
* @throws IOException
|
||||
*/
|
||||
void writeHeaderAndData(FSDataOutputStream out) throws IOException {
|
||||
long offset = out.getPos();
|
||||
|
@ -1061,9 +1052,7 @@ public class HFileBlock implements Cacheable {
|
|||
* data when not using compression) into the given stream. Can be called in
|
||||
* the "writing" state or in the "block ready" state. If called in the
|
||||
* "writing" state, transitions the writer to the "block ready" state.
|
||||
*
|
||||
* @param out the output stream to write the
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void finishBlockAndWriteHeaderAndData(DataOutputStream out)
|
||||
throws IOException {
|
||||
|
@ -1082,7 +1071,6 @@ public class HFileBlock implements Cacheable {
|
|||
* the header + data + checksums stored on disk.
|
||||
*
|
||||
* @return header and data as they would be stored on disk in a byte array
|
||||
* @throws IOException
|
||||
*/
|
||||
byte[] getHeaderAndDataForTest() throws IOException {
|
||||
ensureBlockReady();
|
||||
|
@ -1167,9 +1155,7 @@ public class HFileBlock implements Cacheable {
|
|||
* @return the number of bytes written
|
||||
*/
|
||||
public int encodedBlockSizeWritten() {
|
||||
if (state != State.WRITING)
|
||||
return 0;
|
||||
return this.encodedDataSizeWritten;
|
||||
return state != State.WRITING ? 0 : this.encodedDataSizeWritten;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -1180,8 +1166,7 @@ public class HFileBlock implements Cacheable {
|
|||
* @return the number of bytes written
|
||||
*/
|
||||
int blockSizeWritten() {
|
||||
if (state != State.WRITING) return 0;
|
||||
return this.unencodedDataSizeWritten;
|
||||
return state != State.WRITING ? 0 : this.unencodedDataSizeWritten;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -1234,7 +1219,6 @@ public class HFileBlock implements Cacheable {
|
|||
*
|
||||
* @param bw the block-writable object to write as a block
|
||||
* @param out the file system output stream
|
||||
* @throws IOException
|
||||
*/
|
||||
void writeBlock(BlockWritable bw, FSDataOutputStream out)
|
||||
throws IOException {
|
||||
|
@ -1431,7 +1415,8 @@ public class HFileBlock implements Cacheable {
|
|||
* next blocks header seems unnecessary given we usually get the block size
|
||||
* from the hfile index. Review!
|
||||
*/
|
||||
private AtomicReference<PrefetchedHeader> prefetchedHeader = new AtomicReference<>(new PrefetchedHeader());
|
||||
private AtomicReference<PrefetchedHeader> prefetchedHeader =
|
||||
new AtomicReference<>(new PrefetchedHeader());
|
||||
|
||||
/** The size of the file we are reading from, or -1 if unknown. */
|
||||
private long fileSize;
|
||||
|
@ -1643,10 +1628,9 @@ public class HFileBlock implements Cacheable {
|
|||
|
||||
/**
|
||||
* @return Check <code>onDiskSizeWithHeaderL</code> size is healthy and then return it as an int
|
||||
* @throws IOException
|
||||
*/
|
||||
private static int checkAndGetSizeAsInt(final long onDiskSizeWithHeaderL, final int hdrSize)
|
||||
throws IOException {
|
||||
throws IOException {
|
||||
if ((onDiskSizeWithHeaderL < hdrSize && onDiskSizeWithHeaderL != -1)
|
||||
|| onDiskSizeWithHeaderL >= Integer.MAX_VALUE) {
|
||||
throw new IOException("Invalid onDisksize=" + onDiskSizeWithHeaderL
|
||||
|
@ -1659,11 +1643,10 @@ public class HFileBlock implements Cacheable {
|
|||
/**
|
||||
* Verify the passed in onDiskSizeWithHeader aligns with what is in the header else something
|
||||
* is not right.
|
||||
* @throws IOException
|
||||
*/
|
||||
private void verifyOnDiskSizeMatchesHeader(final int passedIn, final ByteBuff headerBuf,
|
||||
final long offset, boolean verifyChecksum)
|
||||
throws IOException {
|
||||
final long offset, boolean verifyChecksum)
|
||||
throws IOException {
|
||||
// Assert size provided aligns with what is in the header
|
||||
int fromHeader = getOnDiskSizeWithHeader(headerBuf, verifyChecksum);
|
||||
if (passedIn != fromHeader) {
|
||||
|
@ -1827,7 +1810,8 @@ public class HFileBlock implements Cacheable {
|
|||
|
||||
@Override
|
||||
public void setIncludesMemStoreTS(boolean includesMemstoreTS) {
|
||||
this.fileContext.setIncludesMvcc(includesMemstoreTS);
|
||||
this.fileContext = new HFileContextBuilder(this.fileContext)
|
||||
.withIncludesMvcc(includesMemstoreTS).build();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -2073,7 +2057,7 @@ public class HFileBlock implements Cacheable {
|
|||
|
||||
/**
|
||||
* @return This HFileBlocks fileContext which will a derivative of the
|
||||
* fileContext for the file from which this block's data was originally read.
|
||||
* fileContext for the file from which this block's data was originally read.
|
||||
*/
|
||||
HFileContext getHFileContext() {
|
||||
return this.fileContext;
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
/**
|
||||
/*
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
|
@ -33,7 +33,6 @@ import java.util.Objects;
|
|||
import java.util.Set;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
|
@ -52,7 +51,14 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.BytesBytesP
|
|||
import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos;
|
||||
|
||||
/**
|
||||
* Metadata for HFile. Conjured by the writer. Read in by the reader.
|
||||
* Metadata Map of attributes for HFile written out as HFile Trailer. Created by the Writer and
|
||||
* added to the tail of the file just before close. Metadata includes core attributes such as last
|
||||
* key seen, comparator used writing the file, etc. Clients can add their own attributes via
|
||||
* {@link #append(byte[], byte[], boolean)} and they'll be persisted and available at read time.
|
||||
* Reader creates the HFileInfo on open by reading the tail of the HFile. The parse of the HFile
|
||||
* trailer also creates a {@link HFileContext}, a read-only data structure that includes bulk of
|
||||
* the HFileInfo and extras that is safe to pass around when working on HFiles.
|
||||
* @see HFileContext
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class HFileInfo implements SortedMap<byte[], byte[]> {
|
||||
|
@ -62,7 +68,6 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
|
|||
static final byte [] AVG_KEY_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN");
|
||||
static final byte [] AVG_VALUE_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN");
|
||||
static final byte [] CREATE_TIME_TS = Bytes.toBytes(RESERVED_PREFIX + "CREATE_TIME_TS");
|
||||
static final byte [] COMPARATOR = Bytes.toBytes(RESERVED_PREFIX + "COMPARATOR");
|
||||
static final byte [] TAGS_COMPRESSED = Bytes.toBytes(RESERVED_PREFIX + "TAGS_COMPRESSED");
|
||||
public static final byte [] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX + "MAX_TAGS_LEN");
|
||||
private final SortedMap<byte [], byte []> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
|
||||
|
@ -380,7 +385,8 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
|
|||
HFileContextBuilder builder = new HFileContextBuilder()
|
||||
.withHBaseCheckSum(true)
|
||||
.withHFileName(path.getName())
|
||||
.withCompression(trailer.getCompressionCodec());
|
||||
.withCompression(trailer.getCompressionCodec())
|
||||
.withCellComparator(trailer.createComparator(trailer.getComparatorClassName()));
|
||||
// Check for any key material available
|
||||
byte[] keyBytes = trailer.getEncryptionKey();
|
||||
if (keyBytes != null) {
|
||||
|
|
|
@ -22,7 +22,6 @@ import java.io.IOException;
|
|||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.hadoop.conf.Configurable;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
|
@ -49,12 +48,10 @@ import org.apache.hadoop.hbase.util.Bytes;
|
|||
import org.apache.hadoop.hbase.util.IdLock;
|
||||
import org.apache.hadoop.hbase.util.ObjectIntPair;
|
||||
import org.apache.hadoop.io.WritableUtils;
|
||||
|
||||
import org.apache.htrace.core.TraceScope;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
/**
|
||||
|
@ -85,9 +82,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
*/
|
||||
protected HFileDataBlockEncoder dataBlockEncoder = NoOpDataBlockEncoder.INSTANCE;
|
||||
|
||||
/** Key comparator */
|
||||
protected CellComparator comparator = CellComparator.getInstance();
|
||||
|
||||
/** Block cache configuration. */
|
||||
protected final CacheConfig cacheConf;
|
||||
|
||||
|
@ -145,8 +139,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
this.primaryReplicaReader = context.isPrimaryReplicaReader();
|
||||
this.fileInfo = fileInfo;
|
||||
this.trailer = fileInfo.getTrailer();
|
||||
// Comparator class name is stored in the trailer in version 2.
|
||||
this.comparator = trailer.createComparator();
|
||||
this.hfileContext = fileInfo.getHFileContext();
|
||||
this.fsBlockReader = new HFileBlock.FSReaderImpl(context, hfileContext,
|
||||
cacheConf.getByteBuffAllocator());
|
||||
|
@ -238,7 +230,7 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
/** @return comparator */
|
||||
@Override
|
||||
public CellComparator getComparator() {
|
||||
return comparator;
|
||||
return this.hfileContext.getCellComparator();
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
|
@ -398,8 +390,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
}
|
||||
|
||||
protected void assertSeeked() {
|
||||
if (!isSeeked())
|
||||
if (!isSeeked()) {
|
||||
throw new NotSeekedException(reader.getPath());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -476,7 +469,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
|
||||
/**
|
||||
* Read mvcc. Does checks to see if we even need to read the mvcc at all.
|
||||
* @param offsetFromPos
|
||||
*/
|
||||
protected void readMvccVersion(final int offsetFromPos) {
|
||||
// See if we even need to decode mvcc.
|
||||
|
@ -493,7 +485,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
|
||||
/**
|
||||
* Actually do the mvcc read. Does no checks.
|
||||
* @param offsetFromPos
|
||||
*/
|
||||
private void _readMvccVersion(int offsetFromPos) {
|
||||
// This is Bytes#bytesToVint inlined so can save a few instructions in this hot method; i.e.
|
||||
|
@ -681,7 +672,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
* 0 if we are at the given key, 1 if we are past the given key
|
||||
* -2 if the key is earlier than the first key of the file while
|
||||
* using a faked index key
|
||||
* @throws IOException
|
||||
*/
|
||||
public int seekTo(Cell key, boolean rewind) throws IOException {
|
||||
HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader();
|
||||
|
@ -747,7 +737,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
* data block is found.
|
||||
*
|
||||
* @return the next block, or null if there are no more data blocks
|
||||
* @throws IOException
|
||||
*/
|
||||
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH",
|
||||
justification="Yeah, unnecessary null check; could do w/ clean up")
|
||||
|
@ -764,7 +753,7 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
}
|
||||
if (block.getOffset() < 0) {
|
||||
releaseIfNotCurBlock(block);
|
||||
throw new IOException("Invalid block file offset: " + block + ", path=" + reader.getPath());
|
||||
throw new IOException("Invalid block offset: " + block + ", path=" + reader.getPath());
|
||||
}
|
||||
// We are reading the next block without block type validation, because
|
||||
// it might turn out to be a non-data block.
|
||||
|
@ -786,8 +775,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
|
||||
@Override
|
||||
public Cell getCell() {
|
||||
if (!isSeeked())
|
||||
if (!isSeeked()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Cell ret;
|
||||
int cellBufSize = getKVBufSize();
|
||||
|
@ -886,7 +876,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
/**
|
||||
* Set our selves up for the next 'next' invocation, set up next block.
|
||||
* @return True is more to read else false if at the end.
|
||||
* @throws IOException
|
||||
*/
|
||||
private boolean positionForNextBlock() throws IOException {
|
||||
// Methods are small so they get inlined because they are 'hot'.
|
||||
|
@ -942,7 +931,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
*
|
||||
* @return false if empty file; i.e. a call to next would return false and
|
||||
* the current key and value are undefined.
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public boolean seekTo() throws IOException {
|
||||
|
@ -975,7 +963,8 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding());
|
||||
if (newBlock.getOffset() < 0) {
|
||||
releaseIfNotCurBlock(newBlock);
|
||||
throw new IOException("Invalid block offset: " + newBlock.getOffset() + ", path=" + reader.getPath());
|
||||
throw new IOException("Invalid offset=" + newBlock.getOffset() +
|
||||
", path=" + reader.getPath());
|
||||
}
|
||||
updateCurrentBlock(newBlock);
|
||||
}
|
||||
|
@ -993,7 +982,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
}
|
||||
|
||||
/**
|
||||
* @param v
|
||||
* @return True if v <= 0 or v > current block buffer limit.
|
||||
*/
|
||||
protected final boolean checkKeyLen(final int v) {
|
||||
|
@ -1001,7 +989,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
}
|
||||
|
||||
/**
|
||||
* @param v
|
||||
* @return True if v < 0 or v > current block buffer limit.
|
||||
*/
|
||||
protected final boolean checkLen(final int v) {
|
||||
|
@ -1172,9 +1159,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
// so blocks with the old encoding still linger in cache for some
|
||||
// period of time. This event should be rare as it only happens on
|
||||
// schema definition change.
|
||||
LOG.info("Evicting cached block with key " + cacheKey
|
||||
+ " because of a data block encoding mismatch" + "; expected: "
|
||||
+ expectedDataBlockEncoding + ", actual: " + actualDataBlockEncoding + ", path=" + path);
|
||||
LOG.info("Evicting cached block with key {} because data block encoding mismatch; " +
|
||||
"expected {}, actual {}, path={}", cacheKey, actualDataBlockEncoding,
|
||||
expectedDataBlockEncoding, path);
|
||||
// This is an error scenario. so here we need to release the block.
|
||||
returnAndEvictBlock(cache, cacheKey, cachedBlock);
|
||||
}
|
||||
|
@ -1192,10 +1179,8 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
}
|
||||
|
||||
/**
|
||||
* @param metaBlockName
|
||||
* @param cacheBlock Add block to cache, if found
|
||||
* @return block wrapped in a ByteBuffer, with header skipped
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public HFileBlock getMetaBlock(String metaBlockName, boolean cacheBlock)
|
||||
|
@ -1210,8 +1195,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
byte[] mbname = Bytes.toBytes(metaBlockName);
|
||||
int block = metaBlockIndexReader.rootBlockContainingKey(mbname,
|
||||
0, mbname.length);
|
||||
if (block == -1)
|
||||
if (block == -1) {
|
||||
return null;
|
||||
}
|
||||
long blockSize = metaBlockIndexReader.getRootBlockDataSize(block);
|
||||
|
||||
// Per meta key from any given file, synchronize reads for said block. This
|
||||
|
@ -1418,7 +1404,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
/**
|
||||
* @return Midkey for this file. We work with block boundaries only so
|
||||
* returned midkey is an approximation only.
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public Optional<Cell> midKey() throws IOException {
|
||||
|
@ -1455,8 +1440,7 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
DataBlockEncoding encoding = reader.getDataBlockEncoding();
|
||||
dataBlockEncoder = encoding.getEncoder();
|
||||
decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta);
|
||||
seeker = dataBlockEncoder.createSeeker(
|
||||
reader.getComparator(), decodingCtx);
|
||||
seeker = dataBlockEncoder.createSeeker(decodingCtx);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -1475,7 +1459,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
* @param newBlock the block to make current, and read by {@link HFileReaderImpl#readBlock},
|
||||
* it's a totally new block with new allocated {@link ByteBuff}, so if no further
|
||||
* reference to this block, we should release it carefully.
|
||||
* @throws CorruptHFileException
|
||||
*/
|
||||
@Override
|
||||
protected void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException {
|
||||
|
@ -1487,9 +1470,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
short dataBlockEncoderId = newBlock.getDataBlockEncodingId();
|
||||
if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) {
|
||||
String encoderCls = dataBlockEncoder.getClass().getName();
|
||||
throw new CorruptHFileException(
|
||||
"Encoder " + encoderCls + " doesn't support data block encoding "
|
||||
+ DataBlockEncoding.getNameFromId(dataBlockEncoderId) + ",path=" + reader.getPath());
|
||||
throw new CorruptHFileException("Encoder " + encoderCls +
|
||||
" doesn't support data block encoding " +
|
||||
DataBlockEncoding.getNameFromId(dataBlockEncoderId) + ",path=" + reader.getPath());
|
||||
}
|
||||
updateCurrBlockRef(newBlock);
|
||||
ByteBuff encodedBuffer = getEncodedBuffer(newBlock);
|
||||
|
@ -1606,7 +1589,7 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable {
|
|||
}
|
||||
|
||||
private DataInput getBloomFilterMetadata(BlockType blockType)
|
||||
throws IOException {
|
||||
throws IOException {
|
||||
if (blockType != BlockType.GENERAL_BLOOM_META &&
|
||||
blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
|
||||
throw new RuntimeException("Block Type: " + blockType.toString() +
|
||||
|
|
|
@ -25,7 +25,6 @@ import java.net.InetSocketAddress;
|
|||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
|
@ -65,7 +64,8 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
private static final long UNSET = -1;
|
||||
|
||||
/** if this feature is enabled, preCalculate encoded data size before real encoding happens*/
|
||||
public static final String UNIFIED_ENCODED_BLOCKSIZE_RATIO = "hbase.writer.unified.encoded.blocksize.ratio";
|
||||
public static final String UNIFIED_ENCODED_BLOCKSIZE_RATIO =
|
||||
"hbase.writer.unified.encoded.blocksize.ratio";
|
||||
|
||||
/** Block size limit after encoding, used to unify encoded block Cache entry size*/
|
||||
private final int encodedBlockSizeLimit;
|
||||
|
@ -94,9 +94,6 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
/** Total uncompressed bytes, maybe calculate a compression ratio later. */
|
||||
protected long totalUncompressedBytes = 0;
|
||||
|
||||
/** Key comparator. Used to ensure we write in order. */
|
||||
protected final CellComparator comparator;
|
||||
|
||||
/** Meta block names. */
|
||||
protected List<byte[]> metaNames = new ArrayList<>();
|
||||
|
||||
|
@ -165,8 +162,7 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
protected long maxMemstoreTS = 0;
|
||||
|
||||
public HFileWriterImpl(final Configuration conf, CacheConfig cacheConf, Path path,
|
||||
FSDataOutputStream outputStream,
|
||||
CellComparator comparator, HFileContext fileContext) {
|
||||
FSDataOutputStream outputStream, HFileContext fileContext) {
|
||||
this.outputStream = outputStream;
|
||||
this.path = path;
|
||||
this.name = path != null ? path.getName() : outputStream.toString();
|
||||
|
@ -177,8 +173,6 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
} else {
|
||||
this.blockEncoder = NoOpDataBlockEncoder.INSTANCE;
|
||||
}
|
||||
this.comparator = comparator != null ? comparator : CellComparator.getInstance();
|
||||
|
||||
closeOutputStream = path != null;
|
||||
this.cacheConf = cacheConf;
|
||||
float encodeBlockSizeRatio = conf.getFloat(UNIFIED_ENCODED_BLOCKSIZE_RATIO, 1f);
|
||||
|
@ -187,7 +181,6 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("Writer" + (path != null ? " for " + path : "") +
|
||||
" initialized with cacheConf: " + cacheConf +
|
||||
" comparator: " + comparator.getClass().getSimpleName() +
|
||||
" fileContext: " + fileContext);
|
||||
}
|
||||
}
|
||||
|
@ -214,10 +207,9 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
*
|
||||
* @param trailer fixed file trailer
|
||||
* @param out the data output to write the file info to
|
||||
* @throws IOException
|
||||
*/
|
||||
protected final void writeFileInfo(FixedFileTrailer trailer, DataOutputStream out)
|
||||
throws IOException {
|
||||
throws IOException {
|
||||
trailer.setFileInfoOffset(outputStream.getPos());
|
||||
finishFileInfo();
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
@ -239,7 +231,8 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
throw new IOException("Key cannot be null or empty");
|
||||
}
|
||||
if (lastCell != null) {
|
||||
int keyComp = PrivateCellUtil.compareKeyIgnoresMvcc(comparator, lastCell, cell);
|
||||
int keyComp = PrivateCellUtil.compareKeyIgnoresMvcc(this.hFileContext.getCellComparator(),
|
||||
lastCell, cell);
|
||||
if (keyComp > 0) {
|
||||
String message = getLexicalErrorMessage(cell);
|
||||
throw new IOException(message);
|
||||
|
@ -285,8 +278,9 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
}
|
||||
|
||||
public static Compression.Algorithm compressionByName(String algoName) {
|
||||
if (algoName == null)
|
||||
if (algoName == null) {
|
||||
return HFile.DEFAULT_COMPRESSION_ALGORITHM;
|
||||
}
|
||||
return Compression.getCompressionAlgorithmByName(algoName);
|
||||
}
|
||||
|
||||
|
@ -318,17 +312,15 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
|
||||
// Meta data block index writer
|
||||
metaBlockIndexWriter = new HFileBlockIndex.BlockIndexWriter();
|
||||
if (LOG.isTraceEnabled()) LOG.trace("Initialized with " + cacheConf);
|
||||
LOG.trace("Initialized with {}", cacheConf);
|
||||
}
|
||||
|
||||
/**
|
||||
* At a block boundary, write all the inline blocks and opens new block.
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void checkBlockBoundary() throws IOException {
|
||||
//for encoder like prefixTree, encoded size is not available, so we have to compare both encoded size
|
||||
//and unencoded size to blocksize limit.
|
||||
// For encoder like prefixTree, encoded size is not available, so we have to compare both
|
||||
// encoded size and unencoded size to blocksize limit.
|
||||
if (blockWriter.encodedBlockSizeWritten() >= encodedBlockSizeLimit
|
||||
|| blockWriter.blockSizeWritten() >= hFileContext.getBlocksize()) {
|
||||
finishBlock();
|
||||
|
@ -339,7 +331,9 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
|
||||
/** Clean up the data block that is currently being written.*/
|
||||
private void finishBlock() throws IOException {
|
||||
if (!blockWriter.isWriting() || blockWriter.blockSizeWritten() == 0) return;
|
||||
if (!blockWriter.isWriting() || blockWriter.blockSizeWritten() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Update the first data block offset if UNSET; used scanning.
|
||||
if (firstDataBlockOffset == UNSET) {
|
||||
|
@ -350,7 +344,7 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
blockWriter.writeHeaderAndData(outputStream);
|
||||
int onDiskSize = blockWriter.getOnDiskSizeWithHeader();
|
||||
Cell indexEntry =
|
||||
getMidpoint(this.comparator, lastCellOfPreviousBlock, firstCellInBlock);
|
||||
getMidpoint(this.hFileContext.getCellComparator(), lastCellOfPreviousBlock, firstCellInBlock);
|
||||
dataBlockIndexWriter.addEntry(PrivateCellUtil.getCellKeySerializedAsKeyValueKey(indexEntry),
|
||||
lastDataBlockOffset, onDiskSize);
|
||||
totalUncompressedBytes += blockWriter.getUncompressedSizeWithHeader();
|
||||
|
@ -364,11 +358,6 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
* <code>right</code> but that is shorter; i.e. takes up less space. This
|
||||
* trick is used building HFile block index. Its an optimization. It does not
|
||||
* always work. In this case we'll just return the <code>right</code> cell.
|
||||
*
|
||||
* @param comparator
|
||||
* Comparator to use.
|
||||
* @param left
|
||||
* @param right
|
||||
* @return A cell that sorts between <code>left</code> and <code>right</code>.
|
||||
*/
|
||||
public static Cell getMidpoint(final CellComparator comparator, final Cell left,
|
||||
|
@ -407,7 +396,9 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
left.getRowLength(), right.getRowArray(), right.getRowOffset(), right.getRowLength());
|
||||
}
|
||||
// If midRow is null, just return 'right'. Can't do optimization.
|
||||
if (midRow == null) return right;
|
||||
if (midRow == null) {
|
||||
return right;
|
||||
}
|
||||
return PrivateCellUtil.createFirstOnRow(midRow);
|
||||
}
|
||||
// Rows are same. Compare on families.
|
||||
|
@ -428,7 +419,9 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
right.getFamilyLength());
|
||||
}
|
||||
// If midRow is null, just return 'right'. Can't do optimization.
|
||||
if (midRow == null) return right;
|
||||
if (midRow == null) {
|
||||
return right;
|
||||
}
|
||||
// Return new Cell where we use right row and then a mid sort family.
|
||||
return PrivateCellUtil.createFirstOnRowFamily(right, midRow, 0, midRow.length);
|
||||
}
|
||||
|
@ -450,7 +443,9 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
right.getQualifierLength());
|
||||
}
|
||||
// If midRow is null, just return 'right'. Can't do optimization.
|
||||
if (midRow == null) return right;
|
||||
if (midRow == null) {
|
||||
return right;
|
||||
}
|
||||
// Return new Cell where we use right row and family and then a mid sort qualifier.
|
||||
return PrivateCellUtil.createFirstOnRowCol(right, midRow, 0, midRow.length);
|
||||
}
|
||||
|
@ -459,12 +454,6 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
}
|
||||
|
||||
/**
|
||||
* @param leftArray
|
||||
* @param leftOffset
|
||||
* @param leftLength
|
||||
* @param rightArray
|
||||
* @param rightOffset
|
||||
* @param rightLength
|
||||
* @return Return a new array that is between left and right and minimally
|
||||
* sized else just return null as indicator that we could not create a
|
||||
* mid point.
|
||||
|
@ -566,8 +555,6 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
|
||||
/**
|
||||
* Ready a new block for writing.
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
protected void newBlock() throws IOException {
|
||||
// This is where the next block begins.
|
||||
|
@ -678,7 +665,7 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
dataBlockIndexWriter.getTotalUncompressedSize());
|
||||
trailer.setFirstDataBlockOffset(firstDataBlockOffset);
|
||||
trailer.setLastDataBlockOffset(lastDataBlockOffset);
|
||||
trailer.setComparatorClass(comparator.getClass());
|
||||
trailer.setComparatorClass(this.hFileContext.getCellComparator().getClass());
|
||||
trailer.setDataIndexCount(dataBlockIndexWriter.getNumRootEntries());
|
||||
|
||||
|
||||
|
@ -704,8 +691,9 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
|
||||
private void addBloomFilter(final BloomFilterWriter bfw,
|
||||
final BlockType blockType) {
|
||||
if (bfw.getKeyCount() <= 0)
|
||||
if (bfw.getKeyCount() <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (blockType != BlockType.GENERAL_BLOOM_META &&
|
||||
blockType != BlockType.DELETE_FAMILY_BLOOM_META) {
|
||||
|
@ -722,8 +710,9 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
public void writeToBlock(DataOutput out) throws IOException {
|
||||
bfw.getMetaWriter().write(out);
|
||||
Writable dataWriter = bfw.getDataWriter();
|
||||
if (dataWriter != null)
|
||||
if (dataWriter != null) {
|
||||
dataWriter.write(out);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
@ -739,7 +728,6 @@ public class HFileWriterImpl implements HFile.Writer {
|
|||
*
|
||||
* @param cell
|
||||
* Cell to add. Cannot be empty nor null.
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public void append(final Cell cell) throws IOException {
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
/**
|
||||
/*
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
|
@ -35,13 +35,11 @@ import java.util.concurrent.RejectedExecutionHandler;
|
|||
import java.util.concurrent.SynchronousQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.PrivateCellUtil;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
|
@ -124,7 +122,6 @@ public final class MobUtils {
|
|||
* Parses the string to a date.
|
||||
* @param dateString The string format of a date, it's yyyymmdd.
|
||||
* @return A date.
|
||||
* @throws ParseException
|
||||
*/
|
||||
public static Date parseDate(String dateString) throws ParseException {
|
||||
return LOCAL_FORMAT.get().parse(dateString);
|
||||
|
@ -288,7 +285,6 @@ public final class MobUtils {
|
|||
* @param columnDescriptor The descriptor of the current column family.
|
||||
* @param cacheConfig The cacheConfig that disables the block cache.
|
||||
* @param current The current time.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName,
|
||||
ColumnFamilyDescriptor columnDescriptor, CacheConfig cacheConfig, long current)
|
||||
|
@ -384,7 +380,6 @@ public final class MobUtils {
|
|||
* Gets the qualified root dir of the mob files.
|
||||
* @param conf The current configuration.
|
||||
* @return The qualified root dir.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static Path getQualifiedMobRootDir(Configuration conf) throws IOException {
|
||||
Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR));
|
||||
|
@ -505,7 +500,6 @@ public final class MobUtils {
|
|||
* @param tableDir The table directory.
|
||||
* @param family The name of the column family.
|
||||
* @param storeFiles The files to be deleted.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void removeMobFiles(Configuration conf, FileSystem fs, TableName tableName,
|
||||
Path tableDir, byte[] family, Collection<HStoreFile> storeFiles) throws IOException {
|
||||
|
@ -556,7 +550,6 @@ public final class MobUtils {
|
|||
* @param cryptoContext The encryption context.
|
||||
* @param isCompaction If the writer is used in compaction.
|
||||
* @return The writer for the mob file.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
|
||||
ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount,
|
||||
|
@ -580,7 +573,6 @@ public final class MobUtils {
|
|||
* @param cryptoContext The encryption context.
|
||||
* @param isCompaction If the writer is used in compaction.
|
||||
* @return The writer for the mob file.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static StoreFileWriter createRefFileWriter(Configuration conf, FileSystem fs,
|
||||
ColumnFamilyDescriptor family, Path basePath, long maxKeyCount, CacheConfig cacheConfig,
|
||||
|
@ -607,7 +599,6 @@ public final class MobUtils {
|
|||
* @param cryptoContext The encryption context.
|
||||
* @param isCompaction If the writer is used in compaction.
|
||||
* @return The writer for the mob file.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
|
||||
ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount,
|
||||
|
@ -633,7 +624,6 @@ public final class MobUtils {
|
|||
* @param cacheConfig The current cache config.
|
||||
* @param cryptoContext The encryption context.
|
||||
* @return The writer for the del file.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static StoreFileWriter createDelFileWriter(Configuration conf, FileSystem fs,
|
||||
ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount,
|
||||
|
@ -660,10 +650,9 @@ public final class MobUtils {
|
|||
* @param cryptoContext The encryption context.
|
||||
* @param isCompaction If the writer is used in compaction.
|
||||
* @return The writer for the mob file.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
|
||||
ColumnFamilyDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount,
|
||||
ColumnFamilyDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount,
|
||||
Compression.Algorithm compression, CacheConfig cacheConfig, Encryption.Context cryptoContext,
|
||||
boolean isCompaction)
|
||||
throws IOException {
|
||||
|
@ -689,7 +678,6 @@ public final class MobUtils {
|
|||
* @param bloomType The bloom filter type.
|
||||
* @param isCompaction If the writer is used in compaction.
|
||||
* @return The writer for the mob file.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static StoreFileWriter createWriter(Configuration conf, FileSystem fs,
|
||||
ColumnFamilyDescriptor family, Path path, long maxKeyCount,
|
||||
|
@ -718,8 +706,7 @@ public final class MobUtils {
|
|||
.withCreateTime(EnvironmentEdgeManager.currentTime()).build();
|
||||
|
||||
StoreFileWriter w = new StoreFileWriter.Builder(conf, writerCacheConf, fs)
|
||||
.withFilePath(path)
|
||||
.withComparator(CellComparator.getInstance()).withBloomType(bloomType)
|
||||
.withFilePath(path).withBloomType(bloomType)
|
||||
.withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build();
|
||||
return w;
|
||||
}
|
||||
|
@ -732,7 +719,6 @@ public final class MobUtils {
|
|||
* @param targetPath The directory path where the source file is renamed to.
|
||||
* @param cacheConfig The current cache config.
|
||||
* @return The target file path the source file is renamed to.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static Path commitFile(Configuration conf, FileSystem fs, final Path sourceFile,
|
||||
Path targetPath, CacheConfig cacheConfig) throws IOException {
|
||||
|
@ -837,7 +823,8 @@ public final class MobUtils {
|
|||
* @param allFiles Whether add all mob files into the compaction.
|
||||
*/
|
||||
public static void doMobCompaction(Configuration conf, FileSystem fs, TableName tableName,
|
||||
ColumnFamilyDescriptor hcd, ExecutorService pool, boolean allFiles, LockManager.MasterLock lock)
|
||||
ColumnFamilyDescriptor hcd, ExecutorService pool, boolean allFiles,
|
||||
LockManager.MasterLock lock)
|
||||
throws IOException {
|
||||
String className = conf.get(MobConstants.MOB_COMPACTOR_CLASS_KEY,
|
||||
PartitionedMobCompactor.class.getName());
|
||||
|
@ -943,7 +930,8 @@ public final class MobUtils {
|
|||
* @param fileDate The date string parsed from the mob file name.
|
||||
* @return True if the mob file is expired.
|
||||
*/
|
||||
public static boolean isMobFileExpired(ColumnFamilyDescriptor column, long current, String fileDate) {
|
||||
public static boolean isMobFileExpired(ColumnFamilyDescriptor column, long current,
|
||||
String fileDate) {
|
||||
if (column.getMinVersions() > 0) {
|
||||
return false;
|
||||
}
|
||||
|
|
|
@ -52,7 +52,6 @@ import java.util.function.Predicate;
|
|||
import java.util.function.ToLongFunction;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.LongStream;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
|
@ -132,7 +131,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDes
|
|||
* not be called directly but by an HRegion manager.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class HStore implements Store, HeapSize, StoreConfigInformation, PropagatingConfigurationObserver {
|
||||
public class HStore implements Store, HeapSize, StoreConfigInformation,
|
||||
PropagatingConfigurationObserver {
|
||||
public static final String MEMSTORE_CLASS_NAME = "hbase.regionserver.memstore.class";
|
||||
public static final String COMPACTCHECKER_INTERVAL_MULTIPLIER_KEY =
|
||||
"hbase.server.compactchecker.interval.multiplier";
|
||||
|
@ -237,11 +237,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
|
||||
/**
|
||||
* Constructor
|
||||
* @param region
|
||||
* @param family HColumnDescriptor for this column
|
||||
* @param confParam configuration object
|
||||
* failed. Can be null.
|
||||
* @throws IOException
|
||||
* @param confParam configuration object failed. Can be null.
|
||||
*/
|
||||
protected HStore(final HRegion region, final ColumnFamilyDescriptor family,
|
||||
final Configuration confParam, boolean warmup) throws IOException {
|
||||
|
@ -399,7 +396,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
}
|
||||
|
||||
/**
|
||||
* @param family
|
||||
* @return TTL in seconds of the specified family
|
||||
*/
|
||||
public static long determineTTLFromFamily(final ColumnFamilyDescriptor family) {
|
||||
|
@ -521,6 +517,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
* @param hri {@link RegionInfo} for the region.
|
||||
* @param family {@link ColumnFamilyDescriptor} describing the column family
|
||||
* @return Path to family/Store home directory.
|
||||
* @deprecated Since 05/05/2013, HBase-7808, hbase-1.0.0
|
||||
*/
|
||||
@Deprecated
|
||||
public static Path getStoreHomedir(final Path tabledir,
|
||||
|
@ -533,6 +530,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
* @param encodedName Encoded region name.
|
||||
* @param family {@link ColumnFamilyDescriptor} describing the column family
|
||||
* @return Path to family/Store home directory.
|
||||
* @deprecated Since 05/05/2013, HBase-7808, hbase-1.0.0
|
||||
*/
|
||||
@Deprecated
|
||||
public static Path getStoreHomedir(final Path tabledir,
|
||||
|
@ -558,7 +556,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
/**
|
||||
* Creates an unsorted list of StoreFile loaded in parallel
|
||||
* from the given directory.
|
||||
* @throws IOException
|
||||
*/
|
||||
private List<HStoreFile> loadStoreFiles(boolean warmup) throws IOException {
|
||||
Collection<StoreFileInfo> files = fs.getStoreFiles(getColumnFamilyName());
|
||||
|
@ -574,7 +571,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
ThreadPoolExecutor storeFileOpenerThreadPool =
|
||||
this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-"
|
||||
+ this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName());
|
||||
CompletionService<HStoreFile> completionService = new ExecutorCompletionService<>(storeFileOpenerThreadPool);
|
||||
CompletionService<HStoreFile> completionService =
|
||||
new ExecutorCompletionService<>(storeFileOpenerThreadPool);
|
||||
|
||||
int totalValidStoreFile = 0;
|
||||
for (StoreFileInfo storeFileInfo : files) {
|
||||
|
@ -596,9 +594,13 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
compactedStoreFiles.addAll(storeFile.getCompactedStoreFiles());
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
if (ioe == null) ioe = new InterruptedIOException(e.getMessage());
|
||||
if (ioe == null) {
|
||||
ioe = new InterruptedIOException(e.getMessage());
|
||||
}
|
||||
} catch (ExecutionException e) {
|
||||
if (ioe == null) ioe = new IOException(e.getCause());
|
||||
if (ioe == null) {
|
||||
ioe = new IOException(e.getCause());
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
|
@ -650,7 +652,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
/**
|
||||
* Replaces the store files that the store has with the given files. Mainly used by secondary
|
||||
* region replicas to keep up to date with the primary region files.
|
||||
* @throws IOException
|
||||
*/
|
||||
public void refreshStoreFiles(Collection<String> newFiles) throws IOException {
|
||||
List<StoreFileInfo> storeFiles = new ArrayList<>(newFiles.size());
|
||||
|
@ -665,15 +666,20 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
* been opened, and removes the store file readers for store files no longer
|
||||
* available. Mainly used by secondary region replicas to keep up to date with
|
||||
* the primary region files.
|
||||
* @throws IOException
|
||||
*/
|
||||
private void refreshStoreFilesInternal(Collection<StoreFileInfo> newFiles) throws IOException {
|
||||
StoreFileManager sfm = storeEngine.getStoreFileManager();
|
||||
Collection<HStoreFile> currentFiles = sfm.getStorefiles();
|
||||
Collection<HStoreFile> compactedFiles = sfm.getCompactedfiles();
|
||||
if (currentFiles == null) currentFiles = Collections.emptySet();
|
||||
if (newFiles == null) newFiles = Collections.emptySet();
|
||||
if (compactedFiles == null) compactedFiles = Collections.emptySet();
|
||||
if (currentFiles == null) {
|
||||
currentFiles = Collections.emptySet();
|
||||
}
|
||||
if (newFiles == null) {
|
||||
newFiles = Collections.emptySet();
|
||||
}
|
||||
if (compactedFiles == null) {
|
||||
compactedFiles = Collections.emptySet();
|
||||
}
|
||||
|
||||
HashMap<StoreFileInfo, HStoreFile> currentFilesSet = new HashMap<>(currentFiles.size());
|
||||
for (HStoreFile sf : currentFiles) {
|
||||
|
@ -868,12 +874,13 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
}
|
||||
prevCell = cell;
|
||||
} while (scanner.next());
|
||||
LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString()
|
||||
+ " took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime)
|
||||
+ " ms");
|
||||
LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString() +
|
||||
" took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime) + " ms");
|
||||
}
|
||||
} finally {
|
||||
if (reader != null) reader.close();
|
||||
if (reader != null) {
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -881,7 +888,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
* This method should only be called from Region. It is assumed that the ranges of values in the
|
||||
* HFile fit within the stores assigned region. (assertBulkLoadHFileOk checks this)
|
||||
*
|
||||
* @param srcPathStr
|
||||
* @param seqNum sequence Id associated with the HFile
|
||||
*/
|
||||
public Pair<Path, Path> preBulkLoadHFile(String srcPathStr, long seqNum) throws IOException {
|
||||
|
@ -993,13 +999,17 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
ioe.initCause(e);
|
||||
}
|
||||
} catch (ExecutionException e) {
|
||||
if (ioe == null) ioe = new IOException(e.getCause());
|
||||
if (ioe == null) {
|
||||
ioe = new IOException(e.getCause());
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
storeFileCloserThreadPool.shutdownNow();
|
||||
}
|
||||
if (ioe != null) throw ioe;
|
||||
if (ioe != null) {
|
||||
throw ioe;
|
||||
}
|
||||
}
|
||||
LOG.trace("Closed {}", this);
|
||||
return result;
|
||||
|
@ -1027,9 +1037,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
/**
|
||||
* Write out current snapshot. Presumes {@link #snapshot()} has been called previously.
|
||||
* @param logCacheFlushId flush sequence number
|
||||
* @param snapshot
|
||||
* @param status
|
||||
* @param throughputController
|
||||
* @return The path name of the tmp file to which the store was flushed
|
||||
* @throws IOException if exception occurs during process
|
||||
*/
|
||||
|
@ -1081,10 +1088,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
|
||||
/**
|
||||
* @param path The pathname of the tmp file into which the store was flushed
|
||||
* @param logCacheFlushId
|
||||
* @param status
|
||||
* @return store file created.
|
||||
* @throws IOException
|
||||
*/
|
||||
private HStoreFile commitFile(Path path, long logCacheFlushId, MonitoredTask status)
|
||||
throws IOException {
|
||||
|
@ -1107,7 +1111,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
}
|
||||
|
||||
/**
|
||||
* @param maxKeyCount
|
||||
* @param compression Compression algorithm to use
|
||||
* @param isCompaction whether we are creating a new file in a compaction
|
||||
* @param includeMVCCReadpoint - whether to include MVCC or not
|
||||
|
@ -1162,7 +1165,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf,
|
||||
this.getFileSystem())
|
||||
.withOutputDir(familyTempDir)
|
||||
.withComparator(comparator)
|
||||
.withBloomType(family.getBloomFilterType())
|
||||
.withMaxKeyCount(maxKeyCount)
|
||||
.withFavoredNodes(favoredNodes)
|
||||
|
@ -1192,6 +1194,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
.withColumnFamily(family.getName())
|
||||
.withTableName(region.getTableDescriptor()
|
||||
.getTableName().getName())
|
||||
.withCellComparator(this.comparator)
|
||||
.build();
|
||||
return hFileContext;
|
||||
}
|
||||
|
@ -1204,8 +1207,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
/**
|
||||
* Change storeFiles adding into place the Reader produced by this new flush.
|
||||
* @param sfs Store files
|
||||
* @param snapshotId
|
||||
* @throws IOException
|
||||
* @return Whether compaction is required.
|
||||
*/
|
||||
private boolean updateStorefiles(List<HStoreFile> sfs, long snapshotId) throws IOException {
|
||||
|
@ -1237,7 +1238,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
|
||||
/**
|
||||
* Notify all observers that set of Readers has changed.
|
||||
* @throws IOException
|
||||
*/
|
||||
private void notifyChangedReadersObservers(List<HStoreFile> sfs) throws IOException {
|
||||
for (ChangedReadersObserver o : this.changedReaderObservers) {
|
||||
|
@ -1454,7 +1454,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
* See HBASE-2231 for details.
|
||||
*
|
||||
* @param compaction compaction details obtained from requestCompaction()
|
||||
* @throws IOException
|
||||
* @return Storefile we compacted into or null if we failed or opted out early.
|
||||
*/
|
||||
public List<HStoreFile> compact(CompactionContext compaction,
|
||||
|
@ -1521,8 +1520,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
return sfs;
|
||||
}
|
||||
|
||||
private List<HStoreFile> moveCompactedFilesIntoPlace(CompactionRequestImpl cr, List<Path> newFiles,
|
||||
User user) throws IOException {
|
||||
private List<HStoreFile> moveCompactedFilesIntoPlace(CompactionRequestImpl cr,
|
||||
List<Path> newFiles, User user) throws IOException {
|
||||
List<HStoreFile> sfs = new ArrayList<>(newFiles.size());
|
||||
for (Path newFile : newFiles) {
|
||||
assert newFile != null;
|
||||
|
@ -1560,7 +1559,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
newFiles.stream().map(HStoreFile::getPath).collect(Collectors.toList());
|
||||
RegionInfo info = this.region.getRegionInfo();
|
||||
CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info,
|
||||
family.getName(), inputPaths, outputPaths, fs.getStoreDir(getColumnFamilyDescriptor().getNameAsString()));
|
||||
family.getName(), inputPaths, outputPaths,
|
||||
fs.getStoreDir(getColumnFamilyDescriptor().getNameAsString()));
|
||||
// Fix reaching into Region to get the maxWaitForSeqId.
|
||||
// Does this method belong in Region altogether given it is making so many references up there?
|
||||
// Could be Region#writeCompactionMarker(compactionDescriptor);
|
||||
|
@ -1664,7 +1664,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
* Call to complete a compaction. Its for the case where we find in the WAL a compaction
|
||||
* that was not finished. We could find one recovering a WAL after a regionserver crash.
|
||||
* See HBASE-2231.
|
||||
* @param compaction
|
||||
*/
|
||||
public void replayCompactionMarker(CompactionDescriptor compaction, boolean pickCompactionFiles,
|
||||
boolean removeFiles) throws IOException {
|
||||
|
@ -1917,7 +1916,9 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
}
|
||||
|
||||
private void removeUnneededFiles() throws IOException {
|
||||
if (!conf.getBoolean("hbase.store.delete.expired.storefile", true)) return;
|
||||
if (!conf.getBoolean("hbase.store.delete.expired.storefile", true)) {
|
||||
return;
|
||||
}
|
||||
if (getColumnFamilyDescriptor().getMinVersions() > 0) {
|
||||
LOG.debug("Skipping expired store file removal due to min version being {}",
|
||||
getColumnFamilyDescriptor().getMinVersions());
|
||||
|
@ -2120,7 +2121,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
* @param readPt the read point of the current scane
|
||||
* @param includeMemstoreScanner whether the current scanner should include memstorescanner
|
||||
* @return list of scanners recreated on the current Scanners
|
||||
* @throws IOException
|
||||
*/
|
||||
public List<KeyValueScanner> recreateScanners(List<KeyValueScanner> currentFileScanners,
|
||||
boolean cacheBlocks, boolean usePread, boolean isCompaction, ScanQueryMatcher matcher,
|
||||
|
@ -2319,7 +2319,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
* This operation is atomic on each KeyValue (row/family/qualifier) but not necessarily atomic
|
||||
* across all of them.
|
||||
* @param readpoint readpoint below which we can safely remove duplicate KVs
|
||||
* @throws IOException
|
||||
*/
|
||||
public void upsert(Iterable<Cell> cells, long readpoint, MemStoreSizing memstoreSizing)
|
||||
throws IOException {
|
||||
|
@ -2433,7 +2432,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
* snapshot depending on dropMemstoreSnapshot argument.
|
||||
* @param fileNames names of the flushed files
|
||||
* @param dropMemstoreSnapshot whether to drop the prepared memstore snapshot
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public void replayFlush(List<String> fileNames, boolean dropMemstoreSnapshot)
|
||||
|
@ -2465,7 +2463,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
|
||||
/**
|
||||
* Abort the snapshot preparation. Drops the snapshot if any.
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public void abort() throws IOException {
|
||||
|
@ -2720,7 +2717,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
|
|||
LOG.debug("Moving the files {} to archive", filesToRemove);
|
||||
// Only if this is successful it has to be removed
|
||||
try {
|
||||
this.fs.removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(), filesToRemove);
|
||||
this.fs.removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(),
|
||||
filesToRemove);
|
||||
} catch (FailedArchiveException fae) {
|
||||
// Even if archiving some files failed, we still need to clear out any of the
|
||||
// files which were successfully archived. Otherwise we will receive a
|
||||
|
|
|
@ -26,7 +26,6 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_K
|
|||
import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAX_SEQ_ID_KEY;
|
||||
import static org.apache.hadoop.hbase.regionserver.HStoreFile.MOB_CELLS_COUNT;
|
||||
import static org.apache.hadoop.hbase.regionserver.HStoreFile.TIMERANGE_KEY;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.util.Collection;
|
||||
|
@ -37,15 +36,13 @@ import java.util.UUID;
|
|||
import java.util.function.Supplier;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.PrivateCellUtil;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.PrivateCellUtil;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFile;
|
||||
|
@ -62,9 +59,7 @@ import org.apache.hadoop.hbase.util.RowPrefixFixedLengthBloomContext;
|
|||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
|
||||
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
||||
|
||||
/**
|
||||
|
@ -94,7 +89,6 @@ public class StoreFileWriter implements CellSink, ShipperListener {
|
|||
* @param fs file system to write to
|
||||
* @param path file name to create
|
||||
* @param conf user configuration
|
||||
* @param comparator key comparator
|
||||
* @param bloomType bloom filter setting
|
||||
* @param maxKeys the expected maximum number of keys to be added. Was used
|
||||
* for Bloom filter size in {@link HFile} format version 1.
|
||||
|
@ -105,15 +99,14 @@ public class StoreFileWriter implements CellSink, ShipperListener {
|
|||
* @throws IOException problem writing to FS
|
||||
*/
|
||||
private StoreFileWriter(FileSystem fs, Path path, final Configuration conf, CacheConfig cacheConf,
|
||||
final CellComparator comparator, BloomType bloomType, long maxKeys,
|
||||
InetSocketAddress[] favoredNodes, HFileContext fileContext, boolean shouldDropCacheBehind,
|
||||
Supplier<Collection<HStoreFile>> compactedFilesSupplier) throws IOException {
|
||||
BloomType bloomType, long maxKeys, InetSocketAddress[] favoredNodes, HFileContext fileContext,
|
||||
boolean shouldDropCacheBehind, Supplier<Collection<HStoreFile>> compactedFilesSupplier)
|
||||
throws IOException {
|
||||
this.compactedFilesSupplier = compactedFilesSupplier;
|
||||
this.timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC);
|
||||
// TODO : Change all writers to be specifically created for compaction context
|
||||
writer = HFile.getWriterFactory(conf, cacheConf)
|
||||
.withPath(fs, path)
|
||||
.withComparator(comparator)
|
||||
.withFavoredNodes(favoredNodes)
|
||||
.withFileContext(fileContext)
|
||||
.withShouldDropCacheBehind(shouldDropCacheBehind)
|
||||
|
@ -135,14 +128,16 @@ public class StoreFileWriter implements CellSink, ShipperListener {
|
|||
// init bloom context
|
||||
switch (bloomType) {
|
||||
case ROW:
|
||||
bloomContext = new RowBloomContext(generalBloomFilterWriter, comparator);
|
||||
bloomContext =
|
||||
new RowBloomContext(generalBloomFilterWriter, fileContext.getCellComparator());
|
||||
break;
|
||||
case ROWCOL:
|
||||
bloomContext = new RowColBloomContext(generalBloomFilterWriter, comparator);
|
||||
bloomContext =
|
||||
new RowColBloomContext(generalBloomFilterWriter, fileContext.getCellComparator());
|
||||
break;
|
||||
case ROWPREFIX_FIXED_LENGTH:
|
||||
bloomContext = new RowPrefixFixedLengthBloomContext(generalBloomFilterWriter, comparator,
|
||||
Bytes.toInt(bloomParam));
|
||||
bloomContext = new RowPrefixFixedLengthBloomContext(generalBloomFilterWriter,
|
||||
fileContext.getCellComparator(), Bytes.toInt(bloomParam));
|
||||
break;
|
||||
default:
|
||||
throw new IOException(
|
||||
|
@ -159,7 +154,8 @@ public class StoreFileWriter implements CellSink, ShipperListener {
|
|||
this.deleteFamilyBloomFilterWriter = BloomFilterFactory
|
||||
.createDeleteBloomAtWrite(conf, cacheConf,
|
||||
(int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
|
||||
deleteFamilyBloomContext = new RowBloomContext(deleteFamilyBloomFilterWriter, comparator);
|
||||
deleteFamilyBloomContext =
|
||||
new RowBloomContext(deleteFamilyBloomFilterWriter, fileContext.getCellComparator());
|
||||
} else {
|
||||
deleteFamilyBloomFilterWriter = null;
|
||||
}
|
||||
|
@ -396,7 +392,6 @@ public class StoreFileWriter implements CellSink, ShipperListener {
|
|||
}
|
||||
|
||||
/**
|
||||
* @param fs
|
||||
* @param dir Directory to create file in.
|
||||
* @return random filename inside passed <code>dir</code>
|
||||
*/
|
||||
|
@ -414,7 +409,6 @@ public class StoreFileWriter implements CellSink, ShipperListener {
|
|||
private final CacheConfig cacheConf;
|
||||
private final FileSystem fs;
|
||||
|
||||
private CellComparator comparator = CellComparator.getInstance();
|
||||
private BloomType bloomType = BloomType.NONE;
|
||||
private long maxKeyCount = 0;
|
||||
private Path dir;
|
||||
|
@ -473,12 +467,6 @@ public class StoreFileWriter implements CellSink, ShipperListener {
|
|||
return this;
|
||||
}
|
||||
|
||||
public Builder withComparator(CellComparator comparator) {
|
||||
Preconditions.checkNotNull(comparator);
|
||||
this.comparator = comparator;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withBloomType(BloomType bloomType) {
|
||||
Preconditions.checkNotNull(bloomType);
|
||||
this.bloomType = bloomType;
|
||||
|
@ -545,11 +533,7 @@ public class StoreFileWriter implements CellSink, ShipperListener {
|
|||
}
|
||||
}
|
||||
|
||||
if (comparator == null) {
|
||||
comparator = CellComparator.getInstance();
|
||||
}
|
||||
|
||||
return new StoreFileWriter(fs, filePath, conf, cacheConf, comparator, bloomType, maxKeyCount,
|
||||
return new StoreFileWriter(fs, filePath, conf, cacheConf, bloomType, maxKeyCount,
|
||||
favoredNodes, fileContext, shouldDropCacheBehind, compactedFilesSupplier);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
package org.apache.hadoop.hbase.wal;
|
||||
|
||||
import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InterruptedIOException;
|
||||
import java.util.HashMap;
|
||||
|
@ -29,7 +28,6 @@ import java.util.concurrent.ConcurrentMap;
|
|||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
|
@ -208,7 +206,7 @@ public class BoundedRecoveredHFilesOutputSink extends OutputSink {
|
|||
.withFilePath(outputFile);
|
||||
HFileContextBuilder hFileContextBuilder = new HFileContextBuilder();
|
||||
if (isMetaTable) {
|
||||
writerBuilder.withComparator(CellComparatorImpl.META_COMPARATOR);
|
||||
hFileContextBuilder.withCellComparator(CellComparatorImpl.META_COMPARATOR);
|
||||
} else {
|
||||
configContextForNonMetaWriter(tableName, familyName, hFileContextBuilder, writerBuilder);
|
||||
}
|
||||
|
@ -224,9 +222,9 @@ public class BoundedRecoveredHFilesOutputSink extends OutputSink {
|
|||
TableDescriptor tableDesc = tableDescCache.get(tableName);
|
||||
ColumnFamilyDescriptor cfd = tableDesc.getColumnFamily(Bytes.toBytesBinary(familyName));
|
||||
hFileContextBuilder.withCompression(cfd.getCompressionType()).withBlockSize(cfd.getBlocksize())
|
||||
.withCompressTags(cfd.isCompressTags()).withDataBlockEncoding(cfd.getDataBlockEncoding());
|
||||
writerBuilder.withBloomType(cfd.getBloomFilterType())
|
||||
.withComparator(CellComparatorImpl.COMPARATOR);
|
||||
.withCompressTags(cfd.isCompressTags()).withDataBlockEncoding(cfd.getDataBlockEncoding())
|
||||
.withCellComparator(CellComparatorImpl.COMPARATOR);
|
||||
writerBuilder.withBloomType(cfd.getBloomFilterType());
|
||||
}
|
||||
|
||||
private void checkPathValid(Path outputFile) throws IOException {
|
||||
|
|
|
@ -380,7 +380,6 @@ public class HFilePerformanceEvaluation {
|
|||
writer = HFile.getWriterFactoryNoCache(conf)
|
||||
.withPath(fs, mf)
|
||||
.withFileContext(hFileContext)
|
||||
.withComparator(CellComparator.getInstance())
|
||||
.create();
|
||||
}
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
/**
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
|
@ -206,8 +206,8 @@ public class TestDataBlockEncoders {
|
|||
.withIncludesTags(includesTags)
|
||||
.withCompression(Compression.Algorithm.NONE)
|
||||
.build();
|
||||
DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR,
|
||||
encoder.newDataBlockDecodingContext(meta));
|
||||
DataBlockEncoder.EncodedSeeker seeker =
|
||||
encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
|
||||
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
|
||||
encodedSeekers.add(seeker);
|
||||
}
|
||||
|
@ -279,8 +279,8 @@ public class TestDataBlockEncoders {
|
|||
.withIncludesTags(includesTags)
|
||||
.withCompression(Compression.Algorithm.NONE)
|
||||
.build();
|
||||
DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR,
|
||||
encoder.newDataBlockDecodingContext(meta));
|
||||
DataBlockEncoder.EncodedSeeker seeker =
|
||||
encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
|
||||
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
|
||||
int i = 0;
|
||||
do {
|
||||
|
@ -341,8 +341,8 @@ public class TestDataBlockEncoders {
|
|||
HFileContext meta =
|
||||
new HFileContextBuilder().withHBaseCheckSum(false).withIncludesMvcc(includesMemstoreTS)
|
||||
.withIncludesTags(includesTags).withCompression(Compression.Algorithm.NONE).build();
|
||||
DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR,
|
||||
encoder.newDataBlockDecodingContext(meta));
|
||||
DataBlockEncoder.EncodedSeeker seeker =
|
||||
encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
|
||||
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
|
||||
Cell cell = seeker.getCell();
|
||||
Assert.assertEquals(expectedKV.getLength(), ((KeyValue) cell).getLength());
|
||||
|
|
|
@ -25,7 +25,6 @@ import java.util.ArrayList;
|
|||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
|
@ -290,8 +289,8 @@ public class TestSeekToBlockWithEncoders {
|
|||
HFILEBLOCK_DUMMY_HEADER, meta);
|
||||
ByteBuffer encodedBuffer = TestDataBlockEncoders.encodeKeyValues(encoding, kvs,
|
||||
encodingContext, this.useOffheapData);
|
||||
DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR,
|
||||
encoder.newDataBlockDecodingContext(meta));
|
||||
DataBlockEncoder.EncodedSeeker seeker =
|
||||
encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
|
||||
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
|
||||
encodedSeekers.add(seeker);
|
||||
}
|
||||
|
|
|
@ -37,7 +37,6 @@ import org.apache.hadoop.conf.Configuration;
|
|||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.ArrayBackedTag;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
|
@ -394,7 +393,7 @@ public class TestCacheOnWrite {
|
|||
.withDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding())
|
||||
.withIncludesTags(useTags).build();
|
||||
StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs)
|
||||
.withOutputDir(storeFileParentDir).withComparator(CellComparatorImpl.COMPARATOR)
|
||||
.withOutputDir(storeFileParentDir)
|
||||
.withFileContext(meta)
|
||||
.withBloomType(BLOOM_TYPE).withMaxKeyCount(NUM_KV).build();
|
||||
byte[] cf = Bytes.toBytes("fam");
|
||||
|
|
|
@ -131,6 +131,10 @@ public class TestFixedFileTrailer {
|
|||
t.createComparator(KeyValue.META_COMPARATOR.getClass().getName()).getClass());
|
||||
assertEquals(CellComparatorImpl.MetaCellComparator.class, t.createComparator(
|
||||
CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
|
||||
assertEquals(CellComparatorImpl.META_COMPARATOR.getClass(), t.createComparator(
|
||||
CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass());
|
||||
assertEquals(CellComparatorImpl.COMPARATOR.getClass(), t.createComparator(
|
||||
CellComparatorImpl.MetaCellComparator.COMPARATOR.getClass().getName()).getClass());
|
||||
assertNull(t.createComparator(Bytes.BYTES_RAWCOMPARATOR.getClass().getName()));
|
||||
assertNull(t.createComparator("org.apache.hadoop.hbase.KeyValue$RawBytesComparator"));
|
||||
} catch (IOException e) {
|
||||
|
|
|
@ -50,7 +50,6 @@ import org.apache.hadoop.hbase.Cell;
|
|||
import org.apache.hadoop.hbase.CellBuilder;
|
||||
import org.apache.hadoop.hbase.CellBuilderFactory;
|
||||
import org.apache.hadoop.hbase.CellBuilderType;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.CellUtil;
|
||||
import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
|
||||
|
@ -277,7 +276,7 @@ public class TestHFile {
|
|||
HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
|
||||
StoreFileWriter sfw =
|
||||
new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir)
|
||||
.withComparator(CellComparatorImpl.COMPARATOR).withFileContext(meta).build();
|
||||
.withFileContext(meta).build();
|
||||
|
||||
final int rowLen = 32;
|
||||
Random RNG = new Random();
|
||||
|
@ -360,7 +359,7 @@ public class TestHFile {
|
|||
.withCompressTags(false)
|
||||
.build();
|
||||
HFileWriterImpl writer = new HFileWriterImpl(conf, cacheConf, path, mockedOutputStream,
|
||||
CellComparator.getInstance(), fileContext);
|
||||
fileContext);
|
||||
CellBuilder cellBuilder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY);
|
||||
byte[] row = Bytes.toBytes("foo");
|
||||
byte[] qualifier = Bytes.toBytes("qualifier");
|
||||
|
@ -516,7 +515,6 @@ public class TestHFile {
|
|||
Writer writer = HFile.getWriterFactory(conf, cacheConf)
|
||||
.withOutputStream(fout)
|
||||
.withFileContext(meta)
|
||||
.withComparator(CellComparatorImpl.COMPARATOR)
|
||||
.create();
|
||||
LOG.info(Objects.toString(writer));
|
||||
writeRecords(writer, useTags);
|
||||
|
|
|
@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
|
|||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
|
@ -66,9 +65,8 @@ public class TestHFileReaderImpl {
|
|||
HFileContext context =
|
||||
new HFileContextBuilder().withBlockSize(blocksize).withIncludesTags(true).build();
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
HFile.Writer writer =
|
||||
HFile.getWriterFactoryNoCache(conf).withOutputStream(fout).withFileContext(context)
|
||||
.withComparator(CellComparatorImpl.COMPARATOR).create();
|
||||
HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
|
||||
.withOutputStream(fout).withFileContext(context).create();
|
||||
// 4 bytes * 3 * 2 for each key/value +
|
||||
// 3 for keys, 15 for values = 42 (woot)
|
||||
writer.append(toKV("c"));
|
||||
|
|
|
@ -37,7 +37,6 @@ import org.apache.hadoop.conf.Configuration;
|
|||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
|
@ -165,7 +164,7 @@ public class TestHFileScannerImplReferenceCount {
|
|||
.withCompression(compression).withDataBlockEncoding(encoding).build();
|
||||
try (HFile.Writer writer =
|
||||
new HFile.WriterFactory(conf, new CacheConfig(conf)).withPath(fs, hfilePath)
|
||||
.withFileContext(context).withComparator(CellComparatorImpl.COMPARATOR).create()) {
|
||||
.withFileContext(context).create()) {
|
||||
Random rand = new Random(9713312); // Just a fixed seed.
|
||||
for (int i = 0; i < cellCount; ++i) {
|
||||
byte[] keyBytes = Bytes.add(Bytes.toBytes(i), SUFFIX);
|
||||
|
|
|
@ -27,7 +27,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
|
|||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.RawLocalFileSystem;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
|
@ -142,7 +141,6 @@ public class TestHFileSeek extends TestCase {
|
|||
Writer writer = HFile.getWriterFactoryNoCache(conf)
|
||||
.withOutputStream(fout)
|
||||
.withFileContext(context)
|
||||
.withComparator(CellComparatorImpl.COMPARATOR)
|
||||
.create();
|
||||
try {
|
||||
BytesWritable key = new BytesWritable();
|
||||
|
|
|
@ -36,7 +36,6 @@ import org.apache.hadoop.fs.Path;
|
|||
import org.apache.hadoop.hbase.ArrayBackedTag;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
|
@ -132,7 +131,6 @@ public class TestHFileWriterV3 {
|
|||
HFile.Writer writer = new HFile.WriterFactory(conf, cacheConfig)
|
||||
.withPath(fs, hfilePath)
|
||||
.withFileContext(context)
|
||||
.withComparator(CellComparatorImpl.COMPARATOR)
|
||||
.create();
|
||||
|
||||
Random rand = new Random(9713312); // Just a fixed seed.
|
||||
|
|
|
@ -26,7 +26,6 @@ import java.util.Random;
|
|||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
|
@ -145,7 +144,6 @@ public class TestPrefetch {
|
|||
.build();
|
||||
StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs)
|
||||
.withOutputDir(storeFileParentDir)
|
||||
.withComparator(CellComparatorImpl.COMPARATOR)
|
||||
.withFileContext(meta)
|
||||
.build();
|
||||
|
||||
|
|
|
@ -25,7 +25,6 @@ import java.util.List;
|
|||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.ArrayBackedTag;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
|
@ -69,8 +68,6 @@ public class TestReseekTo {
|
|||
TEST_UTIL.getConfiguration(), cacheConf)
|
||||
.withOutputStream(fout)
|
||||
.withFileContext(context)
|
||||
// NOTE: This test is dependent on this deprecated nonstandard comparator
|
||||
.withComparator(CellComparatorImpl.COMPARATOR)
|
||||
.create();
|
||||
int numberOfKeys = 1000;
|
||||
|
||||
|
|
|
@ -33,7 +33,6 @@ import org.apache.hadoop.fs.Path;
|
|||
import org.apache.hadoop.hbase.ArrayBackedTag;
|
||||
import org.apache.hadoop.hbase.ByteBufferKeyValue;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
|
@ -123,8 +122,7 @@ public class TestSeekTo {
|
|||
.withIncludesTags(true).build();
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout)
|
||||
.withFileContext(context)
|
||||
.withComparator(CellComparatorImpl.COMPARATOR).create();
|
||||
.withFileContext(context).create();
|
||||
// 4 bytes * 3 * 2 for each key/value +
|
||||
// 3 for keys, 15 for values = 42 (woot)
|
||||
writer.append(toKV("c", tagUsage));
|
||||
|
|
|
@ -23,7 +23,7 @@ import org.apache.hadoop.hbase.client.Admin;
|
|||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileContext;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
|
||||
import org.apache.hadoop.hbase.master.MasterFileSystem;
|
||||
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
|
||||
import org.apache.hadoop.hbase.testclassification.MasterTests;
|
||||
|
@ -84,7 +84,7 @@ public class TestIgnoreUnknownFamily {
|
|||
Path familyDir = new Path(regionDir, Bytes.toString(UNKNOWN_FAMILY));
|
||||
StoreFileWriter writer =
|
||||
new StoreFileWriter.Builder(mfs.getConfiguration(), mfs.getFileSystem())
|
||||
.withOutputDir(familyDir).withFileContext(new HFileContext()).build();
|
||||
.withOutputDir(familyDir).withFileContext(new HFileContextBuilder().build()).build();
|
||||
writer.close();
|
||||
}
|
||||
|
||||
|
|
|
@ -52,7 +52,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
|
|||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFile;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileContext;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
|
||||
import org.apache.hadoop.hbase.testclassification.SmallTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.Pair;
|
||||
|
@ -281,7 +281,7 @@ public class TestBulkLoad {
|
|||
FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation), null);
|
||||
try {
|
||||
hFileFactory.withOutputStream(out);
|
||||
hFileFactory.withFileContext(new HFileContext());
|
||||
hFileFactory.withFileContext(new HFileContextBuilder().build());
|
||||
HFile.Writer writer = hFileFactory.create();
|
||||
try {
|
||||
writer.append(new KeyValue(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)
|
||||
|
|
|
@ -335,7 +335,7 @@ public class TestBulkLoadReplication extends TestReplicationBase {
|
|||
new FSDataOutputStream(new FileOutputStream(hFileLocation), null);
|
||||
try {
|
||||
hFileFactory.withOutputStream(out);
|
||||
hFileFactory.withFileContext(new HFileContext());
|
||||
hFileFactory.withFileContext(new HFileContextBuilder().build());
|
||||
HFile.Writer writer = hFileFactory.create();
|
||||
try {
|
||||
writer.append(new KeyValue(cellBuilder.build()));
|
||||
|
|
|
@ -24,9 +24,7 @@ import java.io.IOException;
|
|||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellUtil;
|
||||
|
@ -41,6 +39,8 @@ import org.apache.hadoop.hbase.TableName;
|
|||
import org.apache.hadoop.hbase.client.Delete;
|
||||
import org.apache.hadoop.hbase.client.Durability;
|
||||
import org.apache.hadoop.hbase.client.Put;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
|
||||
import org.apache.hadoop.hbase.client.Result;
|
||||
import org.apache.hadoop.hbase.client.Scan;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||
|
@ -61,7 +61,8 @@ import org.slf4j.LoggerFactory;
|
|||
|
||||
/**
|
||||
* TestGet is a medley of tests of get all done up as a single test.
|
||||
* This class
|
||||
* It was originally written to test a method since removed, getClosestAtOrBefore
|
||||
* but the test is retained because it runs some interesting exercises.
|
||||
*/
|
||||
@Category({RegionServerTests.class, MediumTests.class})
|
||||
public class TestGetClosestAtOrBefore {
|
||||
|
@ -88,7 +89,6 @@ public class TestGetClosestAtOrBefore {
|
|||
|
||||
@Test
|
||||
public void testUsingMetaAndBinary() throws IOException {
|
||||
FileSystem filesystem = FileSystem.get(conf);
|
||||
Path rootdir = UTIL.getDataTestDirOnTestFS();
|
||||
// Up flush size else we bind up when we use default catalog flush of 16k.
|
||||
TableDescriptors tds = new FSTableDescriptors(UTIL.getConfiguration());
|
||||
|
@ -103,13 +103,14 @@ public class TestGetClosestAtOrBefore {
|
|||
final int last = 128;
|
||||
final int interval = 2;
|
||||
for (int i = 0; i <= last; i += interval) {
|
||||
HRegionInfo hri = new HRegionInfo(htd.getTableName(),
|
||||
i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i),
|
||||
i == last ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i + interval));
|
||||
|
||||
RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
|
||||
.setStartKey(i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte)i))
|
||||
.setEndKey(i == last ? HConstants.EMPTY_BYTE_ARRAY :
|
||||
Bytes.toBytes((byte)i + interval)).build();
|
||||
Put put =
|
||||
MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
|
||||
put.setDurability(Durability.SKIP_WAL);
|
||||
LOG.info("Put {}", put);
|
||||
mr.put(put);
|
||||
}
|
||||
}
|
||||
|
@ -117,7 +118,7 @@ public class TestGetClosestAtOrBefore {
|
|||
try {
|
||||
List<Cell> keys = new ArrayList<>();
|
||||
while (s.next(keys)) {
|
||||
LOG.info(Objects.toString(keys));
|
||||
LOG.info("Scan {}", keys);
|
||||
keys.clear();
|
||||
}
|
||||
} finally {
|
||||
|
@ -133,13 +134,14 @@ public class TestGetClosestAtOrBefore {
|
|||
findRow(mr, 'C', 46, 46);
|
||||
findRow(mr, 'C', 43, 42);
|
||||
// Now delete 'C' and make sure I don't get entries from 'B'.
|
||||
byte[] firstRowInC = HRegionInfo.createRegionName(TableName.valueOf("" + 'C'),
|
||||
byte[] firstRowInC = RegionInfo.createRegionName(TableName.valueOf("" + 'C'),
|
||||
HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false);
|
||||
Scan scan = new Scan(firstRowInC);
|
||||
Scan scan = new Scan().withStartRow(firstRowInC);
|
||||
s = mr.getScanner(scan);
|
||||
try {
|
||||
List<Cell> keys = new ArrayList<>();
|
||||
while (s.next(keys)) {
|
||||
LOG.info("Delete {}", keys);
|
||||
mr.delete(new Delete(CellUtil.cloneRow(keys.get(0))));
|
||||
keys.clear();
|
||||
}
|
||||
|
|
|
@ -63,7 +63,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
|
|||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.executor.ExecutorService;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFile;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileContext;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegion.FlushResultImpl;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult;
|
||||
import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
|
||||
|
@ -1657,7 +1657,7 @@ public class TestHRegionReplayEvents {
|
|||
FSDataOutputStream out = TEST_UTIL.getTestFileSystem().create(testFile);
|
||||
try {
|
||||
hFileFactory.withOutputStream(out);
|
||||
hFileFactory.withFileContext(new HFileContext());
|
||||
hFileFactory.withFileContext(new HFileContextBuilder().build());
|
||||
HFile.Writer writer = hFileFactory.create();
|
||||
try {
|
||||
writer.append(new KeyValue(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)
|
||||
|
|
|
@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.client.Table;
|
|||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFile;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileContext;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
|
||||
import org.apache.hadoop.hbase.tool.BulkLoadHFiles;
|
||||
|
@ -161,7 +162,7 @@ public class TestScannerWithBulkload {
|
|||
Path path = new Path(pathStr);
|
||||
HFile.WriterFactory wf = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration());
|
||||
Assert.assertNotNull(wf);
|
||||
HFileContext context = new HFileContext();
|
||||
HFileContext context = new HFileContextBuilder().build();
|
||||
HFile.Writer writer = wf.withPath(fs, path).withFileContext(context).create();
|
||||
KeyValue kv = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l,
|
||||
Bytes.toBytes("version2"));
|
||||
|
|
|
@ -35,7 +35,6 @@ import org.apache.hadoop.conf.Configuration;
|
|||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.CellComparator;
|
||||
import org.apache.hadoop.hbase.CellComparatorImpl;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
|
@ -184,7 +183,7 @@ public class TestStoreScannerClosure {
|
|||
Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile");
|
||||
HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build();
|
||||
StoreFileWriter sfw = new StoreFileWriter.Builder(CONF, fs).withOutputDir(storeFileParentDir)
|
||||
.withComparator(CellComparatorImpl.COMPARATOR).withFileContext(meta).build();
|
||||
.withFileContext(meta).build();
|
||||
|
||||
final int rowLen = 32;
|
||||
Random RNG = new Random();
|
||||
|
|
Loading…
Reference in New Issue