From 6cdc4b1f0574cc424480ec2f2e3129f280fa8472 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Wed, 22 Jan 2020 20:00:21 -0800 Subject: [PATCH] HBASE-23705 Add CellComparator to HFileContext (#1062) Codecs don't have access to what CellComparator to use. Backfill. M hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java Adds a new compareRows with default implementation that takes a ByteBuffer. Needed by the index in a block encoder implementation. M hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java Adds implementation for meta of new compareRows method. Adds utility method for figuring comparator based off tablename. M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java Comparator is in context. Remove redundant handling. M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java Comparator is in context. Remove redundant handling. Clean javadoc. M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java Clean javadoc. M hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java Cache context so can use it to get comparator to use later. M hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java Cache cellcomparator to use. Javdoc on diff between HFileContext and HFileInfo. M hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java Add CellComparator M hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java Remove comparator caching. Get from context instead. M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java Skip a reflection if we can. M hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java Javadoc. Removed unused filed. Signed-off-by: Anoop Sam John Signed-off-by: Ramkrishna Signed-off-by: Jan Hentschel --- .../apache/hadoop/hbase/CellComparator.java | 24 ++- .../hadoop/hbase/CellComparatorImpl.java | 43 ++++- .../org/apache/hadoop/hbase/TableName.java | 3 + .../io/encoding/AbstractDataBlockEncoder.java | 15 +- .../io/encoding/BufferedDataBlockEncoder.java | 15 +- .../io/encoding/CopyKeyDataBlockEncoder.java | 10 +- .../hbase/io/encoding/DataBlockEncoder.java | 19 +- .../io/encoding/DiffKeyDeltaEncoder.java | 7 +- .../io/encoding/FastDiffDeltaEncoder.java | 7 +- .../encoding/HFileBlockDecodingContext.java | 5 +- .../HFileBlockDefaultDecodingContext.java | 6 +- .../HFileBlockDefaultEncodingContext.java | 13 +- .../encoding/HFileBlockEncodingContext.java | 5 - .../io/encoding/PrefixKeyDeltaEncoder.java | 7 +- .../hbase/io/encoding/RowIndexCodecV1.java | 15 +- .../hbase/io/encoding/RowIndexEncoderV1.java | 6 +- .../hbase/io/encoding/RowIndexSeekerV1.java | 30 +-- .../hadoop/hbase/io/hfile/HFileContext.java | 31 +++- .../hbase/io/hfile/HFileContextBuilder.java | 10 +- .../hbase/mapreduce/HFileOutputFormat2.java | 171 +++++++----------- .../hbase/io/hfile/FixedFileTrailer.java | 167 +++++++++-------- .../apache/hadoop/hbase/io/hfile/HFile.java | 62 +++---- .../hadoop/hbase/io/hfile/HFileBlock.java | 60 +++--- .../hadoop/hbase/io/hfile/HFileInfo.java | 16 +- .../hbase/io/hfile/HFileReaderImpl.java | 53 ++---- .../hbase/io/hfile/HFileWriterImpl.java | 70 +++---- .../org/apache/hadoop/hbase/mob/MobUtils.java | 26 +-- .../hadoop/hbase/regionserver/HStore.java | 88 +++++---- .../hbase/regionserver/StoreFileWriter.java | 42 ++--- .../wal/BoundedRecoveredHFilesOutputSink.java | 10 +- .../hbase/HFilePerformanceEvaluation.java | 1 - .../io/encoding/TestDataBlockEncoders.java | 14 +- .../encoding/TestSeekToBlockWithEncoders.java | 5 +- .../hbase/io/hfile/TestCacheOnWrite.java | 3 +- .../hbase/io/hfile/TestFixedFileTrailer.java | 4 + .../hadoop/hbase/io/hfile/TestHFile.java | 6 +- .../hbase/io/hfile/TestHFileReaderImpl.java | 6 +- .../TestHFileScannerImplReferenceCount.java | 3 +- .../hadoop/hbase/io/hfile/TestHFileSeek.java | 2 - .../hbase/io/hfile/TestHFileWriterV3.java | 2 - .../hadoop/hbase/io/hfile/TestPrefetch.java | 2 - .../hadoop/hbase/io/hfile/TestReseekTo.java | 3 - .../hadoop/hbase/io/hfile/TestSeekTo.java | 4 +- .../procedure/TestIgnoreUnknownFamily.java | 4 +- .../hbase/regionserver/TestBulkLoad.java | 4 +- .../regionserver/TestBulkLoadReplication.java | 2 +- .../TestGetClosestAtOrBefore.java | 24 +-- .../regionserver/TestHRegionReplayEvents.java | 4 +- .../regionserver/TestScannerWithBulkload.java | 3 +- .../regionserver/TestStoreScannerClosure.java | 3 +- 50 files changed, 511 insertions(+), 624 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java index 83a868d9caf..474f77215a1 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java @@ -17,8 +17,9 @@ */ package org.apache.hadoop.hbase; +import java.nio.ByteBuffer; import java.util.Comparator; - +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -31,11 +32,12 @@ import org.apache.yetus.audience.InterfaceStability; public interface CellComparator extends Comparator { /** * A comparator for ordering cells in user-space tables. Useful when writing cells in sorted - * order as necessary for bulk import (i.e. via MapReduce) + * order as necessary for bulk import (i.e. via MapReduce). *

* CAUTION: This comparator may provide inaccurate ordering for cells from system tables, * and should not be relied upon in that case. */ + // For internal use, see CellComparatorImpl utility methods. static CellComparator getInstance() { return CellComparatorImpl.COMPARATOR; } @@ -80,6 +82,24 @@ public interface CellComparator extends Comparator { */ int compareRows(Cell cell, byte[] bytes, int offset, int length); + /** + * @param row ByteBuffer that wraps a row; will read from current position and will reading all + * remaining; will not disturb the ByteBuffer internal state. + * @return greater than 0 if leftCell is bigger, less than 0 if rightCell is bigger, 0 if both + * cells are equal + */ + default int compareRows(ByteBuffer row, Cell cell) { + if (cell instanceof ByteBufferExtendedCell) { + return ByteBufferUtils.compareTo(row, row.position(), row.remaining(), + ((ByteBufferExtendedCell) cell).getRowByteBuffer(), + ((ByteBufferExtendedCell) cell).getRowPosition(), + cell.getRowLength()); + } + return ByteBufferUtils.compareTo(row, row.position(), row.remaining(), + cell.getRowArray(), cell.getRowOffset(), + cell.getRowLength()); + } + /** * Lexographically compares the two cells excluding the row part. It compares family, qualifier, * timestamp and the type diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java index c647318716f..e6c8e3ded62 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hbase; +import java.nio.ByteBuffer; import java.util.Comparator; - import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; @@ -26,7 +26,6 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hbase.thirdparty.com.google.common.primitives.Longs; /** @@ -377,6 +376,26 @@ public class CellComparatorImpl implements CellComparator { return result; } + @Override + public int compareRows(ByteBuffer row, Cell cell) { + byte [] array; + int offset; + int len = row.remaining(); + if (row.hasArray()) { + array = row.array(); + offset = row.position() + row.arrayOffset(); + } else { + // We copy the row array if offheap just so we can do a compare. We do this elsewhere too + // in BBUtils when Cell is backed by an offheap ByteBuffer. Needs fixing so no copy. TODO. + array = new byte[len]; + offset = 0; + ByteBufferUtils.copyFromBufferToArray(array, row, row.position(), + 0, len); + } + // Reverse result since we swap the order of the params we pass below. + return -compareRows(cell, array, offset, len); + } + @Override public Comparator getSimpleComparator() { return this; @@ -387,4 +406,24 @@ public class CellComparatorImpl implements CellComparator { public Comparator getSimpleComparator() { return new BBKVComparator(this); } + + /** + * Utility method that makes a guess at comparator to use based off passed tableName. + * Use in extreme when no comparator specified. + * @return CellComparator to use going off the {@code tableName} passed. + */ + public static CellComparator getCellComparator(TableName tableName) { + return getCellComparator(tableName.toBytes()); + } + + /** + * Utility method that makes a guess at comparator to use based off passed tableName. + * Use in extreme when no comparator specified. + * @return CellComparator to use going off the {@code tableName} passed. + */ + public static CellComparator getCellComparator(byte [] tableName) { + // FYI, TableName.toBytes does not create an array; just returns existing array pointer. + return Bytes.equals(tableName, TableName.META_TABLE_NAME.toBytes())? + CellComparatorImpl.META_COMPARATOR: CellComparatorImpl.COMPARATOR; + } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java index 51c87578715..f096ef9a7ab 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/TableName.java @@ -292,6 +292,9 @@ public final class TableName implements Comparable { return qualifierAsString; } + /** + * @return A pointer to TableName as String bytes. + */ public byte[] toBytes() { return name; } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java index ab95717e3e1..e96b800e7ef 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/AbstractDataBlockEncoder.java @@ -18,10 +18,8 @@ package org.apache.hadoop.hbase.io.encoding; import java.io.IOException; import java.nio.ByteBuffer; - import org.apache.hadoop.hbase.ByteBufferKeyOnlyKeyValue; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.io.hfile.HFileContext; @@ -59,14 +57,13 @@ public abstract class AbstractDataBlockEncoder implements DataBlockEncoder { } } - protected abstract static class AbstractEncodedSeeker implements - EncodedSeeker { + /** + * Decorates EncodedSeeker with a {@link HFileBlockDecodingContext} + */ + protected abstract static class AbstractEncodedSeeker implements EncodedSeeker { protected HFileBlockDecodingContext decodingCtx; - protected final CellComparator comparator; - public AbstractEncodedSeeker(CellComparator comparator, - HFileBlockDecodingContext decodingCtx) { - this.comparator = comparator; + public AbstractEncodedSeeker(HFileBlockDecodingContext decodingCtx) { this.decodingCtx = decodingCtx; } @@ -77,7 +74,5 @@ public abstract class AbstractDataBlockEncoder implements DataBlockEncoder { protected boolean includesTags() { return this.decodingCtx.getHFileContext().isIncludesTags(); } - } - } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index bb22ad83891..755647d6274 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -732,9 +732,8 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder { protected final ObjectIntPair tmpPair = new ObjectIntPair<>(); protected STATE current, previous; - public BufferedEncodedSeeker(CellComparator comparator, - HFileBlockDecodingContext decodingCtx) { - super(comparator, decodingCtx); + public BufferedEncodedSeeker(HFileBlockDecodingContext decodingCtx) { + super(decodingCtx); if (decodingCtx.getHFileContext().isCompressTags()) { try { tagCompressionContext = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); @@ -1008,11 +1007,7 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder { } /** - * @param cell - * @param out - * @param encodingCtx * @return unencoded size added - * @throws IOException */ protected final int afterEncodingKeyValue(Cell cell, DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx) throws IOException { @@ -1102,7 +1097,7 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder { public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { - throw new IOException (this.getClass().getName() + " only accepts " + throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } @@ -1154,8 +1149,8 @@ abstract class BufferedDataBlockEncoder extends AbstractDataBlockEncoder { .getEncodingState(); // Write the unencodedDataSizeWritten (with header size) Bytes.putInt(uncompressedBytesWithHeader, - HConstants.HFILEBLOCK_HEADER_SIZE + DataBlockEncoding.ID_SIZE, state.unencodedDataSizeWritten - ); + HConstants.HFILEBLOCK_HEADER_SIZE + DataBlockEncoding.ID_SIZE, + state.unencodedDataSizeWritten); postEncoding(encodingCtx); } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java index d7ab009f535..02bb8431f1f 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java @@ -20,9 +20,7 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; - import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; @@ -48,7 +46,8 @@ public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder { + "encoding context."); } - HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; + HFileBlockDefaultEncodingContext encodingCtx = + (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(out); NoneEncoder encoder = new NoneEncoder(out, encodingCtx); @@ -81,9 +80,8 @@ public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder { } @Override - public EncodedSeeker createSeeker(CellComparator comparator, - final HFileBlockDecodingContext decodingCtx) { - return new BufferedEncodedSeeker(comparator, decodingCtx) { + public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) { + return new BufferedEncodedSeeker(decodingCtx) { @Override protected void decodeNext() { current.keyLength = currentBuffer.getInt(); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java index e6f339da3b7..d3c41fbac12 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java @@ -20,7 +20,6 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; - import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.io.hfile.HFileContext; @@ -45,20 +44,13 @@ public interface DataBlockEncoder { * Starts encoding for a block of KeyValues. Call * {@link #endBlockEncoding(HFileBlockEncodingContext, DataOutputStream, byte[])} to finish * encoding of a block. - * @param encodingCtx - * @param out - * @throws IOException */ void startBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException; /** * Encodes a KeyValue. - * @param cell - * @param encodingCtx - * @param out * @return unencoded kv size written - * @throws IOException */ int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException; @@ -66,10 +58,6 @@ public interface DataBlockEncoder { /** * Ends encoding for a block of KeyValues. Gives a chance for the encoder to do the finishing * stuff for the encoded block. It must be called at the end of block encoding. - * @param encodingCtx - * @param out - * @param uncompressedBytesWithHeader - * @throws IOException */ void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out, byte[] uncompressedBytesWithHeader) throws IOException; @@ -77,7 +65,6 @@ public interface DataBlockEncoder { /** * Decode. * @param source Compressed stream of KeyValues. - * @param decodingCtx * @return Uncompressed block of KeyValues. * @throws IOException If there is an error in source. */ @@ -96,11 +83,9 @@ public interface DataBlockEncoder { /** * Create a HFileBlock seeker which find KeyValues within a block. - * @param comparator what kind of comparison should be used - * @param decodingCtx * @return A newly created seeker. */ - EncodedSeeker createSeeker(CellComparator comparator, HFileBlockDecodingContext decodingCtx); + EncodedSeeker createSeeker(HFileBlockDecodingContext decodingCtx); /** * Creates a encoder specific encoding context @@ -188,8 +173,6 @@ public interface DataBlockEncoder { /** * Compare the given key against the current key - * @param comparator - * @param key * @return -1 is the passed key is smaller than the current key, 0 if equal and 1 if greater */ public int compareKey(CellComparator comparator, Cell key); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java index ab93d1953a7..afb3e6d663f 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java @@ -20,9 +20,7 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; - import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; @@ -382,9 +380,8 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder { } @Override - public EncodedSeeker createSeeker(CellComparator comparator, - HFileBlockDecodingContext decodingCtx) { - return new BufferedEncodedSeeker(comparator, decodingCtx) { + public EncodedSeeker createSeeker(HFileBlockDecodingContext decodingCtx) { + return new BufferedEncodedSeeker(decodingCtx) { private byte[] familyNameWithSize; private static final int TIMESTAMP_WITH_TYPE_LENGTH = Bytes.SIZEOF_LONG + Bytes.SIZEOF_BYTE; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java index aa9a43683e6..a5774e5cd7b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java @@ -21,9 +21,7 @@ import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; - import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; @@ -397,9 +395,8 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder { } @Override - public EncodedSeeker createSeeker(CellComparator comparator, - final HFileBlockDecodingContext decodingCtx) { - return new BufferedEncodedSeeker(comparator, decodingCtx) { + public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) { + return new BufferedEncodedSeeker(decodingCtx) { private void decode(boolean isFirst) { byte flag = currentBuffer.get(); if ((flag & FLAG_SAME_KEY_LENGTH) == 0) { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java index 7f29302ed87..2cbffa9ec06 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDecodingContext.java @@ -17,20 +17,18 @@ package org.apache.hadoop.hbase.io.encoding; import java.io.IOException; - import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.yetus.audience.InterfaceAudience; /** * A decoding context that is created by a reader's encoder, and is shared - * across the reader's all read operations. + * across all of the reader's read operations. * * @see HFileBlockEncodingContext for encoding */ @InterfaceAudience.Private public interface HFileBlockDecodingContext { - /** * Perform all actions that need to be done before the encoder's real decoding * process. Decompression needs to be done if @@ -46,7 +44,6 @@ public interface HFileBlockDecodingContext { * ByteBuffer pointed after the header but before the data * @param onDiskBlock * on disk data to be decoded - * @throws IOException */ void prepareDecoding( int onDiskSizeWithoutHeader, diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java index 97d0e6bd66a..e321a259c2d 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultDecodingContext.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.io.encoding; import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; - import org.apache.commons.io.IOUtils; import org.apache.hadoop.hbase.io.ByteBuffInputStream; import org.apache.hadoop.hbase.io.TagCompressionContext; @@ -41,11 +40,10 @@ import org.apache.yetus.audience.InterfaceAudience; * */ @InterfaceAudience.Private -public class HFileBlockDefaultDecodingContext implements - HFileBlockDecodingContext { +public class HFileBlockDefaultDecodingContext implements HFileBlockDecodingContext { private final HFileContext fileContext; private TagCompressionContext tagCompressionContext; - + public HFileBlockDefaultDecodingContext(HFileContext fileContext) { this.fileContext = fileContext; } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java index d029e1c8f7b..169f97915b2 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java @@ -17,13 +17,11 @@ package org.apache.hadoop.hbase.io.encoding; import static org.apache.hadoop.hbase.io.compress.Compression.Algorithm.NONE; - import java.io.ByteArrayInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.security.SecureRandom; - import org.apache.hadoop.hbase.io.ByteArrayOutputStream; import org.apache.hadoop.hbase.io.TagCompressionContext; import org.apache.hadoop.hbase.io.compress.Compression; @@ -36,7 +34,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.compress.CompressionOutputStream; import org.apache.hadoop.io.compress.Compressor; import org.apache.yetus.audience.InterfaceAudience; - import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; /** @@ -47,8 +44,7 @@ import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; * */ @InterfaceAudience.Private -public class HFileBlockDefaultEncodingContext implements - HFileBlockEncodingContext { +public class HFileBlockDefaultEncodingContext implements HFileBlockEncodingContext { private BlockType blockType; private final DataBlockEncoding encodingAlgo; @@ -113,7 +109,6 @@ public class HFileBlockDefaultEncodingContext implements /** * prepare to start a new encoding. - * @throws IOException */ public void prepareEncoding(DataOutputStream out) throws IOException { if (encodingAlgo != null && encodingAlgo != DataBlockEncoding.NONE) { @@ -133,7 +128,8 @@ public class HFileBlockDefaultEncodingContext implements } private Bytes compressAfterEncoding(byte[] uncompressedBytesWithHeaderBuffer, - int uncompressedBytesWithHeaderOffset, int uncompressedBytesWithHeaderLength, byte[] headerBytes) + int uncompressedBytesWithHeaderOffset, int uncompressedBytesWithHeaderLength, + byte[] headerBytes) throws IOException { Encryption.Context cryptoContext = fileContext.getEncryptionContext(); if (cryptoContext != Encryption.Context.NONE) { @@ -158,7 +154,8 @@ public class HFileBlockDefaultEncodingContext implements compressedByteStream.reset(); compressionStream.resetState(); compressionStream.write(uncompressedBytesWithHeaderBuffer, - headerBytes.length + uncompressedBytesWithHeaderOffset, uncompressedBytesWithHeaderLength - headerBytes.length); + headerBytes.length + uncompressedBytesWithHeaderOffset, + uncompressedBytesWithHeaderLength - headerBytes.length); compressionStream.flush(); compressionStream.finish(); byte[] plaintext = compressedByteStream.toByteArray(); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockEncodingContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockEncodingContext.java index 96022294835..487dd45517f 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockEncodingContext.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockEncodingContext.java @@ -17,7 +17,6 @@ package org.apache.hadoop.hbase.io.encoding; import java.io.IOException; - import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.util.Bytes; @@ -46,9 +45,6 @@ public interface HFileBlockEncodingContext { /** * Do any action that needs to be performed after the encoding. * Compression is also included if a non-null compression algorithm is used - * - * @param blockType - * @throws IOException */ void postEncoding(BlockType blockType) throws IOException; @@ -64,7 +60,6 @@ public interface HFileBlockEncodingContext { /** * Sets the encoding state. - * @param state */ void setEncodingState(EncodingState state); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java index 176bea3dfb4..7af0c8c9feb 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java @@ -21,9 +21,7 @@ import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; - import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.PrivateCellUtil; @@ -195,9 +193,8 @@ public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder { } @Override - public EncodedSeeker createSeeker(CellComparator comparator, - final HFileBlockDecodingContext decodingCtx) { - return new BufferedEncodedSeeker(comparator, decodingCtx) { + public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) { + return new BufferedEncodedSeeker(decodingCtx) { @Override protected void decodeNext() { current.keyLength = ByteBuff.readCompressedInt(currentBuffer); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java index 12cd1ea214d..5de20ddbf5f 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexCodecV1.java @@ -22,10 +22,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; - import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.io.ByteArrayOutputStream; @@ -71,7 +68,8 @@ public class RowIndexCodecV1 extends AbstractDataBlockEncoder { + "encoding context."); } - HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; + HFileBlockDefaultEncodingContext encodingCtx = + (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(out); RowIndexEncoderV1 encoder = new RowIndexEncoderV1(out, encodingCtx); @@ -115,8 +113,7 @@ public class RowIndexCodecV1 extends AbstractDataBlockEncoder { dup.limit(sourceAsBuffer.position() + onDiskSize); return dup.slice(); } else { - RowIndexSeekerV1 seeker = new RowIndexSeekerV1(CellComparatorImpl.COMPARATOR, - decodingCtx); + RowIndexSeekerV1 seeker = new RowIndexSeekerV1(decodingCtx); seeker.setCurrentBuffer(new SingleByteBuff(sourceAsBuffer)); List kvs = new ArrayList<>(); kvs.add(seeker.getCell()); @@ -151,9 +148,7 @@ public class RowIndexCodecV1 extends AbstractDataBlockEncoder { } @Override - public EncodedSeeker createSeeker(CellComparator comparator, - HFileBlockDecodingContext decodingCtx) { - return new RowIndexSeekerV1(comparator, decodingCtx); + public EncodedSeeker createSeeker(HFileBlockDecodingContext decodingCtx) { + return new RowIndexSeekerV1(decodingCtx); } - } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java index 23887141dcc..711b9db30bc 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexEncoderV1.java @@ -12,9 +12,7 @@ package org.apache.hadoop.hbase.io.encoding; import java.io.DataOutputStream; import java.io.IOException; - import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.io.ByteArrayOutputStream; import org.apache.yetus.audience.InterfaceAudience; @@ -32,10 +30,12 @@ public class RowIndexEncoderV1 { private NoneEncoder encoder; private int startOffset = -1; private ByteArrayOutputStream rowsOffsetBAOS = new ByteArrayOutputStream(64 * 4); + private final HFileBlockEncodingContext context; public RowIndexEncoderV1(DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx) { this.out = out; this.encoder = new NoneEncoder(out, encodingCtx); + this.context = encodingCtx; } public int write(Cell cell) throws IOException { @@ -56,7 +56,7 @@ public class RowIndexEncoderV1 { throw new IOException("Key cannot be null or empty"); } if (lastCell != null) { - int keyComp = CellComparatorImpl.COMPARATOR.compareRows(lastCell, cell); + int keyComp = this.context.getHFileContext().getCellComparator().compareRows(lastCell, cell); if (keyComp > 0) { throw new IOException("Added a key not lexically larger than" + " previous. Current cell = " + cell + ", lastCell = " + lastCell); diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java index 9c0532ef828..7ff7555ceb2 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java @@ -17,8 +17,6 @@ package org.apache.hadoop.hbase.io.encoding; import java.nio.ByteBuffer; - -import org.apache.hadoop.hbase.ByteBufferExtendedCell; import org.apache.hadoop.hbase.ByteBufferKeyOnlyKeyValue; import org.apache.hadoop.hbase.ByteBufferKeyValue; import org.apache.hadoop.hbase.Cell; @@ -50,10 +48,11 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker { private int rowNumber; private ByteBuff rowOffsets = null; + private final CellComparator cellComparator; - public RowIndexSeekerV1(CellComparator comparator, - HFileBlockDecodingContext decodingCtx) { - super(comparator, decodingCtx); + public RowIndexSeekerV1(HFileBlockDecodingContext decodingCtx) { + super(decodingCtx); + this.cellComparator = decodingCtx.getHFileContext().getCellComparator(); } @Override @@ -131,8 +130,7 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker { int comp = 0; while (low <= high) { mid = low + ((high - low) >> 1); - ByteBuffer row = getRow(mid); - comp = compareRows(row, seekCell); + comp = this.cellComparator.compareRows(getRow(mid), seekCell); if (comp < 0) { low = mid + 1; } else if (comp > 0) { @@ -154,19 +152,6 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker { } } - private int compareRows(ByteBuffer row, Cell seekCell) { - if (seekCell instanceof ByteBufferExtendedCell) { - return ByteBufferUtils.compareTo(row, row.position(), row.remaining(), - ((ByteBufferExtendedCell) seekCell).getRowByteBuffer(), - ((ByteBufferExtendedCell) seekCell).getRowPosition(), - seekCell.getRowLength()); - } else { - return ByteBufferUtils.compareTo(row, row.position(), row.remaining(), - seekCell.getRowArray(), seekCell.getRowOffset(), - seekCell.getRowLength()); - } - } - private ByteBuffer getRow(int index) { int offset = rowOffsets.getIntAfterPosition(index * Bytes.SIZEOF_INT); ByteBuff block = currentBuffer.duplicate(); @@ -191,8 +176,8 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker { } } do { - int comp; - comp = PrivateCellUtil.compareKeyIgnoresMvcc(comparator, seekCell, current.currentKey); + int comp = + PrivateCellUtil.compareKeyIgnoresMvcc(this.cellComparator, seekCell, current.currentKey); if (comp == 0) { // exact match if (seekBefore) { if (!previous.isValid()) { @@ -400,5 +385,4 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker { return ret; } } - } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java index d606497e2f1..ea4782d0351 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContext.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.io.hfile; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.compress.Compression; @@ -28,9 +30,10 @@ import org.apache.hadoop.hbase.util.ClassSize; import org.apache.yetus.audience.InterfaceAudience; /** - * This carries the information on some of the meta data about the HFile. This - * meta data is used across the HFileWriter/Readers and the HFileBlocks. - * This helps to add new information to the HFile. + * Read-only HFile Context Information. Meta data that is used by HFileWriter/Readers and by + * HFileBlocks. Create one using the {@link HFileContextBuilder} (See HFileInfo and the HFile + * Trailer class). + * @see HFileContextBuilder */ @InterfaceAudience.Private public class HFileContext implements HeapSize, Cloneable { @@ -42,7 +45,7 @@ public class HFileContext implements HeapSize, Cloneable { //byte[] headers for column family and table name 2 * ClassSize.ARRAY + 2 * ClassSize.REFERENCE); - public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024; + private static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024; /** Whether checksum is enabled or not**/ private boolean usesHBaseChecksum = true; @@ -67,6 +70,7 @@ public class HFileContext implements HeapSize, Cloneable { private String hfileName; private byte[] columnFamily; private byte[] tableName; + private CellComparator cellComparator; //Empty constructor. Go with setters public HFileContext() { @@ -74,7 +78,6 @@ public class HFileContext implements HeapSize, Cloneable { /** * Copy constructor - * @param context */ public HFileContext(HFileContext context) { this.usesHBaseChecksum = context.usesHBaseChecksum; @@ -91,13 +94,14 @@ public class HFileContext implements HeapSize, Cloneable { this.hfileName = context.hfileName; this.columnFamily = context.columnFamily; this.tableName = context.tableName; + this.cellComparator = context.cellComparator; } HFileContext(boolean useHBaseChecksum, boolean includesMvcc, boolean includesTags, Compression.Algorithm compressAlgo, boolean compressTags, ChecksumType checksumType, int bytesPerChecksum, int blockSize, DataBlockEncoding encoding, Encryption.Context cryptoContext, long fileCreateTime, String hfileName, - byte[] columnFamily, byte[] tableName) { + byte[] columnFamily, byte[] tableName, CellComparator cellComparator) { this.usesHBaseChecksum = useHBaseChecksum; this.includesMvcc = includesMvcc; this.includesTags = includesTags; @@ -114,11 +118,14 @@ public class HFileContext implements HeapSize, Cloneable { this.hfileName = hfileName; this.columnFamily = columnFamily; this.tableName = tableName; + // If no cellComparator specified, make a guess based off tablename. If hbase:meta, then should + // be the meta table comparator. Comparators are per table. + this.cellComparator = cellComparator != null ? cellComparator : this.tableName != null ? + CellComparatorImpl.getCellComparator(this.tableName) : CellComparator.getInstance(); } /** - * @return true when on-disk blocks from this file are compressed, and/or encrypted; - * false otherwise. + * @return true when on-disk blocks are compressed, and/or encrypted; false otherwise. */ public boolean isCompressedOrEncrypted() { Compression.Algorithm compressAlgo = getCompression(); @@ -208,6 +215,11 @@ public class HFileContext implements HeapSize, Cloneable { public byte[] getTableName() { return this.tableName; } + + public CellComparator getCellComparator() { + return this.cellComparator; + } + /** * HeapSize implementation. NOTE : The heap size should be altered when new state variable are * added. @@ -263,8 +275,9 @@ public class HFileContext implements HeapSize, Cloneable { sb.append(", columnFamily="); sb.append(Bytes.toStringBinary(columnFamily)); } + sb.append(", cellComparator="); + sb.append(this.cellComparator); sb.append("]"); return sb.toString(); } - } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java index 5fa56264f30..a44f273c58f 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileContextBuilder.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.io.hfile; +import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.crypto.Encryption; @@ -56,6 +57,7 @@ public class HFileContextBuilder { private String hfileName = null; private byte[] columnFamily = null; private byte[] tableName = null; + private CellComparator cellComparator; public HFileContextBuilder() {} @@ -77,6 +79,7 @@ public class HFileContextBuilder { this.hfileName = hfc.getHFileName(); this.columnFamily = hfc.getColumnFamily(); this.tableName = hfc.getTableName(); + this.cellComparator = hfc.getCellComparator(); } public HFileContextBuilder withHBaseCheckSum(boolean useHBaseCheckSum) { @@ -149,9 +152,14 @@ public class HFileContextBuilder { return this; } + public HFileContextBuilder withCellComparator(CellComparator cellComparator) { + this.cellComparator = cellComparator; + return this; + } + public HFileContext build() { return new HFileContext(usesHBaseChecksum, includesMvcc, includesTags, compression, compressTags, checksumType, bytesPerChecksum, blocksize, encoding, cryptoContext, - fileCreateTime, hfileName, columnFamily, tableName); + fileCreateTime, hfileName, columnFamily, tableName, cellComparator); } } diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 35a905212b7..ab1ede6ba11 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,7 +21,6 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TASK_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.BULKLOAD_TIME_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_KEY; - import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.InetSocketAddress; @@ -38,13 +37,11 @@ import java.util.TreeSet; import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; - import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; @@ -91,7 +88,6 @@ import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; /** @@ -179,18 +175,17 @@ public class HFileOutputFormat2 return combineTableNameSuffix(tableName, family); } - static RecordWriter - createRecordWriter(final TaskAttemptContext context, final OutputCommitter committer) - throws IOException { + static RecordWriter createRecordWriter( + final TaskAttemptContext context, final OutputCommitter committer) throws IOException { // Get the path of the temporary output file final Path outputDir = ((FileOutputCommitter)committer).getWorkPath(); final Configuration conf = context.getConfiguration(); - final boolean writeMultipleTables = conf.getBoolean(MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY, false) ; + final boolean writeMultipleTables = + conf.getBoolean(MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY, false); final String writeTableNames = conf.get(OUTPUT_TABLE_NAME_CONF_KEY); - if (writeTableNames==null || writeTableNames.isEmpty()) { - throw new IllegalArgumentException("Configuration parameter " + OUTPUT_TABLE_NAME_CONF_KEY - + " cannot be empty"); + if (writeTableNames == null || writeTableNames.isEmpty()) { + throw new IllegalArgumentException("" + OUTPUT_TABLE_NAME_CONF_KEY + " cannot be empty"); } final FileSystem fs = outputDir.getFileSystem(conf); // These configs. are from hbase-*.xml @@ -199,19 +194,12 @@ public class HFileOutputFormat2 // Invented config. Add to hbase-*.xml if other than default compression. final String defaultCompressionStr = conf.get("hfile.compression", Compression.Algorithm.NONE.getName()); - final Algorithm defaultCompression = HFileWriterImpl - .compressionByName(defaultCompressionStr); + final Algorithm defaultCompression = HFileWriterImpl.compressionByName(defaultCompressionStr); String compressionStr = conf.get(COMPRESSION_OVERRIDE_CONF_KEY); - final Algorithm overriddenCompression; - if (compressionStr != null) { - overriddenCompression = Compression.getCompressionAlgorithmByName(compressionStr); - } else { - overriddenCompression = null; - } - + final Algorithm overriddenCompression = compressionStr != null ? + Compression.getCompressionAlgorithmByName(compressionStr): null; final boolean compactionExclude = conf.getBoolean( "hbase.mapreduce.hfileoutputformat.compaction.exclude", false); - final Set allTableNames = Arrays.stream(writeTableNames.split( Bytes.toString(tableSeparator))).collect(Collectors.toSet()); @@ -224,24 +212,17 @@ public class HFileOutputFormat2 String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_OVERRIDE_CONF_KEY); final Map datablockEncodingMap = createFamilyDataBlockEncodingMap(conf); - final DataBlockEncoding overriddenEncoding; - if (dataBlockEncodingStr != null) { - overriddenEncoding = DataBlockEncoding.valueOf(dataBlockEncodingStr); - } else { - overriddenEncoding = null; - } + final DataBlockEncoding overriddenEncoding = dataBlockEncodingStr != null ? + DataBlockEncoding.valueOf(dataBlockEncodingStr) : null; return new RecordWriter() { // Map of families to writers and how much has been output on the writer. - private final Map writers = - new TreeMap<>(Bytes.BYTES_COMPARATOR); - private final Map previousRows = - new TreeMap<>(Bytes.BYTES_COMPARATOR); + private final Map writers = new TreeMap<>(Bytes.BYTES_COMPARATOR); + private final Map previousRows = new TreeMap<>(Bytes.BYTES_COMPARATOR); private final long now = EnvironmentEdgeManager.currentTime(); @Override - public void write(ImmutableBytesWritable row, V cell) - throws IOException { + public void write(ImmutableBytesWritable row, V cell) throws IOException { Cell kv = cell; // null input == user explicitly wants to flush if (row == null && kv == null) { @@ -255,12 +236,11 @@ public class HFileOutputFormat2 byte[] tableNameBytes = null; if (writeMultipleTables) { tableNameBytes = MultiTableHFileOutputFormat.getTableName(row.get()); - tableNameBytes = - TableName.valueOf(tableNameBytes).getNameWithNamespaceInclAsString() + tableNameBytes = TableName.valueOf(tableNameBytes).getNameWithNamespaceInclAsString() .getBytes(Charset.defaultCharset()); if (!allTableNames.contains(Bytes.toString(tableNameBytes))) { - throw new IllegalArgumentException("TableName '" + Bytes.toString(tableNameBytes) + - "' not" + " expected"); + throw new IllegalArgumentException("TableName " + Bytes.toString(tableNameBytes) + + " not expected"); } } else { tableNameBytes = Bytes.toBytes(writeTableNames); @@ -275,9 +255,7 @@ public class HFileOutputFormat2 if (wl == null) { Path writerPath = null; if (writeMultipleTables) { - writerPath = new Path(outputDir,new Path(tableRelPath, Bytes - .toString(family))); - + writerPath = new Path(outputDir,new Path(tableRelPath, Bytes.toString(family))); } else { writerPath = new Path(outputDir, Bytes.toString(family)); @@ -303,33 +281,23 @@ public class HFileOutputFormat2 connection.getRegionLocator(TableName.valueOf(tableName))) { loc = locator.getRegionLocation(rowKey); } catch (Throwable e) { - LOG.warn("There's something wrong when locating rowkey: " + - Bytes.toString(rowKey) + " for tablename: " + tableName, e); + LOG.warn("Something wrong locating rowkey {} in {}", + Bytes.toString(rowKey), tableName, e); loc = null; } } if (null == loc) { - if (LOG.isTraceEnabled()) { - LOG.trace("failed to get region location, so use default writer for rowkey: " + - Bytes.toString(rowKey)); - } + LOG.trace("Failed get of location, use default writer {}", Bytes.toString(rowKey)); wl = getNewWriter(tableNameBytes, family, conf, null); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("first rowkey: [" + Bytes.toString(rowKey) + "]"); - } + LOG.debug("First rowkey: [{}]", Bytes.toString(rowKey)); InetSocketAddress initialIsa = new InetSocketAddress(loc.getHostname(), loc.getPort()); if (initialIsa.isUnresolved()) { - if (LOG.isTraceEnabled()) { - LOG.trace("failed to resolve bind address: " + loc.getHostname() + ":" - + loc.getPort() + ", so use default writer"); - } + LOG.trace("Failed resolve address {}, use default writer", loc.getHostnamePort()); wl = getNewWriter(tableNameBytes, family, conf, null); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("use favored nodes writer: " + initialIsa.getHostString()); - } + LOG.debug("Use favored nodes writer: {}", initialIsa.getHostString()); wl = getNewWriter(tableNameBytes, family, conf, new InetSocketAddress[] { initialIsa }); } @@ -369,29 +337,27 @@ public class HFileOutputFormat2 private void closeWriter(WriterLength wl) throws IOException { if (wl.writer != null) { - LOG.info( - "Writer=" + wl.writer.getPath() + ((wl.written == 0)? "": ", wrote=" + wl.written)); + LOG.info("Writer=" + wl.writer.getPath() + + ((wl.written == 0)? "": ", wrote=" + wl.written)); close(wl.writer); + wl.writer = null; } - wl.writer = null; wl.written = 0; } /* * Create a new StoreFile.Writer. - * @param family * @return A WriterLength, containing a new StoreFile.Writer. - * @throws IOException */ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="BX_UNBOXING_IMMEDIATELY_REBOXED", justification="Not important") - private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration - conf, InetSocketAddress[] favoredNodes) throws IOException { + private WriterLength getNewWriter(byte[] tableName, byte[] family, Configuration conf, + InetSocketAddress[] favoredNodes) throws IOException { byte[] tableAndFamily = getTableNameSuffixedWithFamily(tableName, family); Path familydir = new Path(outputDir, Bytes.toString(family)); if (writeMultipleTables) { familydir = new Path(outputDir, - new Path(getTableRelativePath(tableName), Bytes.toString(family))); + new Path(getTableRelativePath(tableName), Bytes.toString(family))); } WriterLength wl = new WriterLength(); Algorithm compression = overriddenCompression; @@ -409,12 +375,9 @@ public class HFileOutputFormat2 encoding = encoding == null ? datablockEncodingMap.get(tableAndFamily) : encoding; encoding = encoding == null ? DataBlockEncoding.NONE : encoding; HFileContextBuilder contextBuilder = new HFileContextBuilder() - .withCompression(compression) - .withChecksumType(HStore.getChecksumType(conf)) - .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)) - .withBlockSize(blockSize) - .withColumnFamily(family) - .withTableName(tableName); + .withCompression(compression).withChecksumType(HStore.getChecksumType(conf)) + .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blockSize) + .withColumnFamily(family).withTableName(tableName); if (HFile.getFormatVersion(conf) >= HFile.MIN_FORMAT_VERSION_WITH_TAGS) { contextBuilder.withIncludesTags(true); @@ -423,16 +386,13 @@ public class HFileOutputFormat2 contextBuilder.withDataBlockEncoding(encoding); HFileContext hFileContext = contextBuilder.build(); if (null == favoredNodes) { - wl.writer = - new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, fs) - .withOutputDir(familydir).withBloomType(bloomType) - .withComparator(CellComparator.getInstance()).withFileContext(hFileContext).build(); + wl.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, fs) + .withOutputDir(familydir).withBloomType(bloomType) + .withFileContext(hFileContext).build(); } else { - wl.writer = - new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, new HFileSystem(fs)) - .withOutputDir(familydir).withBloomType(bloomType) - .withComparator(CellComparator.getInstance()).withFileContext(hFileContext) - .withFavoredNodes(favoredNodes).build(); + wl.writer = new StoreFileWriter.Builder(conf, CacheConfig.DISABLED, new HFileSystem(fs)) + .withOutputDir(familydir).withBloomType(bloomType) + .withFileContext(hFileContext).withFavoredNodes(favoredNodes).build(); } this.writers.put(tableAndFamily, wl); @@ -441,22 +401,17 @@ public class HFileOutputFormat2 private void close(final StoreFileWriter w) throws IOException { if (w != null) { - w.appendFileInfo(BULKLOAD_TIME_KEY, - Bytes.toBytes(System.currentTimeMillis())); - w.appendFileInfo(BULKLOAD_TASK_KEY, - Bytes.toBytes(context.getTaskAttemptID().toString())); - w.appendFileInfo(MAJOR_COMPACTION_KEY, - Bytes.toBytes(true)); - w.appendFileInfo(EXCLUDE_FROM_MINOR_COMPACTION_KEY, - Bytes.toBytes(compactionExclude)); + w.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis())); + w.appendFileInfo(BULKLOAD_TASK_KEY, Bytes.toBytes(context.getTaskAttemptID().toString())); + w.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(true)); + w.appendFileInfo(EXCLUDE_FROM_MINOR_COMPACTION_KEY, Bytes.toBytes(compactionExclude)); w.appendTrackedTimestampsToMetadata(); w.close(); } } @Override - public void close(TaskAttemptContext c) - throws IOException, InterruptedException { + public void close(TaskAttemptContext c) throws IOException, InterruptedException { for (WriterLength wl: this.writers.values()) { close(wl.writer); } @@ -496,21 +451,18 @@ public class HFileOutputFormat2 throws IOException { ArrayList ret = new ArrayList<>(); - for(RegionLocator regionLocator : regionLocators) - { + for(RegionLocator regionLocator : regionLocators) { TableName tableName = regionLocator.getName(); LOG.info("Looking up current regions for table " + tableName); byte[][] byteKeys = regionLocator.getStartKeys(); for (byte[] byteKey : byteKeys) { byte[] fullKey = byteKey; //HFileOutputFormat2 use case - if (writeMultipleTables) - { + if (writeMultipleTables) { //MultiTableHFileOutputFormat use case fullKey = combineTableNameSuffix(tableName.getName(), byteKey); } if (LOG.isDebugEnabled()) { - LOG.debug("SplitPoint startkey for table [" + tableName + "]: [" + Bytes.toStringBinary - (fullKey) + "]"); + LOG.debug("SplitPoint startkey for " + tableName + ": " + Bytes.toStringBinary(fullKey)); } ret.add(new ImmutableBytesWritable(fullKey)); } @@ -537,8 +489,8 @@ public class HFileOutputFormat2 TreeSet sorted = new TreeSet<>(startKeys); ImmutableBytesWritable first = sorted.first(); if (writeMultipleTables) { - first = new ImmutableBytesWritable(MultiTableHFileOutputFormat.getSuffix(sorted.first - ().get())); + first = + new ImmutableBytesWritable(MultiTableHFileOutputFormat.getSuffix(sorted.first().get())); } if (!first.equals(HConstants.EMPTY_BYTE_ARRAY)) { throw new IllegalArgumentException( @@ -641,11 +593,10 @@ public class HFileOutputFormat2 /* Now get the region start keys for every table required */ List allTableNames = new ArrayList<>(multiTableInfo.size()); - List regionLocators = new ArrayList<>( multiTableInfo.size()); - List tableDescriptors = new ArrayList<>( multiTableInfo.size()); + List regionLocators = new ArrayList<>(multiTableInfo.size()); + List tableDescriptors = new ArrayList<>(multiTableInfo.size()); - for( TableInfo tableInfo : multiTableInfo ) - { + for(TableInfo tableInfo : multiTableInfo) { regionLocators.add(tableInfo.getRegionLocator()); String tn = writeMultipleTables? tableInfo.getRegionLocator().getName().getNameWithNamespaceInclAsString(): @@ -653,10 +604,12 @@ public class HFileOutputFormat2 allTableNames.add(tn); tableDescriptors.add(tableInfo.getTableDescriptor()); } - // Record tablenames for creating writer by favored nodes, and decoding compression, block size and other attributes of columnfamily per table + // Record tablenames for creating writer by favored nodes, and decoding compression, + // block size and other attributes of columnfamily per table conf.set(OUTPUT_TABLE_NAME_CONF_KEY, StringUtils.join(allTableNames, Bytes .toString(tableSeparator))); - List startKeys = getRegionStartKeys(regionLocators, writeMultipleTables); + List startKeys = + getRegionStartKeys(regionLocators, writeMultipleTables); // Use table's region boundaries for TOP split points. LOG.info("Configuring " + startKeys.size() + " reduce partitions " + "to match current region count for all tables"); @@ -852,9 +805,11 @@ public class HFileOutputFormat2 TotalOrderPartitioner.setPartitionFile(conf, partitionsPath); } - @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE") + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = + "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE") @VisibleForTesting - static String serializeColumnFamilyAttribute(Function fn, List allTables) + static String serializeColumnFamilyAttribute(Function fn, + List allTables) throws UnsupportedEncodingException { StringBuilder attributeValue = new StringBuilder(); int i = 0; @@ -869,8 +824,8 @@ public class HFileOutputFormat2 attributeValue.append('&'); } attributeValue.append(URLEncoder.encode( - Bytes.toString(combineTableNameSuffix(tableDescriptor.getTableName().getName(), familyDescriptor.getName())), - "UTF-8")); + Bytes.toString(combineTableNameSuffix(tableDescriptor.getTableName().getName(), + familyDescriptor.getName())), "UTF-8")); attributeValue.append('='); attributeValue.append(URLEncoder.encode(fn.apply(familyDescriptor), "UTF-8")); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java index 3c74d114e1e..35fb2818e63 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java @@ -26,27 +26,25 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; - import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.CellComparatorImpl.MetaCellComparator; import org.apache.hadoop.hbase.KeyValue; -import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.io.compress.Compression; -import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos; import org.apache.hadoop.hbase.util.Bytes; - +import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos; /** * The {@link HFile} has a fixed trailer which contains offsets to other * variable parts of the file. Also includes basic metadata on this file. The * trailer size is fixed within a given {@link HFile} format version only, but * we always store the version number as the last four-byte integer of the file. - * The version number itself is split into two portions, a major + * The version number itself is split into two portions, a major * version and a minor version. The last three bytes of a file are the major * version and a single preceding byte is the minor number. The major version * determines which readers/writers to use to read/write a hfile while a minor @@ -77,16 +75,24 @@ public class FixedFileTrailer { */ private long loadOnOpenDataOffset; - /** The number of entries in the root data index. */ + /** + * The number of entries in the root data index. + */ private int dataIndexCount; - /** Total uncompressed size of all blocks of the data index */ + /** + * Total uncompressed size of all blocks of the data index + */ private long uncompressedDataIndexSize; - /** The number of entries in the meta index */ + /** + * The number of entries in the meta index + */ private int metaIndexCount; - /** The total uncompressed size of keys/values stored in the file. */ + /** + * The total uncompressed size of keys/values stored in the file. + */ private long totalUncompressedBytes; /** @@ -95,7 +101,9 @@ public class FixedFileTrailer { */ private long entryCount; - /** The compression codec used for all blocks. */ + /** + * The compression codec used for all blocks. + */ private Compression.Algorithm compressionCodec = Compression.Algorithm.NONE; /** @@ -104,7 +112,9 @@ public class FixedFileTrailer { */ private int numDataIndexLevels; - /** The offset of the first data block. */ + /** + * The offset of the first data block. + */ private long firstDataBlockOffset; /** @@ -113,17 +123,25 @@ public class FixedFileTrailer { */ private long lastDataBlockOffset; - /** Raw key comparator class name in version 3 */ + /** + * Raw key comparator class name in version 3 + */ // We could write the actual class name from 2.0 onwards and handle BC private String comparatorClassName = CellComparator.getInstance().getClass().getName(); - /** The encryption key */ + /** + * The encryption key + */ private byte[] encryptionKey; - /** The {@link HFile} format major version. */ + /** + * The {@link HFile} format major version. + */ private final int majorVersion; - /** The {@link HFile} format minor version. */ + /** + * The {@link HFile} format minor version. + */ private final int minorVersion; FixedFileTrailer(int majorVersion, int minorVersion) { @@ -133,7 +151,7 @@ public class FixedFileTrailer { } private static int[] computeTrailerSizeByVersion() { - int versionToSize[] = new int[HFile.MAX_FORMAT_VERSION + 1]; + int[] versionToSize = new int[HFile.MAX_FORMAT_VERSION + 1]; // We support only 2 major versions now. ie. V2, V3 versionToSize[2] = 212; for (int version = 3; version <= HFile.MAX_FORMAT_VERSION; version++) { @@ -147,14 +165,13 @@ public class FixedFileTrailer { private static int getMaxTrailerSize() { int maxSize = 0; - for (int version = HFile.MIN_FORMAT_VERSION; - version <= HFile.MAX_FORMAT_VERSION; - ++version) + for (int version = HFile.MIN_FORMAT_VERSION; version <= HFile.MAX_FORMAT_VERSION; ++version) { maxSize = Math.max(getTrailerSize(version), maxSize); + } return maxSize; } - private static final int TRAILER_SIZE[] = computeTrailerSizeByVersion(); + private static final int[] TRAILER_SIZE = computeTrailerSizeByVersion(); private static final int MAX_TRAILER_SIZE = getMaxTrailerSize(); private static final int NOT_PB_SIZE = BlockType.MAGIC_LENGTH + Bytes.SIZEOF_INT; @@ -171,9 +188,6 @@ public class FixedFileTrailer { * Write the trailer to a data stream. We support writing version 1 for * testing and for determining version 1 trailer size. It is also easy to see * what fields changed in version 2. - * - * @param outputStream - * @throws IOException */ void serialize(DataOutputStream outputStream) throws IOException { HFile.checkFormatVersion(majorVersion); @@ -193,18 +207,18 @@ public class FixedFileTrailer { @org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting HFileProtos.FileTrailerProto toProtobuf() { HFileProtos.FileTrailerProto.Builder builder = HFileProtos.FileTrailerProto.newBuilder() - .setFileInfoOffset(fileInfoOffset) - .setLoadOnOpenDataOffset(loadOnOpenDataOffset) - .setUncompressedDataIndexSize(uncompressedDataIndexSize) - .setTotalUncompressedBytes(totalUncompressedBytes) - .setDataIndexCount(dataIndexCount) - .setMetaIndexCount(metaIndexCount) - .setEntryCount(entryCount) - .setNumDataIndexLevels(numDataIndexLevels) - .setFirstDataBlockOffset(firstDataBlockOffset) - .setLastDataBlockOffset(lastDataBlockOffset) - .setComparatorClassName(getHBase1CompatibleName(comparatorClassName)) - .setCompressionCodec(compressionCodec.ordinal()); + .setFileInfoOffset(fileInfoOffset) + .setLoadOnOpenDataOffset(loadOnOpenDataOffset) + .setUncompressedDataIndexSize(uncompressedDataIndexSize) + .setTotalUncompressedBytes(totalUncompressedBytes) + .setDataIndexCount(dataIndexCount) + .setMetaIndexCount(metaIndexCount) + .setEntryCount(entryCount) + .setNumDataIndexLevels(numDataIndexLevels) + .setFirstDataBlockOffset(firstDataBlockOffset) + .setLastDataBlockOffset(lastDataBlockOffset) + .setComparatorClassName(getHBase1CompatibleName(comparatorClassName)) + .setCompressionCodec(compressionCodec.ordinal()); if (encryptionKey != null) { builder.setEncryptionKey(UnsafeByteOperations.unsafeWrap(encryptionKey)); } @@ -239,9 +253,6 @@ public class FixedFileTrailer { * Deserialize the fixed file trailer from the given stream. The version needs * to already be specified. Make sure this is consistent with * {@link #serialize(DataOutputStream)}. - * - * @param inputStream - * @throws IOException */ void deserialize(DataInputStream inputStream) throws IOException { HFile.checkFormatVersion(majorVersion); @@ -249,7 +260,7 @@ public class FixedFileTrailer { BlockType.TRAILER.readAndCheck(inputStream); if (majorVersion > 2 - || (majorVersion == 2 && minorVersion >= HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION)) { + || (majorVersion == 2 && minorVersion >= HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION)) { deserializeFromPB(inputStream); } else { deserializeFromWritable(inputStream); @@ -263,14 +274,12 @@ public class FixedFileTrailer { /** * Deserialize the file trailer as protobuf - * @param inputStream - * @throws IOException */ void deserializeFromPB(DataInputStream inputStream) throws IOException { // read PB and skip padding int start = inputStream.available(); HFileProtos.FileTrailerProto trailerProto = - HFileProtos.FileTrailerProto.PARSER.parseDelimitedFrom(inputStream); + HFileProtos.FileTrailerProto.PARSER.parseDelimitedFrom(inputStream); int size = start - inputStream.available(); inputStream.skip(getTrailerSize() - NOT_PB_SIZE - size); @@ -320,8 +329,6 @@ public class FixedFileTrailer { /** * Deserialize the file trailer as writable data - * @param input - * @throws IOException */ void deserializeFromWritable(DataInput input) throws IOException { fileInfoOffset = input.readLong(); @@ -339,12 +346,13 @@ public class FixedFileTrailer { // TODO this is a classname encoded into an HFile's trailer. We are going to need to have // some compat code here. setComparatorClass(getComparatorClass(Bytes.readStringFixedSize(input, - MAX_COMPARATOR_NAME_LENGTH))); + MAX_COMPARATOR_NAME_LENGTH))); } - + private void append(StringBuilder sb, String s) { - if (sb.length() > 0) + if (sb.length() > 0) { sb.append(", "); + } sb.append(s); } @@ -375,18 +383,18 @@ public class FixedFileTrailer { /** * Reads a file trailer from the given file. * - * @param istream the input stream with the ability to seek. Does not have to - * be buffered, as only one read operation is made. + * @param istream the input stream with the ability to seek. Does not have to + * be buffered, as only one read operation is made. * @param fileSize the file size. Can be obtained using - * {@link org.apache.hadoop.fs.FileSystem#getFileStatus( - * org.apache.hadoop.fs.Path)}. + * {@link org.apache.hadoop.fs.FileSystem#getFileStatus( + *org.apache.hadoop.fs.Path)}. * @return the fixed file trailer read * @throws IOException if failed to read from the underlying stream, or the - * trailer is corrupted, or the version of the trailer is - * unsupported + * trailer is corrupted, or the version of the trailer is + * unsupported */ public static FixedFileTrailer readFromStream(FSDataInputStream istream, - long fileSize) throws IOException { + long fileSize) throws IOException { int bufferSize = MAX_TRAILER_SIZE; long seekPoint = fileSize - bufferSize; if (seekPoint < 0) { @@ -399,7 +407,7 @@ public class FixedFileTrailer { ByteBuffer buf = ByteBuffer.allocate(bufferSize); istream.readFully(buf.array(), buf.arrayOffset(), - buf.arrayOffset() + buf.limit()); + buf.arrayOffset() + buf.limit()); // Read the version from the last int of the file. buf.position(buf.limit() - Bytes.SIZEOF_INT); @@ -415,30 +423,30 @@ public class FixedFileTrailer { FixedFileTrailer fft = new FixedFileTrailer(majorVersion, minorVersion); fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(), - buf.arrayOffset() + bufferSize - trailerSize, trailerSize))); + buf.arrayOffset() + bufferSize - trailerSize, trailerSize))); return fft; } public void expectMajorVersion(int expected) { if (majorVersion != expected) { throw new IllegalArgumentException("Invalid HFile major version: " - + majorVersion - + " (expected: " + expected + ")"); + + majorVersion + + " (expected: " + expected + ")"); } } public void expectMinorVersion(int expected) { if (minorVersion != expected) { throw new IllegalArgumentException("Invalid HFile minor version: " - + minorVersion + " (expected: " + expected + ")"); + + minorVersion + " (expected: " + expected + ")"); } } public void expectAtLeastMajorVersion(int lowerBound) { if (majorVersion < lowerBound) { throw new IllegalArgumentException("Invalid HFile major version: " - + majorVersion - + " (expected: " + lowerBound + " or higher)."); + + majorVersion + + " (expected: " + lowerBound + " or higher)."); } } @@ -566,7 +574,7 @@ public class FixedFileTrailer { * than the new name; writing the new name will make it so newly-written hfiles are not parseable * by hbase-1.x, a facility we'd like to preserve across rolling upgrade and hbase-1.x clusters * reading hbase-2.x produce. - * + *

* The Comparators in hbase-2.x work the same as they did in hbase-1.x; they compare * KeyValues. In hbase-2.x they were renamed making use of the more generic 'Cell' * nomenclature to indicate that we intend to move away from KeyValues post hbase-2. A naming @@ -579,9 +587,9 @@ public class FixedFileTrailer { * * @param comparator String class name of the Comparator used in this hfile. * @return What to store in the trailer as our comparator name. + * @see #getComparatorClass(String) * @since hbase-2.0.0. * @deprecated Since hbase-2.0.0. Will be removed in hbase-3.0.0. - * @see #getComparatorClass(String) */ @Deprecated private String getHBase1CompatibleName(final String comparator) { @@ -596,20 +604,20 @@ public class FixedFileTrailer { @SuppressWarnings("unchecked") private static Class getComparatorClass(String comparatorClassName) - throws IOException { + throws IOException { Class comparatorKlass; // for BC if (comparatorClassName.equals(KeyValue.COMPARATOR.getLegacyKeyComparatorName()) - || comparatorClassName.equals(KeyValue.COMPARATOR.getClass().getName()) - || (comparatorClassName.equals("org.apache.hadoop.hbase.CellComparator"))) { + || comparatorClassName.equals(KeyValue.COMPARATOR.getClass().getName()) + || (comparatorClassName.equals("org.apache.hadoop.hbase.CellComparator"))) { comparatorKlass = CellComparatorImpl.class; } else if (comparatorClassName.equals(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName()) - || comparatorClassName.equals(KeyValue.META_COMPARATOR.getClass().getName()) - || (comparatorClassName - .equals("org.apache.hadoop.hbase.CellComparator$MetaCellComparator"))) { + || comparatorClassName.equals(KeyValue.META_COMPARATOR.getClass().getName()) + || (comparatorClassName + .equals("org.apache.hadoop.hbase.CellComparator$MetaCellComparator"))) { comparatorKlass = MetaCellComparator.class; } else if (comparatorClassName.equals("org.apache.hadoop.hbase.KeyValue$RawBytesComparator") - || comparatorClassName.equals("org.apache.hadoop.hbase.util.Bytes$ByteArrayComparator")) { + || comparatorClassName.equals("org.apache.hadoop.hbase.util.Bytes$ByteArrayComparator")) { // When the comparator to be used is Bytes.BYTES_RAWCOMPARATOR, we just return null from here // Bytes.BYTES_RAWCOMPARATOR is not a CellComparator comparatorKlass = null; @@ -624,19 +632,22 @@ public class FixedFileTrailer { return comparatorKlass; } - public static CellComparator createComparator( - String comparatorClassName) throws IOException { + static CellComparator createComparator(String comparatorClassName) throws IOException { + if (comparatorClassName.equals(CellComparatorImpl.COMPARATOR.getClass().getName())) { + return CellComparatorImpl.COMPARATOR; + } else if (comparatorClassName.equals( + CellComparatorImpl.META_COMPARATOR.getClass().getName())) { + return CellComparatorImpl.META_COMPARATOR; + } try { - Class comparatorClass = getComparatorClass(comparatorClassName); - if(comparatorClass != null){ + if (comparatorClass != null) { return comparatorClass.getDeclaredConstructor().newInstance(); } LOG.warn("No Comparator class for " + comparatorClassName + ". Returning Null."); return null; } catch (Exception e) { - throw new IOException("Comparator class " + comparatorClassName + - " is not instantiable", e); + throw new IOException("Comparator class " + comparatorClassName + " is not instantiable", e); } } @@ -650,7 +661,7 @@ public class FixedFileTrailer { } public void setUncompressedDataIndexSize( - long uncompressedDataIndexSize) { + long uncompressedDataIndexSize) { expectAtLeastMajorVersion(2); this.uncompressedDataIndexSize = uncompressedDataIndexSize; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index cb6a3520a3e..46cec4af1f8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -1,4 +1,4 @@ -/** +/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -26,7 +26,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.LongAdder; - import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -52,7 +51,6 @@ import org.apache.hadoop.io.Writable; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; @@ -121,7 +119,7 @@ import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; * an HFile instance? */ @InterfaceAudience.Private -public class HFile { +public final class HFile { // LOG is being used in HFileBlock and CheckSumUtil static final Logger LOG = LoggerFactory.getLogger(HFile.class); @@ -177,6 +175,11 @@ public class HFile { /** Static instance for the metrics so that HFileReaders access the same instance */ static final MetricsIO metrics = new MetricsIO(new MetricsIOWrapperImpl()); + /** + * Shutdown constructor. + */ + private HFile() {} + /** * Number of checksum verification failures. It also * clears the counter. @@ -222,10 +225,11 @@ public class HFile { */ void addInlineBlockWriter(InlineBlockWriter bloomWriter); - // The below three methods take Writables. We'd like to undo Writables but undoing the below would be pretty - // painful. Could take a byte [] or a Message but we want to be backward compatible around hfiles so would need - // to map between Message and Writable or byte [] and current Writable serialization. This would be a bit of work - // to little gain. Thats my thinking at moment. St.Ack 20121129 + // The below three methods take Writables. We'd like to undo Writables but undoing the below + // would be pretty painful. Could take a byte [] or a Message but we want to be backward + // compatible around hfiles so would need to map between Message and Writable or byte [] and + // current Writable serialization. This would be a bit of work to little gain. Thats my + // thinking at moment. St.Ack 20121129 void appendMetaBlock(String bloomFilterMetaKey, Writable metaWriter); @@ -258,7 +262,6 @@ public class HFile { protected FileSystem fs; protected Path path; protected FSDataOutputStream ostream; - protected CellComparator comparator = CellComparator.getInstance(); protected InetSocketAddress[] favoredNodes; private HFileContext fileContext; protected boolean shouldDropBehind = false; @@ -282,12 +285,6 @@ public class HFile { return this; } - public WriterFactory withComparator(CellComparator comparator) { - Preconditions.checkNotNull(comparator); - this.comparator = comparator; - return this; - } - public WriterFactory withFavoredNodes(InetSocketAddress[] favoredNodes) { // Deliberately not checking for null here. this.favoredNodes = favoredNodes; @@ -319,7 +316,7 @@ public class HFile { LOG.debug("Unable to set drop behind on {}", path.getName()); } } - return new HFileWriterImpl(conf, cacheConf, path, ostream, comparator, fileContext); + return new HFileWriterImpl(conf, cacheConf, path, ostream, fileContext); } } @@ -349,16 +346,16 @@ public class HFile { CacheConfig cacheConf) { int version = getFormatVersion(conf); switch (version) { - case 2: - throw new IllegalArgumentException("This should never happen. " + - "Did you change hfile.format.version to read v2? This version of the software writes v3" + - " hfiles only (but it can read v2 files without having to update hfile.format.version " + - "in hbase-site.xml)"); - case 3: - return new HFile.WriterFactory(conf, cacheConf); - default: - throw new IllegalArgumentException("Cannot create writer for HFile " + - "format version " + version); + case 2: + throw new IllegalArgumentException("This should never happen. " + + "Did you change hfile.format.version to read v2? This version of the software writes v3" + + " hfiles only (but it can read v2 files without having to update hfile.format.version " + + "in hbase-site.xml)"); + case 3: + return new HFile.WriterFactory(conf, cacheConf); + default: + throw new IllegalArgumentException("Cannot create writer for HFile " + + "format version " + version); } } @@ -372,18 +369,15 @@ public class HFile { * Read in a file block. * @param offset offset to read. * @param onDiskBlockSize size of the block - * @param cacheBlock - * @param pread * @param isCompaction is this block being read as part of a compaction * @param expectedBlockType the block type we are expecting to read with this read operation, - * or null to read whatever block type is available and avoid checking (that might reduce - * caching efficiency of encoded data blocks) + * or null to read whatever block type is available and avoid checking (that might reduce + * caching efficiency of encoded data blocks) * @param expectedDataBlockEncoding the data block encoding the caller is expecting data blocks - * to be in, or null to not perform this check and return the block irrespective of the - * encoding. This check only applies to data blocks and can be set to null when the caller is - * expecting to read a non-data block and has set expectedBlockType accordingly. + * to be in, or null to not perform this check and return the block irrespective of the + * encoding. This check only applies to data blocks and can be set to null when the caller is + * expecting to read a non-data block and has set expectedBlockType accordingly. * @return Block wrapped in a ByteBuffer. - * @throws IOException */ HFileBlock readBlock(long offset, long onDiskBlockSize, boolean cacheBlock, final boolean pread, final boolean isCompaction, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java index 928b7a30d8f..ec317e6f8eb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hbase.io.hfile; import static org.apache.hadoop.hbase.io.ByteBuffAllocator.HEAP; - import java.io.DataInputStream; import java.io.DataOutput; import java.io.DataOutputStream; @@ -29,18 +28,13 @@ import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; - import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.io.ByteBuffAllocator; -import org.apache.hadoop.hbase.io.util.BlockIOUtils; -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.ByteArrayOutputStream; +import org.apache.hadoop.hbase.io.ByteBuffAllocator; import org.apache.hadoop.hbase.io.ByteBuffInputStream; import org.apache.hadoop.hbase.io.ByteBufferWriterDataOutputStream; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; @@ -50,6 +44,7 @@ import org.apache.hadoop.hbase.io.encoding.HFileBlockDecodingContext; import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultDecodingContext; import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; +import org.apache.hadoop.hbase.io.util.BlockIOUtils; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.nio.MultiByteBuff; import org.apache.hadoop.hbase.nio.SingleByteBuff; @@ -57,7 +52,9 @@ import org.apache.hadoop.hbase.regionserver.ShipperListener; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.ClassSize; - +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; @@ -360,8 +357,8 @@ public class HFileBlock implements Cacheable { final long prevBlockOffset = buf.getLong(Header.PREV_BLOCK_OFFSET_INDEX); // This constructor is called when we deserialize a block from cache and when we read a block in // from the fs. fileCache is null when deserialized from cache so need to make up one. - HFileContextBuilder fileContextBuilder = - fileContext != null ? new HFileContextBuilder(fileContext) : new HFileContextBuilder(); + HFileContextBuilder fileContextBuilder = fileContext != null ? + new HFileContextBuilder(fileContext) : new HFileContextBuilder(); fileContextBuilder.withHBaseCheckSum(usesHBaseChecksum); int onDiskDataSizeWithHeader; if (usesHBaseChecksum) { @@ -407,8 +404,8 @@ public class HFileBlock implements Cacheable { /** * @return the on-disk size of the next block (including the header size and any checksums if - * present) read by peeking into the next block's header; use as a hint when doing - * a read of the next block when scanning or running over a file. + * present) read by peeking into the next block's header; use as a hint when doing + * a read of the next block when scanning or running over a file. */ int getNextBlockOnDiskSize() { return nextBlockOnDiskSize; @@ -465,7 +462,7 @@ public class HFileBlock implements Cacheable { /** * @return the uncompressed size of data part (header and checksum excluded). */ - int getUncompressedSizeWithoutHeader() { + int getUncompressedSizeWithoutHeader() { return uncompressedSizeWithoutHeader; } @@ -705,8 +702,8 @@ public class HFileBlock implements Cacheable { } /** - * Cannot be {@link #UNSET}. Must be a legitimate value. Used re-making the {@link BlockCacheKey} when - * block is returned to the cache. + * Cannot be {@link #UNSET}. Must be a legitimate value. Used re-making the {@link BlockCacheKey} + * when block is returned to the cache. * @return the offset of this block in the file it was read from */ long getOffset() { @@ -892,7 +889,6 @@ public class HFileBlock implements Cacheable { * Starts writing into the block. The previous block's data is discarded. * * @return the stream the user can write their data into - * @throws IOException */ DataOutputStream startWriting(BlockType newBlockType) throws IOException { @@ -922,8 +918,6 @@ public class HFileBlock implements Cacheable { /** * Writes the Cell to this block - * @param cell - * @throws IOException */ void write(Cell cell) throws IOException{ expectState(State.WRITING); @@ -1041,9 +1035,6 @@ public class HFileBlock implements Cacheable { * Similar to {@link #writeHeaderAndData(FSDataOutputStream)}, but records * the offset of this block so that it can be referenced in the next block * of the same type. - * - * @param out - * @throws IOException */ void writeHeaderAndData(FSDataOutputStream out) throws IOException { long offset = out.getPos(); @@ -1061,9 +1052,7 @@ public class HFileBlock implements Cacheable { * data when not using compression) into the given stream. Can be called in * the "writing" state or in the "block ready" state. If called in the * "writing" state, transitions the writer to the "block ready" state. - * * @param out the output stream to write the - * @throws IOException */ protected void finishBlockAndWriteHeaderAndData(DataOutputStream out) throws IOException { @@ -1082,7 +1071,6 @@ public class HFileBlock implements Cacheable { * the header + data + checksums stored on disk. * * @return header and data as they would be stored on disk in a byte array - * @throws IOException */ byte[] getHeaderAndDataForTest() throws IOException { ensureBlockReady(); @@ -1167,9 +1155,7 @@ public class HFileBlock implements Cacheable { * @return the number of bytes written */ public int encodedBlockSizeWritten() { - if (state != State.WRITING) - return 0; - return this.encodedDataSizeWritten; + return state != State.WRITING ? 0 : this.encodedDataSizeWritten; } /** @@ -1180,8 +1166,7 @@ public class HFileBlock implements Cacheable { * @return the number of bytes written */ int blockSizeWritten() { - if (state != State.WRITING) return 0; - return this.unencodedDataSizeWritten; + return state != State.WRITING ? 0 : this.unencodedDataSizeWritten; } /** @@ -1234,7 +1219,6 @@ public class HFileBlock implements Cacheable { * * @param bw the block-writable object to write as a block * @param out the file system output stream - * @throws IOException */ void writeBlock(BlockWritable bw, FSDataOutputStream out) throws IOException { @@ -1431,7 +1415,8 @@ public class HFileBlock implements Cacheable { * next blocks header seems unnecessary given we usually get the block size * from the hfile index. Review! */ - private AtomicReference prefetchedHeader = new AtomicReference<>(new PrefetchedHeader()); + private AtomicReference prefetchedHeader = + new AtomicReference<>(new PrefetchedHeader()); /** The size of the file we are reading from, or -1 if unknown. */ private long fileSize; @@ -1643,10 +1628,9 @@ public class HFileBlock implements Cacheable { /** * @return Check onDiskSizeWithHeaderL size is healthy and then return it as an int - * @throws IOException */ private static int checkAndGetSizeAsInt(final long onDiskSizeWithHeaderL, final int hdrSize) - throws IOException { + throws IOException { if ((onDiskSizeWithHeaderL < hdrSize && onDiskSizeWithHeaderL != -1) || onDiskSizeWithHeaderL >= Integer.MAX_VALUE) { throw new IOException("Invalid onDisksize=" + onDiskSizeWithHeaderL @@ -1659,11 +1643,10 @@ public class HFileBlock implements Cacheable { /** * Verify the passed in onDiskSizeWithHeader aligns with what is in the header else something * is not right. - * @throws IOException */ private void verifyOnDiskSizeMatchesHeader(final int passedIn, final ByteBuff headerBuf, - final long offset, boolean verifyChecksum) - throws IOException { + final long offset, boolean verifyChecksum) + throws IOException { // Assert size provided aligns with what is in the header int fromHeader = getOnDiskSizeWithHeader(headerBuf, verifyChecksum); if (passedIn != fromHeader) { @@ -1827,7 +1810,8 @@ public class HFileBlock implements Cacheable { @Override public void setIncludesMemStoreTS(boolean includesMemstoreTS) { - this.fileContext.setIncludesMvcc(includesMemstoreTS); + this.fileContext = new HFileContextBuilder(this.fileContext) + .withIncludesMvcc(includesMemstoreTS).build(); } @Override @@ -2073,7 +2057,7 @@ public class HFileBlock implements Cacheable { /** * @return This HFileBlocks fileContext which will a derivative of the - * fileContext for the file from which this block's data was originally read. + * fileContext for the file from which this block's data was originally read. */ HFileContext getHFileContext() { return this.fileContext; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java index 89b3d34a775..1dd25737953 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java @@ -1,4 +1,4 @@ -/** +/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -33,7 +33,6 @@ import java.util.Objects; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; - import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -52,7 +51,14 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.BytesBytesP import org.apache.hadoop.hbase.shaded.protobuf.generated.HFileProtos; /** - * Metadata for HFile. Conjured by the writer. Read in by the reader. + * Metadata Map of attributes for HFile written out as HFile Trailer. Created by the Writer and + * added to the tail of the file just before close. Metadata includes core attributes such as last + * key seen, comparator used writing the file, etc. Clients can add their own attributes via + * {@link #append(byte[], byte[], boolean)} and they'll be persisted and available at read time. + * Reader creates the HFileInfo on open by reading the tail of the HFile. The parse of the HFile + * trailer also creates a {@link HFileContext}, a read-only data structure that includes bulk of + * the HFileInfo and extras that is safe to pass around when working on HFiles. + * @see HFileContext */ @InterfaceAudience.Private public class HFileInfo implements SortedMap { @@ -62,7 +68,6 @@ public class HFileInfo implements SortedMap { static final byte [] AVG_KEY_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_KEY_LEN"); static final byte [] AVG_VALUE_LEN = Bytes.toBytes(RESERVED_PREFIX + "AVG_VALUE_LEN"); static final byte [] CREATE_TIME_TS = Bytes.toBytes(RESERVED_PREFIX + "CREATE_TIME_TS"); - static final byte [] COMPARATOR = Bytes.toBytes(RESERVED_PREFIX + "COMPARATOR"); static final byte [] TAGS_COMPRESSED = Bytes.toBytes(RESERVED_PREFIX + "TAGS_COMPRESSED"); public static final byte [] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX + "MAX_TAGS_LEN"); private final SortedMap map = new TreeMap<>(Bytes.BYTES_COMPARATOR); @@ -380,7 +385,8 @@ public class HFileInfo implements SortedMap { HFileContextBuilder builder = new HFileContextBuilder() .withHBaseCheckSum(true) .withHFileName(path.getName()) - .withCompression(trailer.getCompressionCodec()); + .withCompression(trailer.getCompressionCodec()) + .withCellComparator(trailer.createComparator(trailer.getComparatorClassName())); // Check for any key material available byte[] keyBytes = trailer.getEncryptionKey(); if (keyBytes != null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index 2416074a4cc..5c51fe0060c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Optional; - import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -49,12 +48,10 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.IdLock; import org.apache.hadoop.hbase.util.ObjectIntPair; import org.apache.hadoop.io.WritableUtils; - import org.apache.htrace.core.TraceScope; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; /** @@ -85,9 +82,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { */ protected HFileDataBlockEncoder dataBlockEncoder = NoOpDataBlockEncoder.INSTANCE; - /** Key comparator */ - protected CellComparator comparator = CellComparator.getInstance(); - /** Block cache configuration. */ protected final CacheConfig cacheConf; @@ -145,8 +139,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { this.primaryReplicaReader = context.isPrimaryReplicaReader(); this.fileInfo = fileInfo; this.trailer = fileInfo.getTrailer(); - // Comparator class name is stored in the trailer in version 2. - this.comparator = trailer.createComparator(); this.hfileContext = fileInfo.getHFileContext(); this.fsBlockReader = new HFileBlock.FSReaderImpl(context, hfileContext, cacheConf.getByteBuffAllocator()); @@ -238,7 +230,7 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { /** @return comparator */ @Override public CellComparator getComparator() { - return comparator; + return this.hfileContext.getCellComparator(); } @VisibleForTesting @@ -398,8 +390,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { } protected void assertSeeked() { - if (!isSeeked()) + if (!isSeeked()) { throw new NotSeekedException(reader.getPath()); + } } @Override @@ -476,7 +469,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { /** * Read mvcc. Does checks to see if we even need to read the mvcc at all. - * @param offsetFromPos */ protected void readMvccVersion(final int offsetFromPos) { // See if we even need to decode mvcc. @@ -493,7 +485,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { /** * Actually do the mvcc read. Does no checks. - * @param offsetFromPos */ private void _readMvccVersion(int offsetFromPos) { // This is Bytes#bytesToVint inlined so can save a few instructions in this hot method; i.e. @@ -681,7 +672,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { * 0 if we are at the given key, 1 if we are past the given key * -2 if the key is earlier than the first key of the file while * using a faked index key - * @throws IOException */ public int seekTo(Cell key, boolean rewind) throws IOException { HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader(); @@ -747,7 +737,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { * data block is found. * * @return the next block, or null if there are no more data blocks - * @throws IOException */ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH", justification="Yeah, unnecessary null check; could do w/ clean up") @@ -764,7 +753,7 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { } if (block.getOffset() < 0) { releaseIfNotCurBlock(block); - throw new IOException("Invalid block file offset: " + block + ", path=" + reader.getPath()); + throw new IOException("Invalid block offset: " + block + ", path=" + reader.getPath()); } // We are reading the next block without block type validation, because // it might turn out to be a non-data block. @@ -786,8 +775,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { @Override public Cell getCell() { - if (!isSeeked()) + if (!isSeeked()) { return null; + } Cell ret; int cellBufSize = getKVBufSize(); @@ -886,7 +876,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { /** * Set our selves up for the next 'next' invocation, set up next block. * @return True is more to read else false if at the end. - * @throws IOException */ private boolean positionForNextBlock() throws IOException { // Methods are small so they get inlined because they are 'hot'. @@ -942,7 +931,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { * * @return false if empty file; i.e. a call to next would return false and * the current key and value are undefined. - * @throws IOException */ @Override public boolean seekTo() throws IOException { @@ -975,7 +963,8 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding()); if (newBlock.getOffset() < 0) { releaseIfNotCurBlock(newBlock); - throw new IOException("Invalid block offset: " + newBlock.getOffset() + ", path=" + reader.getPath()); + throw new IOException("Invalid offset=" + newBlock.getOffset() + + ", path=" + reader.getPath()); } updateCurrentBlock(newBlock); } @@ -993,7 +982,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { } /** - * @param v * @return True if v <= 0 or v > current block buffer limit. */ protected final boolean checkKeyLen(final int v) { @@ -1001,7 +989,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { } /** - * @param v * @return True if v < 0 or v > current block buffer limit. */ protected final boolean checkLen(final int v) { @@ -1172,9 +1159,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { // so blocks with the old encoding still linger in cache for some // period of time. This event should be rare as it only happens on // schema definition change. - LOG.info("Evicting cached block with key " + cacheKey - + " because of a data block encoding mismatch" + "; expected: " - + expectedDataBlockEncoding + ", actual: " + actualDataBlockEncoding + ", path=" + path); + LOG.info("Evicting cached block with key {} because data block encoding mismatch; " + + "expected {}, actual {}, path={}", cacheKey, actualDataBlockEncoding, + expectedDataBlockEncoding, path); // This is an error scenario. so here we need to release the block. returnAndEvictBlock(cache, cacheKey, cachedBlock); } @@ -1192,10 +1179,8 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { } /** - * @param metaBlockName * @param cacheBlock Add block to cache, if found * @return block wrapped in a ByteBuffer, with header skipped - * @throws IOException */ @Override public HFileBlock getMetaBlock(String metaBlockName, boolean cacheBlock) @@ -1210,8 +1195,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { byte[] mbname = Bytes.toBytes(metaBlockName); int block = metaBlockIndexReader.rootBlockContainingKey(mbname, 0, mbname.length); - if (block == -1) + if (block == -1) { return null; + } long blockSize = metaBlockIndexReader.getRootBlockDataSize(block); // Per meta key from any given file, synchronize reads for said block. This @@ -1418,7 +1404,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { /** * @return Midkey for this file. We work with block boundaries only so * returned midkey is an approximation only. - * @throws IOException */ @Override public Optional midKey() throws IOException { @@ -1455,8 +1440,7 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { DataBlockEncoding encoding = reader.getDataBlockEncoding(); dataBlockEncoder = encoding.getEncoder(); decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta); - seeker = dataBlockEncoder.createSeeker( - reader.getComparator(), decodingCtx); + seeker = dataBlockEncoder.createSeeker(decodingCtx); } @Override @@ -1475,7 +1459,6 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { * @param newBlock the block to make current, and read by {@link HFileReaderImpl#readBlock}, * it's a totally new block with new allocated {@link ByteBuff}, so if no further * reference to this block, we should release it carefully. - * @throws CorruptHFileException */ @Override protected void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException { @@ -1487,9 +1470,9 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { short dataBlockEncoderId = newBlock.getDataBlockEncodingId(); if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) { String encoderCls = dataBlockEncoder.getClass().getName(); - throw new CorruptHFileException( - "Encoder " + encoderCls + " doesn't support data block encoding " - + DataBlockEncoding.getNameFromId(dataBlockEncoderId) + ",path=" + reader.getPath()); + throw new CorruptHFileException("Encoder " + encoderCls + + " doesn't support data block encoding " + + DataBlockEncoding.getNameFromId(dataBlockEncoderId) + ",path=" + reader.getPath()); } updateCurrBlockRef(newBlock); ByteBuff encodedBuffer = getEncodedBuffer(newBlock); @@ -1606,7 +1589,7 @@ public abstract class HFileReaderImpl implements HFile.Reader, Configurable { } private DataInput getBloomFilterMetadata(BlockType blockType) - throws IOException { + throws IOException { if (blockType != BlockType.GENERAL_BLOOM_META && blockType != BlockType.DELETE_FAMILY_BLOOM_META) { throw new RuntimeException("Block Type: " + blockType.toString() + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java index 66a6c0065ec..604ac1f8bc6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterImpl.java @@ -25,7 +25,6 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -65,7 +64,8 @@ public class HFileWriterImpl implements HFile.Writer { private static final long UNSET = -1; /** if this feature is enabled, preCalculate encoded data size before real encoding happens*/ - public static final String UNIFIED_ENCODED_BLOCKSIZE_RATIO = "hbase.writer.unified.encoded.blocksize.ratio"; + public static final String UNIFIED_ENCODED_BLOCKSIZE_RATIO = + "hbase.writer.unified.encoded.blocksize.ratio"; /** Block size limit after encoding, used to unify encoded block Cache entry size*/ private final int encodedBlockSizeLimit; @@ -94,9 +94,6 @@ public class HFileWriterImpl implements HFile.Writer { /** Total uncompressed bytes, maybe calculate a compression ratio later. */ protected long totalUncompressedBytes = 0; - /** Key comparator. Used to ensure we write in order. */ - protected final CellComparator comparator; - /** Meta block names. */ protected List metaNames = new ArrayList<>(); @@ -165,8 +162,7 @@ public class HFileWriterImpl implements HFile.Writer { protected long maxMemstoreTS = 0; public HFileWriterImpl(final Configuration conf, CacheConfig cacheConf, Path path, - FSDataOutputStream outputStream, - CellComparator comparator, HFileContext fileContext) { + FSDataOutputStream outputStream, HFileContext fileContext) { this.outputStream = outputStream; this.path = path; this.name = path != null ? path.getName() : outputStream.toString(); @@ -177,8 +173,6 @@ public class HFileWriterImpl implements HFile.Writer { } else { this.blockEncoder = NoOpDataBlockEncoder.INSTANCE; } - this.comparator = comparator != null ? comparator : CellComparator.getInstance(); - closeOutputStream = path != null; this.cacheConf = cacheConf; float encodeBlockSizeRatio = conf.getFloat(UNIFIED_ENCODED_BLOCKSIZE_RATIO, 1f); @@ -187,7 +181,6 @@ public class HFileWriterImpl implements HFile.Writer { if (LOG.isTraceEnabled()) { LOG.trace("Writer" + (path != null ? " for " + path : "") + " initialized with cacheConf: " + cacheConf + - " comparator: " + comparator.getClass().getSimpleName() + " fileContext: " + fileContext); } } @@ -214,10 +207,9 @@ public class HFileWriterImpl implements HFile.Writer { * * @param trailer fixed file trailer * @param out the data output to write the file info to - * @throws IOException */ protected final void writeFileInfo(FixedFileTrailer trailer, DataOutputStream out) - throws IOException { + throws IOException { trailer.setFileInfoOffset(outputStream.getPos()); finishFileInfo(); long startTime = System.currentTimeMillis(); @@ -239,7 +231,8 @@ public class HFileWriterImpl implements HFile.Writer { throw new IOException("Key cannot be null or empty"); } if (lastCell != null) { - int keyComp = PrivateCellUtil.compareKeyIgnoresMvcc(comparator, lastCell, cell); + int keyComp = PrivateCellUtil.compareKeyIgnoresMvcc(this.hFileContext.getCellComparator(), + lastCell, cell); if (keyComp > 0) { String message = getLexicalErrorMessage(cell); throw new IOException(message); @@ -285,8 +278,9 @@ public class HFileWriterImpl implements HFile.Writer { } public static Compression.Algorithm compressionByName(String algoName) { - if (algoName == null) + if (algoName == null) { return HFile.DEFAULT_COMPRESSION_ALGORITHM; + } return Compression.getCompressionAlgorithmByName(algoName); } @@ -318,17 +312,15 @@ public class HFileWriterImpl implements HFile.Writer { // Meta data block index writer metaBlockIndexWriter = new HFileBlockIndex.BlockIndexWriter(); - if (LOG.isTraceEnabled()) LOG.trace("Initialized with " + cacheConf); + LOG.trace("Initialized with {}", cacheConf); } /** * At a block boundary, write all the inline blocks and opens new block. - * - * @throws IOException */ protected void checkBlockBoundary() throws IOException { - //for encoder like prefixTree, encoded size is not available, so we have to compare both encoded size - //and unencoded size to blocksize limit. + // For encoder like prefixTree, encoded size is not available, so we have to compare both + // encoded size and unencoded size to blocksize limit. if (blockWriter.encodedBlockSizeWritten() >= encodedBlockSizeLimit || blockWriter.blockSizeWritten() >= hFileContext.getBlocksize()) { finishBlock(); @@ -339,7 +331,9 @@ public class HFileWriterImpl implements HFile.Writer { /** Clean up the data block that is currently being written.*/ private void finishBlock() throws IOException { - if (!blockWriter.isWriting() || blockWriter.blockSizeWritten() == 0) return; + if (!blockWriter.isWriting() || blockWriter.blockSizeWritten() == 0) { + return; + } // Update the first data block offset if UNSET; used scanning. if (firstDataBlockOffset == UNSET) { @@ -350,7 +344,7 @@ public class HFileWriterImpl implements HFile.Writer { blockWriter.writeHeaderAndData(outputStream); int onDiskSize = blockWriter.getOnDiskSizeWithHeader(); Cell indexEntry = - getMidpoint(this.comparator, lastCellOfPreviousBlock, firstCellInBlock); + getMidpoint(this.hFileContext.getCellComparator(), lastCellOfPreviousBlock, firstCellInBlock); dataBlockIndexWriter.addEntry(PrivateCellUtil.getCellKeySerializedAsKeyValueKey(indexEntry), lastDataBlockOffset, onDiskSize); totalUncompressedBytes += blockWriter.getUncompressedSizeWithHeader(); @@ -364,11 +358,6 @@ public class HFileWriterImpl implements HFile.Writer { * right but that is shorter; i.e. takes up less space. This * trick is used building HFile block index. Its an optimization. It does not * always work. In this case we'll just return the right cell. - * - * @param comparator - * Comparator to use. - * @param left - * @param right * @return A cell that sorts between left and right. */ public static Cell getMidpoint(final CellComparator comparator, final Cell left, @@ -407,7 +396,9 @@ public class HFileWriterImpl implements HFile.Writer { left.getRowLength(), right.getRowArray(), right.getRowOffset(), right.getRowLength()); } // If midRow is null, just return 'right'. Can't do optimization. - if (midRow == null) return right; + if (midRow == null) { + return right; + } return PrivateCellUtil.createFirstOnRow(midRow); } // Rows are same. Compare on families. @@ -428,7 +419,9 @@ public class HFileWriterImpl implements HFile.Writer { right.getFamilyLength()); } // If midRow is null, just return 'right'. Can't do optimization. - if (midRow == null) return right; + if (midRow == null) { + return right; + } // Return new Cell where we use right row and then a mid sort family. return PrivateCellUtil.createFirstOnRowFamily(right, midRow, 0, midRow.length); } @@ -450,7 +443,9 @@ public class HFileWriterImpl implements HFile.Writer { right.getQualifierLength()); } // If midRow is null, just return 'right'. Can't do optimization. - if (midRow == null) return right; + if (midRow == null) { + return right; + } // Return new Cell where we use right row and family and then a mid sort qualifier. return PrivateCellUtil.createFirstOnRowCol(right, midRow, 0, midRow.length); } @@ -459,12 +454,6 @@ public class HFileWriterImpl implements HFile.Writer { } /** - * @param leftArray - * @param leftOffset - * @param leftLength - * @param rightArray - * @param rightOffset - * @param rightLength * @return Return a new array that is between left and right and minimally * sized else just return null as indicator that we could not create a * mid point. @@ -566,8 +555,6 @@ public class HFileWriterImpl implements HFile.Writer { /** * Ready a new block for writing. - * - * @throws IOException */ protected void newBlock() throws IOException { // This is where the next block begins. @@ -678,7 +665,7 @@ public class HFileWriterImpl implements HFile.Writer { dataBlockIndexWriter.getTotalUncompressedSize()); trailer.setFirstDataBlockOffset(firstDataBlockOffset); trailer.setLastDataBlockOffset(lastDataBlockOffset); - trailer.setComparatorClass(comparator.getClass()); + trailer.setComparatorClass(this.hFileContext.getCellComparator().getClass()); trailer.setDataIndexCount(dataBlockIndexWriter.getNumRootEntries()); @@ -704,8 +691,9 @@ public class HFileWriterImpl implements HFile.Writer { private void addBloomFilter(final BloomFilterWriter bfw, final BlockType blockType) { - if (bfw.getKeyCount() <= 0) + if (bfw.getKeyCount() <= 0) { return; + } if (blockType != BlockType.GENERAL_BLOOM_META && blockType != BlockType.DELETE_FAMILY_BLOOM_META) { @@ -722,8 +710,9 @@ public class HFileWriterImpl implements HFile.Writer { public void writeToBlock(DataOutput out) throws IOException { bfw.getMetaWriter().write(out); Writable dataWriter = bfw.getDataWriter(); - if (dataWriter != null) + if (dataWriter != null) { dataWriter.write(out); + } } }); } @@ -739,7 +728,6 @@ public class HFileWriterImpl implements HFile.Writer { * * @param cell * Cell to add. Cannot be empty nor null. - * @throws IOException */ @Override public void append(final Cell cell) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java index 1d364461ec8..1c00e25be3c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobUtils.java @@ -1,4 +1,4 @@ -/** +/* * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -35,13 +35,11 @@ import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.TableName; @@ -124,7 +122,6 @@ public final class MobUtils { * Parses the string to a date. * @param dateString The string format of a date, it's yyyymmdd. * @return A date. - * @throws ParseException */ public static Date parseDate(String dateString) throws ParseException { return LOCAL_FORMAT.get().parse(dateString); @@ -288,7 +285,6 @@ public final class MobUtils { * @param columnDescriptor The descriptor of the current column family. * @param cacheConfig The cacheConfig that disables the block cache. * @param current The current time. - * @throws IOException */ public static void cleanExpiredMobFiles(FileSystem fs, Configuration conf, TableName tableName, ColumnFamilyDescriptor columnDescriptor, CacheConfig cacheConfig, long current) @@ -384,7 +380,6 @@ public final class MobUtils { * Gets the qualified root dir of the mob files. * @param conf The current configuration. * @return The qualified root dir. - * @throws IOException */ public static Path getQualifiedMobRootDir(Configuration conf) throws IOException { Path hbaseDir = new Path(conf.get(HConstants.HBASE_DIR)); @@ -505,7 +500,6 @@ public final class MobUtils { * @param tableDir The table directory. * @param family The name of the column family. * @param storeFiles The files to be deleted. - * @throws IOException */ public static void removeMobFiles(Configuration conf, FileSystem fs, TableName tableName, Path tableDir, byte[] family, Collection storeFiles) throws IOException { @@ -556,7 +550,6 @@ public final class MobUtils { * @param cryptoContext The encryption context. * @param isCompaction If the writer is used in compaction. * @return The writer for the mob file. - * @throws IOException */ public static StoreFileWriter createWriter(Configuration conf, FileSystem fs, ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount, @@ -580,7 +573,6 @@ public final class MobUtils { * @param cryptoContext The encryption context. * @param isCompaction If the writer is used in compaction. * @return The writer for the mob file. - * @throws IOException */ public static StoreFileWriter createRefFileWriter(Configuration conf, FileSystem fs, ColumnFamilyDescriptor family, Path basePath, long maxKeyCount, CacheConfig cacheConfig, @@ -607,7 +599,6 @@ public final class MobUtils { * @param cryptoContext The encryption context. * @param isCompaction If the writer is used in compaction. * @return The writer for the mob file. - * @throws IOException */ public static StoreFileWriter createWriter(Configuration conf, FileSystem fs, ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount, @@ -633,7 +624,6 @@ public final class MobUtils { * @param cacheConfig The current cache config. * @param cryptoContext The encryption context. * @return The writer for the del file. - * @throws IOException */ public static StoreFileWriter createDelFileWriter(Configuration conf, FileSystem fs, ColumnFamilyDescriptor family, String date, Path basePath, long maxKeyCount, @@ -660,10 +650,9 @@ public final class MobUtils { * @param cryptoContext The encryption context. * @param isCompaction If the writer is used in compaction. * @return The writer for the mob file. - * @throws IOException */ public static StoreFileWriter createWriter(Configuration conf, FileSystem fs, - ColumnFamilyDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount, + ColumnFamilyDescriptor family, MobFileName mobFileName, Path basePath, long maxKeyCount, Compression.Algorithm compression, CacheConfig cacheConfig, Encryption.Context cryptoContext, boolean isCompaction) throws IOException { @@ -689,7 +678,6 @@ public final class MobUtils { * @param bloomType The bloom filter type. * @param isCompaction If the writer is used in compaction. * @return The writer for the mob file. - * @throws IOException */ public static StoreFileWriter createWriter(Configuration conf, FileSystem fs, ColumnFamilyDescriptor family, Path path, long maxKeyCount, @@ -718,8 +706,7 @@ public final class MobUtils { .withCreateTime(EnvironmentEdgeManager.currentTime()).build(); StoreFileWriter w = new StoreFileWriter.Builder(conf, writerCacheConf, fs) - .withFilePath(path) - .withComparator(CellComparator.getInstance()).withBloomType(bloomType) + .withFilePath(path).withBloomType(bloomType) .withMaxKeyCount(maxKeyCount).withFileContext(hFileContext).build(); return w; } @@ -732,7 +719,6 @@ public final class MobUtils { * @param targetPath The directory path where the source file is renamed to. * @param cacheConfig The current cache config. * @return The target file path the source file is renamed to. - * @throws IOException */ public static Path commitFile(Configuration conf, FileSystem fs, final Path sourceFile, Path targetPath, CacheConfig cacheConfig) throws IOException { @@ -837,7 +823,8 @@ public final class MobUtils { * @param allFiles Whether add all mob files into the compaction. */ public static void doMobCompaction(Configuration conf, FileSystem fs, TableName tableName, - ColumnFamilyDescriptor hcd, ExecutorService pool, boolean allFiles, LockManager.MasterLock lock) + ColumnFamilyDescriptor hcd, ExecutorService pool, boolean allFiles, + LockManager.MasterLock lock) throws IOException { String className = conf.get(MobConstants.MOB_COMPACTOR_CLASS_KEY, PartitionedMobCompactor.class.getName()); @@ -943,7 +930,8 @@ public final class MobUtils { * @param fileDate The date string parsed from the mob file name. * @return True if the mob file is expired. */ - public static boolean isMobFileExpired(ColumnFamilyDescriptor column, long current, String fileDate) { + public static boolean isMobFileExpired(ColumnFamilyDescriptor column, long current, + String fileDate) { if (column.getMinVersions() > 0) { return false; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index dbc5876ae59..fcdcb2c5660 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -52,7 +52,6 @@ import java.util.function.Predicate; import java.util.function.ToLongFunction; import java.util.stream.Collectors; import java.util.stream.LongStream; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -132,7 +131,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDes * not be called directly but by an HRegion manager. */ @InterfaceAudience.Private -public class HStore implements Store, HeapSize, StoreConfigInformation, PropagatingConfigurationObserver { +public class HStore implements Store, HeapSize, StoreConfigInformation, + PropagatingConfigurationObserver { public static final String MEMSTORE_CLASS_NAME = "hbase.regionserver.memstore.class"; public static final String COMPACTCHECKER_INTERVAL_MULTIPLIER_KEY = "hbase.server.compactchecker.interval.multiplier"; @@ -237,11 +237,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat /** * Constructor - * @param region * @param family HColumnDescriptor for this column - * @param confParam configuration object - * failed. Can be null. - * @throws IOException + * @param confParam configuration object failed. Can be null. */ protected HStore(final HRegion region, final ColumnFamilyDescriptor family, final Configuration confParam, boolean warmup) throws IOException { @@ -399,7 +396,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat } /** - * @param family * @return TTL in seconds of the specified family */ public static long determineTTLFromFamily(final ColumnFamilyDescriptor family) { @@ -521,6 +517,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat * @param hri {@link RegionInfo} for the region. * @param family {@link ColumnFamilyDescriptor} describing the column family * @return Path to family/Store home directory. + * @deprecated Since 05/05/2013, HBase-7808, hbase-1.0.0 */ @Deprecated public static Path getStoreHomedir(final Path tabledir, @@ -533,6 +530,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat * @param encodedName Encoded region name. * @param family {@link ColumnFamilyDescriptor} describing the column family * @return Path to family/Store home directory. + * @deprecated Since 05/05/2013, HBase-7808, hbase-1.0.0 */ @Deprecated public static Path getStoreHomedir(final Path tabledir, @@ -558,7 +556,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat /** * Creates an unsorted list of StoreFile loaded in parallel * from the given directory. - * @throws IOException */ private List loadStoreFiles(boolean warmup) throws IOException { Collection files = fs.getStoreFiles(getColumnFamilyName()); @@ -574,7 +571,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat ThreadPoolExecutor storeFileOpenerThreadPool = this.region.getStoreFileOpenAndCloseThreadPool("StoreFileOpenerThread-" + this.region.getRegionInfo().getEncodedName() + "-" + this.getColumnFamilyName()); - CompletionService completionService = new ExecutorCompletionService<>(storeFileOpenerThreadPool); + CompletionService completionService = + new ExecutorCompletionService<>(storeFileOpenerThreadPool); int totalValidStoreFile = 0; for (StoreFileInfo storeFileInfo : files) { @@ -596,9 +594,13 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat compactedStoreFiles.addAll(storeFile.getCompactedStoreFiles()); } } catch (InterruptedException e) { - if (ioe == null) ioe = new InterruptedIOException(e.getMessage()); + if (ioe == null) { + ioe = new InterruptedIOException(e.getMessage()); + } } catch (ExecutionException e) { - if (ioe == null) ioe = new IOException(e.getCause()); + if (ioe == null) { + ioe = new IOException(e.getCause()); + } } } } finally { @@ -650,7 +652,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat /** * Replaces the store files that the store has with the given files. Mainly used by secondary * region replicas to keep up to date with the primary region files. - * @throws IOException */ public void refreshStoreFiles(Collection newFiles) throws IOException { List storeFiles = new ArrayList<>(newFiles.size()); @@ -665,15 +666,20 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat * been opened, and removes the store file readers for store files no longer * available. Mainly used by secondary region replicas to keep up to date with * the primary region files. - * @throws IOException */ private void refreshStoreFilesInternal(Collection newFiles) throws IOException { StoreFileManager sfm = storeEngine.getStoreFileManager(); Collection currentFiles = sfm.getStorefiles(); Collection compactedFiles = sfm.getCompactedfiles(); - if (currentFiles == null) currentFiles = Collections.emptySet(); - if (newFiles == null) newFiles = Collections.emptySet(); - if (compactedFiles == null) compactedFiles = Collections.emptySet(); + if (currentFiles == null) { + currentFiles = Collections.emptySet(); + } + if (newFiles == null) { + newFiles = Collections.emptySet(); + } + if (compactedFiles == null) { + compactedFiles = Collections.emptySet(); + } HashMap currentFilesSet = new HashMap<>(currentFiles.size()); for (HStoreFile sf : currentFiles) { @@ -868,12 +874,13 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat } prevCell = cell; } while (scanner.next()); - LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString() - + " took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime) - + " ms"); + LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString() + + " took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime) + " ms"); } } finally { - if (reader != null) reader.close(); + if (reader != null) { + reader.close(); + } } } @@ -881,7 +888,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat * This method should only be called from Region. It is assumed that the ranges of values in the * HFile fit within the stores assigned region. (assertBulkLoadHFileOk checks this) * - * @param srcPathStr * @param seqNum sequence Id associated with the HFile */ public Pair preBulkLoadHFile(String srcPathStr, long seqNum) throws IOException { @@ -993,13 +999,17 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat ioe.initCause(e); } } catch (ExecutionException e) { - if (ioe == null) ioe = new IOException(e.getCause()); + if (ioe == null) { + ioe = new IOException(e.getCause()); + } } } } finally { storeFileCloserThreadPool.shutdownNow(); } - if (ioe != null) throw ioe; + if (ioe != null) { + throw ioe; + } } LOG.trace("Closed {}", this); return result; @@ -1027,9 +1037,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat /** * Write out current snapshot. Presumes {@link #snapshot()} has been called previously. * @param logCacheFlushId flush sequence number - * @param snapshot - * @param status - * @param throughputController * @return The path name of the tmp file to which the store was flushed * @throws IOException if exception occurs during process */ @@ -1081,10 +1088,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat /** * @param path The pathname of the tmp file into which the store was flushed - * @param logCacheFlushId - * @param status * @return store file created. - * @throws IOException */ private HStoreFile commitFile(Path path, long logCacheFlushId, MonitoredTask status) throws IOException { @@ -1107,7 +1111,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat } /** - * @param maxKeyCount * @param compression Compression algorithm to use * @param isCompaction whether we are creating a new file in a compaction * @param includeMVCCReadpoint - whether to include MVCC or not @@ -1162,7 +1165,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf, this.getFileSystem()) .withOutputDir(familyTempDir) - .withComparator(comparator) .withBloomType(family.getBloomFilterType()) .withMaxKeyCount(maxKeyCount) .withFavoredNodes(favoredNodes) @@ -1192,6 +1194,7 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat .withColumnFamily(family.getName()) .withTableName(region.getTableDescriptor() .getTableName().getName()) + .withCellComparator(this.comparator) .build(); return hFileContext; } @@ -1204,8 +1207,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat /** * Change storeFiles adding into place the Reader produced by this new flush. * @param sfs Store files - * @param snapshotId - * @throws IOException * @return Whether compaction is required. */ private boolean updateStorefiles(List sfs, long snapshotId) throws IOException { @@ -1237,7 +1238,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat /** * Notify all observers that set of Readers has changed. - * @throws IOException */ private void notifyChangedReadersObservers(List sfs) throws IOException { for (ChangedReadersObserver o : this.changedReaderObservers) { @@ -1454,7 +1454,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat * See HBASE-2231 for details. * * @param compaction compaction details obtained from requestCompaction() - * @throws IOException * @return Storefile we compacted into or null if we failed or opted out early. */ public List compact(CompactionContext compaction, @@ -1521,8 +1520,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat return sfs; } - private List moveCompactedFilesIntoPlace(CompactionRequestImpl cr, List newFiles, - User user) throws IOException { + private List moveCompactedFilesIntoPlace(CompactionRequestImpl cr, + List newFiles, User user) throws IOException { List sfs = new ArrayList<>(newFiles.size()); for (Path newFile : newFiles) { assert newFile != null; @@ -1560,7 +1559,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat newFiles.stream().map(HStoreFile::getPath).collect(Collectors.toList()); RegionInfo info = this.region.getRegionInfo(); CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(info, - family.getName(), inputPaths, outputPaths, fs.getStoreDir(getColumnFamilyDescriptor().getNameAsString())); + family.getName(), inputPaths, outputPaths, + fs.getStoreDir(getColumnFamilyDescriptor().getNameAsString())); // Fix reaching into Region to get the maxWaitForSeqId. // Does this method belong in Region altogether given it is making so many references up there? // Could be Region#writeCompactionMarker(compactionDescriptor); @@ -1664,7 +1664,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat * Call to complete a compaction. Its for the case where we find in the WAL a compaction * that was not finished. We could find one recovering a WAL after a regionserver crash. * See HBASE-2231. - * @param compaction */ public void replayCompactionMarker(CompactionDescriptor compaction, boolean pickCompactionFiles, boolean removeFiles) throws IOException { @@ -1917,7 +1916,9 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat } private void removeUnneededFiles() throws IOException { - if (!conf.getBoolean("hbase.store.delete.expired.storefile", true)) return; + if (!conf.getBoolean("hbase.store.delete.expired.storefile", true)) { + return; + } if (getColumnFamilyDescriptor().getMinVersions() > 0) { LOG.debug("Skipping expired store file removal due to min version being {}", getColumnFamilyDescriptor().getMinVersions()); @@ -2120,7 +2121,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat * @param readPt the read point of the current scane * @param includeMemstoreScanner whether the current scanner should include memstorescanner * @return list of scanners recreated on the current Scanners - * @throws IOException */ public List recreateScanners(List currentFileScanners, boolean cacheBlocks, boolean usePread, boolean isCompaction, ScanQueryMatcher matcher, @@ -2319,7 +2319,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat * This operation is atomic on each KeyValue (row/family/qualifier) but not necessarily atomic * across all of them. * @param readpoint readpoint below which we can safely remove duplicate KVs - * @throws IOException */ public void upsert(Iterable cells, long readpoint, MemStoreSizing memstoreSizing) throws IOException { @@ -2433,7 +2432,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat * snapshot depending on dropMemstoreSnapshot argument. * @param fileNames names of the flushed files * @param dropMemstoreSnapshot whether to drop the prepared memstore snapshot - * @throws IOException */ @Override public void replayFlush(List fileNames, boolean dropMemstoreSnapshot) @@ -2465,7 +2463,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat /** * Abort the snapshot preparation. Drops the snapshot if any. - * @throws IOException */ @Override public void abort() throws IOException { @@ -2720,7 +2717,8 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat LOG.debug("Moving the files {} to archive", filesToRemove); // Only if this is successful it has to be removed try { - this.fs.removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(), filesToRemove); + this.fs.removeStoreFiles(this.getColumnFamilyDescriptor().getNameAsString(), + filesToRemove); } catch (FailedArchiveException fae) { // Even if archiving some files failed, we still need to clear out any of the // files which were successfully archived. Otherwise we will receive a diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java index 7b4fb4a0c06..02665a34541 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java @@ -26,7 +26,6 @@ import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAJOR_COMPACTION_K import static org.apache.hadoop.hbase.regionserver.HStoreFile.MAX_SEQ_ID_KEY; import static org.apache.hadoop.hbase.regionserver.HStoreFile.MOB_CELLS_COUNT; import static org.apache.hadoop.hbase.regionserver.HStoreFile.TIMERANGE_KEY; - import java.io.IOException; import java.net.InetSocketAddress; import java.util.Collection; @@ -37,15 +36,13 @@ import java.util.UUID; import java.util.function.Supplier; import java.util.regex.Pattern; import java.util.stream.Collectors; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.PrivateCellUtil; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; @@ -62,9 +59,7 @@ import org.apache.hadoop.hbase.util.RowPrefixFixedLengthBloomContext; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; - import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; /** @@ -94,7 +89,6 @@ public class StoreFileWriter implements CellSink, ShipperListener { * @param fs file system to write to * @param path file name to create * @param conf user configuration - * @param comparator key comparator * @param bloomType bloom filter setting * @param maxKeys the expected maximum number of keys to be added. Was used * for Bloom filter size in {@link HFile} format version 1. @@ -105,15 +99,14 @@ public class StoreFileWriter implements CellSink, ShipperListener { * @throws IOException problem writing to FS */ private StoreFileWriter(FileSystem fs, Path path, final Configuration conf, CacheConfig cacheConf, - final CellComparator comparator, BloomType bloomType, long maxKeys, - InetSocketAddress[] favoredNodes, HFileContext fileContext, boolean shouldDropCacheBehind, - Supplier> compactedFilesSupplier) throws IOException { + BloomType bloomType, long maxKeys, InetSocketAddress[] favoredNodes, HFileContext fileContext, + boolean shouldDropCacheBehind, Supplier> compactedFilesSupplier) + throws IOException { this.compactedFilesSupplier = compactedFilesSupplier; this.timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC); // TODO : Change all writers to be specifically created for compaction context writer = HFile.getWriterFactory(conf, cacheConf) .withPath(fs, path) - .withComparator(comparator) .withFavoredNodes(favoredNodes) .withFileContext(fileContext) .withShouldDropCacheBehind(shouldDropCacheBehind) @@ -135,14 +128,16 @@ public class StoreFileWriter implements CellSink, ShipperListener { // init bloom context switch (bloomType) { case ROW: - bloomContext = new RowBloomContext(generalBloomFilterWriter, comparator); + bloomContext = + new RowBloomContext(generalBloomFilterWriter, fileContext.getCellComparator()); break; case ROWCOL: - bloomContext = new RowColBloomContext(generalBloomFilterWriter, comparator); + bloomContext = + new RowColBloomContext(generalBloomFilterWriter, fileContext.getCellComparator()); break; case ROWPREFIX_FIXED_LENGTH: - bloomContext = new RowPrefixFixedLengthBloomContext(generalBloomFilterWriter, comparator, - Bytes.toInt(bloomParam)); + bloomContext = new RowPrefixFixedLengthBloomContext(generalBloomFilterWriter, + fileContext.getCellComparator(), Bytes.toInt(bloomParam)); break; default: throw new IOException( @@ -159,7 +154,8 @@ public class StoreFileWriter implements CellSink, ShipperListener { this.deleteFamilyBloomFilterWriter = BloomFilterFactory .createDeleteBloomAtWrite(conf, cacheConf, (int) Math.min(maxKeys, Integer.MAX_VALUE), writer); - deleteFamilyBloomContext = new RowBloomContext(deleteFamilyBloomFilterWriter, comparator); + deleteFamilyBloomContext = + new RowBloomContext(deleteFamilyBloomFilterWriter, fileContext.getCellComparator()); } else { deleteFamilyBloomFilterWriter = null; } @@ -396,7 +392,6 @@ public class StoreFileWriter implements CellSink, ShipperListener { } /** - * @param fs * @param dir Directory to create file in. * @return random filename inside passed dir */ @@ -414,7 +409,6 @@ public class StoreFileWriter implements CellSink, ShipperListener { private final CacheConfig cacheConf; private final FileSystem fs; - private CellComparator comparator = CellComparator.getInstance(); private BloomType bloomType = BloomType.NONE; private long maxKeyCount = 0; private Path dir; @@ -473,12 +467,6 @@ public class StoreFileWriter implements CellSink, ShipperListener { return this; } - public Builder withComparator(CellComparator comparator) { - Preconditions.checkNotNull(comparator); - this.comparator = comparator; - return this; - } - public Builder withBloomType(BloomType bloomType) { Preconditions.checkNotNull(bloomType); this.bloomType = bloomType; @@ -545,11 +533,7 @@ public class StoreFileWriter implements CellSink, ShipperListener { } } - if (comparator == null) { - comparator = CellComparator.getInstance(); - } - - return new StoreFileWriter(fs, filePath, conf, cacheConf, comparator, bloomType, maxKeyCount, + return new StoreFileWriter(fs, filePath, conf, cacheConf, bloomType, maxKeyCount, favoredNodes, fileContext, shouldDropCacheBehind, compactedFilesSupplier); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java index 96ab8e62b31..406e90d8d19 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/BoundedRecoveredHFilesOutputSink.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hbase.wal; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; - import java.io.IOException; import java.io.InterruptedIOException; import java.util.HashMap; @@ -29,7 +28,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -208,7 +206,7 @@ public class BoundedRecoveredHFilesOutputSink extends OutputSink { .withFilePath(outputFile); HFileContextBuilder hFileContextBuilder = new HFileContextBuilder(); if (isMetaTable) { - writerBuilder.withComparator(CellComparatorImpl.META_COMPARATOR); + hFileContextBuilder.withCellComparator(CellComparatorImpl.META_COMPARATOR); } else { configContextForNonMetaWriter(tableName, familyName, hFileContextBuilder, writerBuilder); } @@ -224,9 +222,9 @@ public class BoundedRecoveredHFilesOutputSink extends OutputSink { TableDescriptor tableDesc = tableDescCache.get(tableName); ColumnFamilyDescriptor cfd = tableDesc.getColumnFamily(Bytes.toBytesBinary(familyName)); hFileContextBuilder.withCompression(cfd.getCompressionType()).withBlockSize(cfd.getBlocksize()) - .withCompressTags(cfd.isCompressTags()).withDataBlockEncoding(cfd.getDataBlockEncoding()); - writerBuilder.withBloomType(cfd.getBloomFilterType()) - .withComparator(CellComparatorImpl.COMPARATOR); + .withCompressTags(cfd.isCompressTags()).withDataBlockEncoding(cfd.getDataBlockEncoding()) + .withCellComparator(CellComparatorImpl.COMPARATOR); + writerBuilder.withBloomType(cfd.getBloomFilterType()); } private void checkPathValid(Path outputFile) throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java index 90fb8a3ca9b..2c4209ce8db 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java @@ -380,7 +380,6 @@ public class HFilePerformanceEvaluation { writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, mf) .withFileContext(hFileContext) - .withComparator(CellComparator.getInstance()) .create(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java index 569ede298f1..3f21f0c2fda 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -206,8 +206,8 @@ public class TestDataBlockEncoders { .withIncludesTags(includesTags) .withCompression(Compression.Algorithm.NONE) .build(); - DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR, - encoder.newDataBlockDecodingContext(meta)); + DataBlockEncoder.EncodedSeeker seeker = + encoder.createSeeker(encoder.newDataBlockDecodingContext(meta)); seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer)); encodedSeekers.add(seeker); } @@ -279,8 +279,8 @@ public class TestDataBlockEncoders { .withIncludesTags(includesTags) .withCompression(Compression.Algorithm.NONE) .build(); - DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR, - encoder.newDataBlockDecodingContext(meta)); + DataBlockEncoder.EncodedSeeker seeker = + encoder.createSeeker(encoder.newDataBlockDecodingContext(meta)); seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer)); int i = 0; do { @@ -341,8 +341,8 @@ public class TestDataBlockEncoders { HFileContext meta = new HFileContextBuilder().withHBaseCheckSum(false).withIncludesMvcc(includesMemstoreTS) .withIncludesTags(includesTags).withCompression(Compression.Algorithm.NONE).build(); - DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR, - encoder.newDataBlockDecodingContext(meta)); + DataBlockEncoder.EncodedSeeker seeker = + encoder.createSeeker(encoder.newDataBlockDecodingContext(meta)); seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer)); Cell cell = seeker.getCell(); Assert.assertEquals(expectedKV.getLength(), ((KeyValue) cell).getLength()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekToBlockWithEncoders.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekToBlockWithEncoders.java index e7b14263398..d9f22bc245e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekToBlockWithEncoders.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestSeekToBlockWithEncoders.java @@ -25,7 +25,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -290,8 +289,8 @@ public class TestSeekToBlockWithEncoders { HFILEBLOCK_DUMMY_HEADER, meta); ByteBuffer encodedBuffer = TestDataBlockEncoders.encodeKeyValues(encoding, kvs, encodingContext, this.useOffheapData); - DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR, - encoder.newDataBlockDecodingContext(meta)); + DataBlockEncoder.EncodedSeeker seeker = + encoder.createSeeker(encoder.newDataBlockDecodingContext(meta)); seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer)); encodedSeekers.add(seeker); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java index cb7042adefd..609ff9d3c09 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java @@ -37,7 +37,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ArrayBackedTag; -import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -394,7 +393,7 @@ public class TestCacheOnWrite { .withDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding()) .withIncludesTags(useTags).build(); StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs) - .withOutputDir(storeFileParentDir).withComparator(CellComparatorImpl.COMPARATOR) + .withOutputDir(storeFileParentDir) .withFileContext(meta) .withBloomType(BLOOM_TYPE).withMaxKeyCount(NUM_KV).build(); byte[] cf = Bytes.toBytes("fam"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java index d25ce4796d7..14006a822cf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java @@ -131,6 +131,10 @@ public class TestFixedFileTrailer { t.createComparator(KeyValue.META_COMPARATOR.getClass().getName()).getClass()); assertEquals(CellComparatorImpl.MetaCellComparator.class, t.createComparator( CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass()); + assertEquals(CellComparatorImpl.META_COMPARATOR.getClass(), t.createComparator( + CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass().getName()).getClass()); + assertEquals(CellComparatorImpl.COMPARATOR.getClass(), t.createComparator( + CellComparatorImpl.MetaCellComparator.COMPARATOR.getClass().getName()).getClass()); assertNull(t.createComparator(Bytes.BYTES_RAWCOMPARATOR.getClass().getName())); assertNull(t.createComparator("org.apache.hadoop.hbase.KeyValue$RawBytesComparator")); } catch (IOException e) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index ef7400d55be..b3010edf3f4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -50,7 +50,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellBuilder; import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; -import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.ExtendedCellBuilderFactory; @@ -277,7 +276,7 @@ public class TestHFile { HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build(); StoreFileWriter sfw = new StoreFileWriter.Builder(conf, fs).withOutputDir(storeFileParentDir) - .withComparator(CellComparatorImpl.COMPARATOR).withFileContext(meta).build(); + .withFileContext(meta).build(); final int rowLen = 32; Random RNG = new Random(); @@ -360,7 +359,7 @@ public class TestHFile { .withCompressTags(false) .build(); HFileWriterImpl writer = new HFileWriterImpl(conf, cacheConf, path, mockedOutputStream, - CellComparator.getInstance(), fileContext); + fileContext); CellBuilder cellBuilder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY); byte[] row = Bytes.toBytes("foo"); byte[] qualifier = Bytes.toBytes("qualifier"); @@ -516,7 +515,6 @@ public class TestHFile { Writer writer = HFile.getWriterFactory(conf, cacheConf) .withOutputStream(fout) .withFileContext(meta) - .withComparator(CellComparatorImpl.COMPARATOR) .create(); LOG.info(Objects.toString(writer)); writeRecords(writer, useTags); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java index 41568b305ed..8c3a6329fad 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderImpl.java @@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; @@ -66,9 +65,8 @@ public class TestHFileReaderImpl { HFileContext context = new HFileContextBuilder().withBlockSize(blocksize).withIncludesTags(true).build(); Configuration conf = TEST_UTIL.getConfiguration(); - HFile.Writer writer = - HFile.getWriterFactoryNoCache(conf).withOutputStream(fout).withFileContext(context) - .withComparator(CellComparatorImpl.COMPARATOR).create(); + HFile.Writer writer = HFile.getWriterFactoryNoCache(conf) + .withOutputStream(fout).withFileContext(context).create(); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) writer.append(toKV("c")); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileScannerImplReferenceCount.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileScannerImplReferenceCount.java index 47cbd852c3e..e326eccd294 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileScannerImplReferenceCount.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileScannerImplReferenceCount.java @@ -37,7 +37,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -165,7 +164,7 @@ public class TestHFileScannerImplReferenceCount { .withCompression(compression).withDataBlockEncoding(encoding).build(); try (HFile.Writer writer = new HFile.WriterFactory(conf, new CacheConfig(conf)).withPath(fs, hfilePath) - .withFileContext(context).withComparator(CellComparatorImpl.COMPARATOR).create()) { + .withFileContext(context).create()) { Random rand = new Random(9713312); // Just a fixed seed. for (int i = 0; i < cellCount; ++i) { byte[] keyBytes = Bytes.add(Bytes.toBytes(i), SUFFIX); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java index 0dc353918ec..d6a97c3a586 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java @@ -27,7 +27,6 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; -import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; @@ -142,7 +141,6 @@ public class TestHFileSeek extends TestCase { Writer writer = HFile.getWriterFactoryNoCache(conf) .withOutputStream(fout) .withFileContext(context) - .withComparator(CellComparatorImpl.COMPARATOR) .create(); try { BytesWritable key = new BytesWritable(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java index 4f4d36b2bc8..3cec8368933 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV3.java @@ -36,7 +36,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -132,7 +131,6 @@ public class TestHFileWriterV3 { HFile.Writer writer = new HFile.WriterFactory(conf, cacheConfig) .withPath(fs, hfilePath) .withFileContext(context) - .withComparator(CellComparatorImpl.COMPARATOR) .create(); Random rand = new Random(9713312); // Just a fixed seed. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java index 13656800681..da62e88dde0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java @@ -26,7 +26,6 @@ import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -145,7 +144,6 @@ public class TestPrefetch { .build(); StoreFileWriter sfw = new StoreFileWriter.Builder(conf, cacheConf, fs) .withOutputDir(storeFileParentDir) - .withComparator(CellComparatorImpl.COMPARATOR) .withFileContext(meta) .build(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java index 1ae861c39f6..de68578cd4f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java @@ -25,7 +25,6 @@ import java.util.List; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ArrayBackedTag; -import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -69,8 +68,6 @@ public class TestReseekTo { TEST_UTIL.getConfiguration(), cacheConf) .withOutputStream(fout) .withFileContext(context) - // NOTE: This test is dependent on this deprecated nonstandard comparator - .withComparator(CellComparatorImpl.COMPARATOR) .create(); int numberOfKeys = 1000; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java index 9ab1f24b006..e45383a7a13 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.ByteBufferKeyValue; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; @@ -123,8 +122,7 @@ public class TestSeekTo { .withIncludesTags(true).build(); Configuration conf = TEST_UTIL.getConfiguration(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout) - .withFileContext(context) - .withComparator(CellComparatorImpl.COMPARATOR).create(); + .withFileContext(context).create(); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) writer.append(toKV("c", tagUsage)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestIgnoreUnknownFamily.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestIgnoreUnknownFamily.java index a99ea7aa64f..89ec68acf4d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestIgnoreUnknownFamily.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestIgnoreUnknownFamily.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; -import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.master.MasterFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -84,7 +84,7 @@ public class TestIgnoreUnknownFamily { Path familyDir = new Path(regionDir, Bytes.toString(UNKNOWN_FAMILY)); StoreFileWriter writer = new StoreFileWriter.Builder(mfs.getConfiguration(), mfs.getFileSystem()) - .withOutputDir(familyDir).withFileContext(new HFileContext()).build(); + .withOutputDir(familyDir).withFileContext(new HFileContextBuilder().build()).build(); writer.close(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java index 52d1ce76c9d..ed9b33e1cf8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoad.java @@ -52,7 +52,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -281,7 +281,7 @@ public class TestBulkLoad { FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation), null); try { hFileFactory.withOutputStream(out); - hFileFactory.withFileContext(new HFileContext()); + hFileFactory.withFileContext(new HFileContextBuilder().build()); HFile.Writer writer = hFileFactory.create(); try { writer.append(new KeyValue(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java index 063c70bc330..33ab4e3bc34 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplication.java @@ -335,7 +335,7 @@ public class TestBulkLoadReplication extends TestReplicationBase { new FSDataOutputStream(new FileOutputStream(hFileLocation), null); try { hFileFactory.withOutputStream(out); - hFileFactory.withFileContext(new HFileContext()); + hFileFactory.withFileContext(new HFileContextBuilder().build()); HFile.Writer writer = hFileFactory.create(); try { writer.append(new KeyValue(cellBuilder.build())); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java index 1d31a4a37dd..31aff48964f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java @@ -24,9 +24,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; -import java.util.Objects; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; @@ -41,6 +39,8 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.TableDescriptor; @@ -61,7 +61,8 @@ import org.slf4j.LoggerFactory; /** * TestGet is a medley of tests of get all done up as a single test. - * This class + * It was originally written to test a method since removed, getClosestAtOrBefore + * but the test is retained because it runs some interesting exercises. */ @Category({RegionServerTests.class, MediumTests.class}) public class TestGetClosestAtOrBefore { @@ -88,7 +89,6 @@ public class TestGetClosestAtOrBefore { @Test public void testUsingMetaAndBinary() throws IOException { - FileSystem filesystem = FileSystem.get(conf); Path rootdir = UTIL.getDataTestDirOnTestFS(); // Up flush size else we bind up when we use default catalog flush of 16k. TableDescriptors tds = new FSTableDescriptors(UTIL.getConfiguration()); @@ -103,13 +103,14 @@ public class TestGetClosestAtOrBefore { final int last = 128; final int interval = 2; for (int i = 0; i <= last; i += interval) { - HRegionInfo hri = new HRegionInfo(htd.getTableName(), - i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i), - i == last ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i + interval)); - + RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()) + .setStartKey(i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte)i)) + .setEndKey(i == last ? HConstants.EMPTY_BYTE_ARRAY : + Bytes.toBytes((byte)i + interval)).build(); Put put = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime()); put.setDurability(Durability.SKIP_WAL); + LOG.info("Put {}", put); mr.put(put); } } @@ -117,7 +118,7 @@ public class TestGetClosestAtOrBefore { try { List keys = new ArrayList<>(); while (s.next(keys)) { - LOG.info(Objects.toString(keys)); + LOG.info("Scan {}", keys); keys.clear(); } } finally { @@ -133,13 +134,14 @@ public class TestGetClosestAtOrBefore { findRow(mr, 'C', 46, 46); findRow(mr, 'C', 43, 42); // Now delete 'C' and make sure I don't get entries from 'B'. - byte[] firstRowInC = HRegionInfo.createRegionName(TableName.valueOf("" + 'C'), + byte[] firstRowInC = RegionInfo.createRegionName(TableName.valueOf("" + 'C'), HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false); - Scan scan = new Scan(firstRowInC); + Scan scan = new Scan().withStartRow(firstRowInC); s = mr.getScanner(scan); try { List keys = new ArrayList<>(); while (s.next(keys)) { + LOG.info("Delete {}", keys); mr.delete(new Delete(CellUtil.cloneRow(keys.get(0)))); keys.clear(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java index 770ee8d1ca3..726d2c49431 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java @@ -63,7 +63,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.regionserver.HRegion.FlushResultImpl; import org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult; import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController; @@ -1657,7 +1657,7 @@ public class TestHRegionReplayEvents { FSDataOutputStream out = TEST_UTIL.getTestFileSystem().create(testFile); try { hFileFactory.withOutputStream(out); - hFileFactory.withFileContext(new HFileContext()); + hFileFactory.withFileContext(new HFileContextBuilder().build()); HFile.Writer writer = hFileFactory.create(); try { writer.append(new KeyValue(ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java index 94c8af555c3..aeeaf3d1d8e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.tool.BulkLoadHFiles; @@ -161,7 +162,7 @@ public class TestScannerWithBulkload { Path path = new Path(pathStr); HFile.WriterFactory wf = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration()); Assert.assertNotNull(wf); - HFileContext context = new HFileContext(); + HFileContext context = new HFileContextBuilder().build(); HFile.Writer writer = wf.withPath(fs, path).withFileContext(context).create(); KeyValue kv = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l, Bytes.toBytes("version2")); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java index 610d0b16d61..63e83608950 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScannerClosure.java @@ -35,7 +35,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.CellComparatorImpl; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -184,7 +183,7 @@ public class TestStoreScannerClosure { Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile"); HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build(); StoreFileWriter sfw = new StoreFileWriter.Builder(CONF, fs).withOutputDir(storeFileParentDir) - .withComparator(CellComparatorImpl.COMPARATOR).withFileContext(meta).build(); + .withFileContext(meta).build(); final int rowLen = 32; Random RNG = new Random();