Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-16644 Errors when reading legit HFile Trailer of old (v2.0) format file

This commit is contained in:
Mikhail Antonov 2016-10-04 21:10:42 -07:00
parent 0daeb635d0
commit 5ae516bd63
1 changed files with 14 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
View File

@ -371,15 +371,16 @@ public class HFileBlock implements Cacheable {
final int uncompressedSizeWithoutHeader = final int uncompressedSizeWithoutHeader =
buf.getInt(Header.UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX); buf.getInt(Header.UNCOMPRESSED_SIZE_WITHOUT_HEADER_INDEX);
final long prevBlockOffset = buf.getLong(Header.PREV_BLOCK_OFFSET_INDEX); final long prevBlockOffset = buf.getLong(Header.PREV_BLOCK_OFFSET_INDEX);
byte checksumType = buf.get(Header.CHECKSUM_TYPE_INDEX);
int bytesPerChecksum = buf.getInt(Header.BYTES_PER_CHECKSUM_INDEX);
int onDiskDataSizeWithHeader = buf.getInt(Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX);
// This constructor is called when we deserialize a block from cache and when we read a block in // This constructor is called when we deserialize a block from cache and when we read a block in
// from the fs. fileCache is null when deserialized from cache so need to make up one. // from the fs. fileCache is null when deserialized from cache so need to make up one.
HFileContextBuilder fileContextBuilder = fileContext != null? HFileContextBuilder fileContextBuilder = fileContext != null?
new HFileContextBuilder(fileContext): new HFileContextBuilder(); new HFileContextBuilder(fileContext): new HFileContextBuilder();
fileContextBuilder.withHBaseCheckSum(usesHBaseChecksum); fileContextBuilder.withHBaseCheckSum(usesHBaseChecksum);
int onDiskDataSizeWithHeader;
if (usesHBaseChecksum) { if (usesHBaseChecksum) {
byte checksumType = buf.get(Header.CHECKSUM_TYPE_INDEX);
int bytesPerChecksum = buf.getInt(Header.BYTES_PER_CHECKSUM_INDEX);
onDiskDataSizeWithHeader = buf.getInt(Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX);
// Use the checksum type and bytes per checksum from header, not from filecontext. // Use the checksum type and bytes per checksum from header, not from filecontext.
fileContextBuilder.withChecksumType(ChecksumType.codeToType(checksumType)); fileContextBuilder.withChecksumType(ChecksumType.codeToType(checksumType));
fileContextBuilder.withBytesPerCheckSum(bytesPerChecksum); fileContextBuilder.withBytesPerCheckSum(bytesPerChecksum);
@ -419,11 +420,12 @@ public class HFileBlock implements Cacheable {
/** /**
* Parse total ondisk size including header and checksum. * Parse total ondisk size including header and checksum.
* @param headerBuf Header ByteBuffer. Presumed exact size of header. * @param headerBuf Header ByteBuffer. Presumed exact size of header.
* @param verifyChecksum true if checksum verification is in use.
* @return Size of the block with header included. * @return Size of the block with header included.
*/ */
private static int getOnDiskSizeWithHeader(final ByteBuffer headerBuf) { private static int getOnDiskSizeWithHeader(final ByteBuffer headerBuf, boolean verifyChecksum) {
// Set hbase checksum to true always calling headerSize. return headerBuf.getInt(Header.ON_DISK_SIZE_WITHOUT_HEADER_INDEX) +
return headerBuf.getInt(Header.ON_DISK_SIZE_WITHOUT_HEADER_INDEX) + headerSize(true); headerSize(verifyChecksum);
} }
/** /**
@ -1659,10 +1661,10 @@ public class HFileBlock implements Cacheable {
* @throws IOException * @throws IOException
*/ */
private void verifyOnDiskSizeMatchesHeader(final int passedIn, final ByteBuffer headerBuf, private void verifyOnDiskSizeMatchesHeader(final int passedIn, final ByteBuffer headerBuf,
final long offset) final long offset, boolean verifyChecksum)
throws IOException { throws IOException {
// Assert size provided aligns with what is in the header // Assert size provided aligns with what is in the header
int fromHeader = getOnDiskSizeWithHeader(headerBuf); int fromHeader = getOnDiskSizeWithHeader(headerBuf, verifyChecksum);
if (passedIn != fromHeader) { if (passedIn != fromHeader) {
throw new IOException("Passed in onDiskSizeWithHeader=" + passedIn + " != " + fromHeader + throw new IOException("Passed in onDiskSizeWithHeader=" + passedIn + " != " + fromHeader +
", offset=" + offset + ", fileContext=" + this.fileContext); ", offset=" + offset + ", fileContext=" + this.fileContext);
@ -1703,7 +1705,8 @@ public class HFileBlock implements Cacheable {
readAtOffset(is, headerBuf.array(), headerBuf.arrayOffset(), hdrSize, false, readAtOffset(is, headerBuf.array(), headerBuf.arrayOffset(), hdrSize, false,
offset, pread); offset, pread);
} }
onDiskSizeWithHeader = getOnDiskSizeWithHeader(headerBuf); onDiskSizeWithHeader = getOnDiskSizeWithHeader(headerBuf,
this.fileContext.isUseHBaseChecksum());
} }
int preReadHeaderSize = headerBuf == null? 0 : hdrSize; int preReadHeaderSize = headerBuf == null? 0 : hdrSize;
// Allocate enough space to fit the next block's header too; saves a seek next time through. // Allocate enough space to fit the next block's header too; saves a seek next time through.
@ -1722,7 +1725,8 @@ public class HFileBlock implements Cacheable {
} }
// Do a few checks before we go instantiate HFileBlock. // Do a few checks before we go instantiate HFileBlock.
assert onDiskSizeWithHeader > this.hdrSize; assert onDiskSizeWithHeader > this.hdrSize;
verifyOnDiskSizeMatchesHeader(onDiskSizeWithHeader, headerBuf, offset); verifyOnDiskSizeMatchesHeader(onDiskSizeWithHeader, headerBuf, offset,
this.fileContext.isUseHBaseChecksum());
ByteBuffer onDiskBlockByteBuffer = ByteBuffer.wrap(onDiskBlock, 0, onDiskSizeWithHeader); ByteBuffer onDiskBlockByteBuffer = ByteBuffer.wrap(onDiskBlock, 0, onDiskSizeWithHeader);
// Verify checksum of the data before using it for building HFileBlock. // Verify checksum of the data before using it for building HFileBlock.
if (verifyChecksum && if (verifyChecksum &&