Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-25287 Forgetting to unbuffer streams results in many CLOSE_WAIT sockets when loading files (#2699) 

Signed-off-by: Andrew Purtell <apurtell@apache.org>
Signed-off-by: Duo Zhang <zhangduo@apache.org>
This commit is contained in:
haxl 2020-12-10 22:15:39 +08:00 committed by GitHub
parent a9b8c10f80
commit 635c911532
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 30 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
View File

@ -342,8 +342,8 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
Path path = context.getFilePath(); Path path = context.getFilePath();
checkFileVersion(path); checkFileVersion(path);
this.hfileContext = createHFileContext(path, trailer, conf); this.hfileContext = createHFileContext(path, trailer, conf);
} catch (Throwable t) {
context.getInputStreamWrapper().unbuffer(); context.getInputStreamWrapper().unbuffer();
} catch (Throwable t) {
IOUtils.closeQuietly(context.getInputStreamWrapper()); IOUtils.closeQuietly(context.getInputStreamWrapper());
throw new CorruptHFileException("Problem reading HFile Trailer from file " throw new CorruptHFileException("Problem reading HFile Trailer from file "
+ context.getFilePath(), t); + context.getFilePath(), t);
@ -355,28 +355,36 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
*/ */
public void initMetaAndIndex(HFile.Reader reader) throws IOException { public void initMetaAndIndex(HFile.Reader reader) throws IOException {
ReaderContext context = reader.getContext(); ReaderContext context = reader.getContext();
HFileBlock.FSReader blockReader = reader.getUncachedBlockReader(); try {
// Initialize an block iterator, and parse load-on-open blocks in the following. HFileBlock.FSReader blockReader = reader.getUncachedBlockReader();
blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(), // Initialize an block iterator, and parse load-on-open blocks in the following.
context.getFileSize() - trailer.getTrailerSize()); blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(),
// Data index. We also read statistics about the block index written after context.getFileSize() - trailer.getTrailerSize());
// the root level. // Data index. We also read statistics about the block index written after
this.dataIndexReader = new HFileBlockIndex // the root level.
.CellBasedKeyBlockIndexReader(trailer.createComparator(), trailer.getNumDataIndexLevels()); this.dataIndexReader =
dataIndexReader.readMultiLevelIndexRoot(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), new HFileBlockIndex.CellBasedKeyBlockIndexReader(trailer.createComparator(), trailer.getNumDataIndexLevels());
trailer.getDataIndexCount()); dataIndexReader
reader.setDataBlockIndexReader(dataIndexReader); .readMultiLevelIndexRoot(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), trailer.getDataIndexCount());
// Meta index. reader.setDataBlockIndexReader(dataIndexReader);
this.metaIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1); // Meta index.
metaIndexReader.readRootIndex(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), this.metaIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1);
metaIndexReader.readRootIndex(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
trailer.getMetaIndexCount()); trailer.getMetaIndexCount());
reader.setMetaBlockIndexReader(metaIndexReader); reader.setMetaBlockIndexReader(metaIndexReader);
loadMetaInfo(blockIter, hfileContext); loadMetaInfo(blockIter, hfileContext);
reader.setDataBlockEncoder(HFileDataBlockEncoderImpl.createFromFileInfo(this)); reader.setDataBlockEncoder(HFileDataBlockEncoderImpl.createFromFileInfo(this));
// Load-On-Open info // Load-On-Open info
HFileBlock b; HFileBlock b;
while ((b = blockIter.nextBlock()) != null) { while ((b = blockIter.nextBlock()) != null) {
loadOnOpenBlocks.add(b); loadOnOpenBlocks.add(b);
}
// close the block reader
context.getInputStreamWrapper().unbuffer();
} catch (Throwable t) {
IOUtils.closeQuietly(context.getInputStreamWrapper());
throw new CorruptHFileException("Problem reading data index and meta index from file "
+ context.getFilePath(), t);
} }
} }