Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-25287 Forgetting to unbuffer streams results in many CLOSE_WAIT sockets when loading files (#2699) 

Signed-off-by: Andrew Purtell <apurtell@apache.org>
Signed-off-by: Duo Zhang <zhangduo@apache.org>
This commit is contained in:
haxl 2020-12-10 22:15:39 +08:00 committed by GitHub
parent a9b8c10f80
commit 635c911532
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 30 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
View File

@ -342,8 +342,8 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
Path path = context.getFilePath();
checkFileVersion(path);
this.hfileContext = createHFileContext(path, trailer, conf);
} catch (Throwable t) {
context.getInputStreamWrapper().unbuffer();
} catch (Throwable t) {
IOUtils.closeQuietly(context.getInputStreamWrapper());
throw new CorruptHFileException("Problem reading HFile Trailer from file "
+ context.getFilePath(), t);
@ -355,28 +355,36 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
*/
public void initMetaAndIndex(HFile.Reader reader) throws IOException {
ReaderContext context = reader.getContext();
HFileBlock.FSReader blockReader = reader.getUncachedBlockReader();
// Initialize an block iterator, and parse load-on-open blocks in the following.
blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(),
context.getFileSize() - trailer.getTrailerSize());
// Data index. We also read statistics about the block index written after
// the root level.
this.dataIndexReader = new HFileBlockIndex
.CellBasedKeyBlockIndexReader(trailer.createComparator(), trailer.getNumDataIndexLevels());
dataIndexReader.readMultiLevelIndexRoot(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
trailer.getDataIndexCount());
reader.setDataBlockIndexReader(dataIndexReader);
// Meta index.
this.metaIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1);
metaIndexReader.readRootIndex(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
try {
HFileBlock.FSReader blockReader = reader.getUncachedBlockReader();
// Initialize an block iterator, and parse load-on-open blocks in the following.
blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(),
context.getFileSize() - trailer.getTrailerSize());
// Data index. We also read statistics about the block index written after
// the root level.
this.dataIndexReader =
new HFileBlockIndex.CellBasedKeyBlockIndexReader(trailer.createComparator(), trailer.getNumDataIndexLevels());
dataIndexReader
.readMultiLevelIndexRoot(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), trailer.getDataIndexCount());
reader.setDataBlockIndexReader(dataIndexReader);
// Meta index.
this.metaIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1);
metaIndexReader.readRootIndex(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
trailer.getMetaIndexCount());
reader.setMetaBlockIndexReader(metaIndexReader);
loadMetaInfo(blockIter, hfileContext);
reader.setDataBlockEncoder(HFileDataBlockEncoderImpl.createFromFileInfo(this));
// Load-On-Open info
HFileBlock b;
while ((b = blockIter.nextBlock()) != null) {
loadOnOpenBlocks.add(b);
reader.setMetaBlockIndexReader(metaIndexReader);
loadMetaInfo(blockIter, hfileContext);
reader.setDataBlockEncoder(HFileDataBlockEncoderImpl.createFromFileInfo(this));
// Load-On-Open info
HFileBlock b;
while ((b = blockIter.nextBlock()) != null) {
loadOnOpenBlocks.add(b);
}
// close the block reader
context.getInputStreamWrapper().unbuffer();
} catch (Throwable t) {
IOUtils.closeQuietly(context.getInputStreamWrapper());
throw new CorruptHFileException("Problem reading data index and meta index from file "
+ context.getFilePath(), t);
}
}