Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-25287 Forgetting to unbuffer streams results in many CLOSE_WAIT sockets when loading files (#2699) 

Signed-off-by: Andrew Purtell <apurtell@apache.org>
Signed-off-by: Duo Zhang <zhangduo@apache.org>
This commit is contained in:
haxl 2020-12-10 22:15:39 +08:00 committed by GitHub
parent a9b8c10f80
commit 635c911532
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 30 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java
View File

@ -342,8 +342,8 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
Path path = context.getFilePath(); Path path = context.getFilePath();
checkFileVersion(path); checkFileVersion(path);
this.hfileContext = createHFileContext(path, trailer, conf); this.hfileContext = createHFileContext(path, trailer, conf);
} catch (Throwable t) {
context.getInputStreamWrapper().unbuffer(); context.getInputStreamWrapper().unbuffer();
} catch (Throwable t) {
IOUtils.closeQuietly(context.getInputStreamWrapper()); IOUtils.closeQuietly(context.getInputStreamWrapper());
throw new CorruptHFileException("Problem reading HFile Trailer from file " throw new CorruptHFileException("Problem reading HFile Trailer from file "
+ context.getFilePath(), t); + context.getFilePath(), t);
@ -355,16 +355,17 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
*/ */
public void initMetaAndIndex(HFile.Reader reader) throws IOException { public void initMetaAndIndex(HFile.Reader reader) throws IOException {
ReaderContext context = reader.getContext(); ReaderContext context = reader.getContext();
try {
HFileBlock.FSReader blockReader = reader.getUncachedBlockReader(); HFileBlock.FSReader blockReader = reader.getUncachedBlockReader();
// Initialize an block iterator, and parse load-on-open blocks in the following. // Initialize an block iterator, and parse load-on-open blocks in the following.
blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(), blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(),
context.getFileSize() - trailer.getTrailerSize()); context.getFileSize() - trailer.getTrailerSize());
// Data index. We also read statistics about the block index written after // Data index. We also read statistics about the block index written after
// the root level. // the root level.
this.dataIndexReader = new HFileBlockIndex this.dataIndexReader =
.CellBasedKeyBlockIndexReader(trailer.createComparator(), trailer.getNumDataIndexLevels()); new HFileBlockIndex.CellBasedKeyBlockIndexReader(trailer.createComparator(), trailer.getNumDataIndexLevels());
dataIndexReader.readMultiLevelIndexRoot(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), dataIndexReader
trailer.getDataIndexCount()); .readMultiLevelIndexRoot(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), trailer.getDataIndexCount());
reader.setDataBlockIndexReader(dataIndexReader); reader.setDataBlockIndexReader(dataIndexReader);
// Meta index. // Meta index.
this.metaIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1); this.metaIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1);
@ -378,6 +379,13 @@ public class HFileInfo implements SortedMap<byte[], byte[]> {
while ((b = blockIter.nextBlock()) != null) { while ((b = blockIter.nextBlock()) != null) {
loadOnOpenBlocks.add(b); loadOnOpenBlocks.add(b);
} }
// close the block reader
context.getInputStreamWrapper().unbuffer();
} catch (Throwable t) {
IOUtils.closeQuietly(context.getInputStreamWrapper());
throw new CorruptHFileException("Problem reading data index and meta index from file "
+ context.getFilePath(), t);
}
} }
private HFileContext createHFileContext(Path path, private HFileContext createHFileContext(Path path,