From f526ce6814b2cac1bb04fda04b7f875317a16264 Mon Sep 17 00:00:00 2001 From: haxl Date: Thu, 10 Dec 2020 22:15:39 +0800 Subject: [PATCH] HBASE-25287 Forgetting to unbuffer streams results in many CLOSE_WAIT sockets when loading files (#2699) Signed-off-by: Andrew Purtell Signed-off-by: Duo Zhang --- .../hadoop/hbase/io/hfile/HFileInfo.java | 52 +++++++++++-------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java index ad469c092b6..0751a240e3c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java @@ -340,8 +340,8 @@ public class HFileInfo implements SortedMap { Path path = context.getFilePath(); checkFileVersion(path); this.hfileContext = createHFileContext(path, trailer, conf); - } catch (Throwable t) { context.getInputStreamWrapper().unbuffer(); + } catch (Throwable t) { IOUtils.closeQuietly(context.getInputStreamWrapper()); throw new CorruptHFileException("Problem reading HFile Trailer from file " + context.getFilePath(), t); @@ -353,28 +353,36 @@ public class HFileInfo implements SortedMap { */ public void initMetaAndIndex(HFile.Reader reader) throws IOException { ReaderContext context = reader.getContext(); - HFileBlock.FSReader blockReader = reader.getUncachedBlockReader(); - // Initialize an block iterator, and parse load-on-open blocks in the following. - blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(), - context.getFileSize() - trailer.getTrailerSize()); - // Data index. We also read statistics about the block index written after - // the root level. - this.dataIndexReader = new HFileBlockIndex - .CellBasedKeyBlockIndexReader(trailer.createComparator(), trailer.getNumDataIndexLevels()); - dataIndexReader.readMultiLevelIndexRoot(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), - trailer.getDataIndexCount()); - reader.setDataBlockIndexReader(dataIndexReader); - // Meta index. - this.metaIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1); - metaIndexReader.readRootIndex(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), + try { + HFileBlock.FSReader blockReader = reader.getUncachedBlockReader(); + // Initialize an block iterator, and parse load-on-open blocks in the following. + blockIter = blockReader.blockRange(trailer.getLoadOnOpenDataOffset(), + context.getFileSize() - trailer.getTrailerSize()); + // Data index. We also read statistics about the block index written after + // the root level. + this.dataIndexReader = + new HFileBlockIndex.CellBasedKeyBlockIndexReader(trailer.createComparator(), trailer.getNumDataIndexLevels()); + dataIndexReader + .readMultiLevelIndexRoot(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), trailer.getDataIndexCount()); + reader.setDataBlockIndexReader(dataIndexReader); + // Meta index. + this.metaIndexReader = new HFileBlockIndex.ByteArrayKeyBlockIndexReader(1); + metaIndexReader.readRootIndex(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), trailer.getMetaIndexCount()); - reader.setMetaBlockIndexReader(metaIndexReader); - loadMetaInfo(blockIter, hfileContext); - reader.setDataBlockEncoder(HFileDataBlockEncoderImpl.createFromFileInfo(this)); - // Load-On-Open info - HFileBlock b; - while ((b = blockIter.nextBlock()) != null) { - loadOnOpenBlocks.add(b); + reader.setMetaBlockIndexReader(metaIndexReader); + loadMetaInfo(blockIter, hfileContext); + reader.setDataBlockEncoder(HFileDataBlockEncoderImpl.createFromFileInfo(this)); + // Load-On-Open info + HFileBlock b; + while ((b = blockIter.nextBlock()) != null) { + loadOnOpenBlocks.add(b); + } + // close the block reader + context.getInputStreamWrapper().unbuffer(); + } catch (Throwable t) { + IOUtils.closeQuietly(context.getInputStreamWrapper()); + throw new CorruptHFileException("Problem reading data index and meta index from file " + + context.getFilePath(), t); } }