diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 559be7ca050..0ad5065d4f4 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -174,6 +174,9 @@ Optimizations * LUCENE-4941: Sort "from" terms only once when using JoinUtil. (Martijn van Groningen) +* LUCENE-5050: Close the stored fields and term vectors index files as soon as + the index has been loaded into memory to save file descriptors. (Adrien Grand) + New Features * LUCENE-4766: Added a PatternCaptureGroupTokenFilter that uses Java regexes to diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java index 00cc4f43299..c73942f0cdd 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java @@ -17,7 +17,6 @@ package org.apache.lucene.codecs.compressing; * limitations under the License. */ -import java.io.Closeable; import java.io.IOException; import java.util.Arrays; @@ -25,16 +24,13 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.packed.PackedInts; /** * Random-access reader for {@link CompressingStoredFieldsIndexWriter}. * @lucene.internal */ -public final class CompressingStoredFieldsIndexReader implements Closeable, Cloneable { - - final IndexInput fieldsIndexIn; +public final class CompressingStoredFieldsIndexReader implements Cloneable { static long moveLowOrderBitToSign(long n) { return ((n >>> 1) ^ -(n & 1)); @@ -48,8 +44,9 @@ public final class CompressingStoredFieldsIndexReader implements Closeable, Clon final PackedInts.Reader[] docBasesDeltas; // delta from the avg final PackedInts.Reader[] startPointersDeltas; // delta from the avg + // It is the responsibility of the caller to close fieldsIndexIn after this constructor + // has been called CompressingStoredFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws IOException { - this.fieldsIndexIn = fieldsIndexIn; maxDoc = si.getDocCount(); int[] docBases = new int[16]; long[] startPointers = new long[16]; @@ -106,17 +103,6 @@ public final class CompressingStoredFieldsIndexReader implements Closeable, Clon this.startPointersDeltas = Arrays.copyOf(startPointersDeltas, blockCount); } - private CompressingStoredFieldsIndexReader(CompressingStoredFieldsIndexReader other) { - this.fieldsIndexIn = null; - this.maxDoc = other.maxDoc; - this.docBases = other.docBases; - this.startPointers = other.startPointers; - this.avgChunkDocs = other.avgChunkDocs; - this.avgChunkSizes = other.avgChunkSizes; - this.docBasesDeltas = other.docBasesDeltas; - this.startPointersDeltas = other.startPointersDeltas; - } - private int block(int docID) { int lo = 0, hi = docBases.length - 1; while (lo <= hi) { @@ -172,16 +158,7 @@ public final class CompressingStoredFieldsIndexReader implements Closeable, Clon @Override public CompressingStoredFieldsIndexReader clone() { - if (fieldsIndexIn == null) { - return this; - } else { - return new CompressingStoredFieldsIndexReader(this); - } - } - - @Override - public void close() throws IOException { - IOUtils.close(fieldsIndexIn); + return this; } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java index db01bb3bf06..29fed881e12 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java @@ -96,19 +96,22 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader { numDocs = si.getDocCount(); IndexInput indexStream = null; try { - fieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION), context); + // Load the index into memory final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION); indexStream = d.openInput(indexStreamFN, context); - final String codecNameIdx = formatName + CODEC_SFX_IDX; - final String codecNameDat = formatName + CODEC_SFX_DAT; CodecUtil.checkHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT); + assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer(); + indexReader = new CompressingStoredFieldsIndexReader(indexStream, si); + indexStream.close(); + indexStream = null; + + // Open the data file and read metadata + final String fieldsStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION); + fieldsStream = d.openInput(fieldsStreamFN, context); + final String codecNameDat = formatName + CODEC_SFX_DAT; CodecUtil.checkHeader(fieldsStream, codecNameDat, VERSION_START, VERSION_CURRENT); assert CodecUtil.headerLength(codecNameDat) == fieldsStream.getFilePointer(); - assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer(); - - indexReader = new CompressingStoredFieldsIndexReader(indexStream, si); - indexStream = null; packedIntsVersion = fieldsStream.readVInt(); decompressor = compressionMode.newDecompressor(); @@ -137,7 +140,7 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader { @Override public void close() throws IOException { if (!closed) { - IOUtils.close(fieldsStream, indexReader); + IOUtils.close(fieldsStream); closed = true; } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java index 545c86ca684..48c8d5c8435 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java @@ -102,19 +102,22 @@ public final class CompressingTermVectorsReader extends TermVectorsReader implem numDocs = si.getDocCount(); IndexInput indexStream = null; try { - vectorsStream = d.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION), context); + // Load the index into memory final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_INDEX_EXTENSION); indexStream = d.openInput(indexStreamFN, context); - final String codecNameIdx = formatName + CODEC_SFX_IDX; - final String codecNameDat = formatName + CODEC_SFX_DAT; CodecUtil.checkHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT); + assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer(); + indexReader = new CompressingStoredFieldsIndexReader(indexStream, si); + indexStream.close(); + indexStream = null; + + // Open the data file and read metadata + final String vectorsStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION); + vectorsStream = d.openInput(vectorsStreamFN, context); + final String codecNameDat = formatName + CODEC_SFX_DAT; CodecUtil.checkHeader(vectorsStream, codecNameDat, VERSION_START, VERSION_CURRENT); assert CodecUtil.headerLength(codecNameDat) == vectorsStream.getFilePointer(); - assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer(); - - indexReader = new CompressingStoredFieldsIndexReader(indexStream, si); - indexStream = null; packedIntsVersion = vectorsStream.readVInt(); chunkSize = vectorsStream.readVInt(); @@ -161,7 +164,7 @@ public final class CompressingTermVectorsReader extends TermVectorsReader implem @Override public void close() throws IOException { if (!closed) { - IOUtils.close(vectorsStream, indexReader); + IOUtils.close(vectorsStream); closed = true; } }