Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5050: Close the stored fields and term vectors index files eagerly. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1491889 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2013-06-11 17:56:47 +00:00
parent 5a14c04bc1
commit 3e097c87bb
4 changed files with 29 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -174,6 +174,9 @@ Optimizations
* LUCENE-4941: Sort "from" terms only once when using JoinUtil.
(Martijn van Groningen)
* LUCENE-5050: Close the stored fields and term vectors index files as soon as
the index has been loaded into memory to save file descriptors. (Adrien Grand)
New Features
* LUCENE-4766: Added a PatternCaptureGroupTokenFilter that uses Java regexes to

31
lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexReader.java
View File

@ -17,7 +17,6 @@ package org.apache.lucene.codecs.compressing;
* limitations under the License.
*/
import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
@ -25,16 +24,13 @@ import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.PackedInts;
/**
* Random-access reader for {@link CompressingStoredFieldsIndexWriter}.
* @lucene.internal
*/
public final class CompressingStoredFieldsIndexReader implements Closeable, Cloneable {
final IndexInput fieldsIndexIn;
public final class CompressingStoredFieldsIndexReader implements Cloneable {
static long moveLowOrderBitToSign(long n) {
return ((n >>> 1) ^ -(n & 1));
@ -48,8 +44,9 @@ public final class CompressingStoredFieldsIndexReader implements Closeable, Clon
final PackedInts.Reader[] docBasesDeltas; // delta from the avg
final PackedInts.Reader[] startPointersDeltas; // delta from the avg
// It is the responsibility of the caller to close fieldsIndexIn after this constructor
// has been called
CompressingStoredFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws IOException {
this.fieldsIndexIn = fieldsIndexIn;
maxDoc = si.getDocCount();
int[] docBases = new int[16];
long[] startPointers = new long[16];
@ -106,17 +103,6 @@ public final class CompressingStoredFieldsIndexReader implements Closeable, Clon
this.startPointersDeltas = Arrays.copyOf(startPointersDeltas, blockCount);
}
private CompressingStoredFieldsIndexReader(CompressingStoredFieldsIndexReader other) {
this.fieldsIndexIn = null;
this.maxDoc = other.maxDoc;
this.docBases = other.docBases;
this.startPointers = other.startPointers;
this.avgChunkDocs = other.avgChunkDocs;
this.avgChunkSizes = other.avgChunkSizes;
this.docBasesDeltas = other.docBasesDeltas;
this.startPointersDeltas = other.startPointersDeltas;
}
private int block(int docID) {
int lo = 0, hi = docBases.length - 1;
while (lo <= hi) {
@ -172,16 +158,7 @@ public final class CompressingStoredFieldsIndexReader implements Closeable, Clon
@Override
public CompressingStoredFieldsIndexReader clone() {
if (fieldsIndexIn == null) {
return this;
} else {
return new CompressingStoredFieldsIndexReader(this);
}
}
@Override
public void close() throws IOException {
IOUtils.close(fieldsIndexIn);
return this;
}
}

19
lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
View File

@ -96,19 +96,22 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {
numDocs = si.getDocCount();
IndexInput indexStream = null;
try {
fieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION), context);
// Load the index into memory
final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION);
indexStream = d.openInput(indexStreamFN, context);
final String codecNameIdx = formatName + CODEC_SFX_IDX;
final String codecNameDat = formatName + CODEC_SFX_DAT;
CodecUtil.checkHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT);
assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer();
indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
indexStream.close();
indexStream = null;
// Open the data file and read metadata
final String fieldsStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION);
fieldsStream = d.openInput(fieldsStreamFN, context);
final String codecNameDat = formatName + CODEC_SFX_DAT;
CodecUtil.checkHeader(fieldsStream, codecNameDat, VERSION_START, VERSION_CURRENT);
assert CodecUtil.headerLength(codecNameDat) == fieldsStream.getFilePointer();
assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer();
indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
indexStream = null;
packedIntsVersion = fieldsStream.readVInt();
decompressor = compressionMode.newDecompressor();
@ -137,7 +140,7 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {
@Override
public void close() throws IOException {
if (!closed) {
IOUtils.close(fieldsStream, indexReader);
IOUtils.close(fieldsStream);
closed = true;
}
}

19
lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
View File

@ -102,19 +102,22 @@ public final class CompressingTermVectorsReader extends TermVectorsReader implem
numDocs = si.getDocCount();
IndexInput indexStream = null;
try {
vectorsStream = d.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION), context);
// Load the index into memory
final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_INDEX_EXTENSION);
indexStream = d.openInput(indexStreamFN, context);
final String codecNameIdx = formatName + CODEC_SFX_IDX;
final String codecNameDat = formatName + CODEC_SFX_DAT;
CodecUtil.checkHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT);
assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer();
indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
indexStream.close();
indexStream = null;
// Open the data file and read metadata
final String vectorsStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION);
vectorsStream = d.openInput(vectorsStreamFN, context);
final String codecNameDat = formatName + CODEC_SFX_DAT;
CodecUtil.checkHeader(vectorsStream, codecNameDat, VERSION_START, VERSION_CURRENT);
assert CodecUtil.headerLength(codecNameDat) == vectorsStream.getFilePointer();
assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer();
indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
indexStream = null;
packedIntsVersion = vectorsStream.readVInt();
chunkSize = vectorsStream.readVInt();
@ -161,7 +164,7 @@ public final class CompressingTermVectorsReader extends TermVectorsReader implem
@Override
public void close() throws IOException {
if (!closed) {
IOUtils.close(vectorsStream, indexReader);
IOUtils.close(vectorsStream);
closed = true;
}
}