From 846338c0dc50952b554ea9f3abfbad886e3849e0 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 30 Jan 2012 16:02:37 +0000 Subject: [PATCH] LUCENE-3728: push compound shared doc stores reading into 3.x codec (only 3.0 indexes, only a hit if you also have vectors) git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene3661@1237746 13f79535-47bb-0310-9956-ffa450edef68 --- .../lucene3x/Lucene3xStoredFieldsReader.java | 15 +++++++-- .../lucene3x/Lucene3xTermVectorsReader.java | 16 +++++++++- .../lucene40/Lucene40SegmentInfosReader.java | 1 - .../lucene/index/SegmentCoreReaders.java | 32 +++---------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java b/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java index 810a6269c15..9b96f1a457a 100644 --- a/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java +++ b/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java @@ -29,13 +29,13 @@ import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.IOUtils; import java.io.Closeable; -import java.nio.charset.Charset; import java.util.Set; /** @@ -93,6 +93,10 @@ public final class Lucene3xStoredFieldsReader extends StoredFieldsReader impleme // The docID offset where our docs begin in the index // file. This will be 0 if we have our own private file. private int docStoreOffset; + + // when we are inside a compound share doc store (CFX), + // (lucene 3.0 indexes only), we privately open our own fd. + private final CompoundFileDirectory storeCFSReader; /** Returns a cloned FieldsReader that shares open * IndexInputs with the original one. It is the caller's @@ -131,6 +135,7 @@ public final class Lucene3xStoredFieldsReader extends StoredFieldsReader impleme this.docStoreOffset = docStoreOffset; this.fieldsStream = fieldsStream; this.indexStream = indexStream; + this.storeCFSReader = null; } public Lucene3xStoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { @@ -140,6 +145,12 @@ public final class Lucene3xStoredFieldsReader extends StoredFieldsReader impleme boolean success = false; fieldInfos = fn; try { + if (docStoreOffset != -1 && si.getDocStoreIsCompoundFile()) { + d = storeCFSReader = new CompoundFileDirectory(si.dir, + IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context, false); + } else { + storeCFSReader = null; + } fieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context); final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION); indexStream = d.openInput(indexStreamFN, context); @@ -200,7 +211,7 @@ public final class Lucene3xStoredFieldsReader extends StoredFieldsReader impleme */ public final void close() throws IOException { if (!closed) { - IOUtils.close(fieldsStream, indexStream); + IOUtils.close(fieldsStream, indexStream, storeCFSReader); closed = true; } } diff --git a/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java b/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java index bd5695b81fd..0df4bb94b30 100644 --- a/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java +++ b/lucene/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java @@ -38,6 +38,7 @@ import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -90,6 +91,12 @@ public class Lucene3xTermVectorsReader extends TermVectorsReader { // file. This will be 0 if we have our own private file. private int docStoreOffset; + // when we are inside a compound share doc store (CFX), + // (lucene 3.0 indexes only), we privately open our own fd. + // TODO: if we are worried, maybe we could eliminate the + // extra fd somehow when you also have vectors... + private final CompoundFileDirectory storeCFSReader; + private final int format; // used by clone @@ -102,6 +109,7 @@ public class Lucene3xTermVectorsReader extends TermVectorsReader { this.numTotalDocs = numTotalDocs; this.docStoreOffset = docStoreOffset; this.format = format; + this.storeCFSReader = null; } public Lucene3xTermVectorsReader(Directory d, SegmentInfo si, FieldInfos fieldInfos, IOContext context) @@ -113,6 +121,12 @@ public class Lucene3xTermVectorsReader extends TermVectorsReader { boolean success = false; try { + if (docStoreOffset != -1 && si.getDocStoreIsCompoundFile()) { + d = storeCFSReader = new CompoundFileDirectory(si.dir, + IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context, false); + } else { + storeCFSReader = null; + } String idxName = IndexFileNames.segmentFileName(segment, "", VECTORS_INDEX_EXTENSION); tvx = d.openInput(idxName, context); format = checkValidFormat(tvx); @@ -170,7 +184,7 @@ public class Lucene3xTermVectorsReader extends TermVectorsReader { } public void close() throws IOException { - IOUtils.close(tvx, tvd, tvf); + IOUtils.close(tvx, tvd, tvf, storeCFSReader); } /** diff --git a/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java b/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java index 0d335e9bf84..ce581ace00e 100644 --- a/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java +++ b/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfosReader.java @@ -23,7 +23,6 @@ import java.util.Map; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.SegmentInfosReader; -import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.ChecksumIndexInput; diff --git a/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java b/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java index 754acaa6ac8..657981e2b42 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java @@ -65,7 +65,6 @@ final class SegmentCoreReaders { final StoredFieldsReader fieldsReaderOrig; final TermVectorsReader termVectorsReaderOrig; final CompoundFileDirectory cfsReader; - final CompoundFileDirectory storeCFSReader; final CloseableThreadLocal fieldsReaderLocal = new CloseableThreadLocal() { @Override @@ -121,34 +120,11 @@ final class SegmentCoreReaders { // kinda jaky to assume the codec handles the case of no norms file at all gracefully?! norms = codec.normsFormat().docsProducer(segmentReadState); perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState); - - final Directory storeDir; - if (si.getDocStoreOffset() != -1) { - if (si.getDocStoreIsCompoundFile()) { - storeCFSReader = new CompoundFileDirectory(dir, - IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), - context, false); - storeDir = storeCFSReader; - assert storeDir != null; - } else { - storeCFSReader = null; - storeDir = dir; - assert storeDir != null; - } - } else if (si.getUseCompoundFile()) { - storeDir = cfsReader; - storeCFSReader = null; - assert storeDir != null; - } else { - storeDir = dir; - storeCFSReader = null; - assert storeDir != null; - } - - fieldsReaderOrig = si.getCodec().storedFieldsFormat().fieldsReader(storeDir, si, fieldInfos, context); + + fieldsReaderOrig = si.getCodec().storedFieldsFormat().fieldsReader(cfsDir, si, fieldInfos, context); if (si.getHasVectors()) { // open term vector files only as needed - termVectorsReaderOrig = si.getCodec().termVectorsFormat().vectorsReader(storeDir, si, fieldInfos, context); + termVectorsReaderOrig = si.getCodec().termVectorsFormat().vectorsReader(cfsDir, si, fieldInfos, context); } else { termVectorsReaderOrig = null; } @@ -175,7 +151,7 @@ final class SegmentCoreReaders { //System.out.println("core.decRef seg=" + owner.getSegmentInfo() + " rc=" + ref); if (ref.decrementAndGet() == 0) { IOUtils.close(termVectorsLocal, fieldsReaderLocal, fields, perDocProducer, - termVectorsReaderOrig, fieldsReaderOrig, cfsReader, storeCFSReader, norms); + termVectorsReaderOrig, fieldsReaderOrig, cfsReader, norms); notifyCoreClosedListeners(); } }