Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3734: Allow customizing/subclassing of DirectoryReader 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1239043 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Uwe Schindler 2012-02-01 10:41:54 +00:00
parent 9f3bffcb1f
commit 5f38919e81
9 changed files with 541 additions and 477 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java
View File

@ -100,7 +100,8 @@ public class MultiPassIndexSplitter {
null)
.setOpenMode(OpenMode.CREATE));
System.err.println("Writing part " + (i + 1) + " ...");
w.addIndexes(input);
// pass the subreaders directly, as our wrapper's numDocs/hasDeletetions are not up-to-date
w.addIndexes(input.getSequentialSubReaders());
w.close();
}
System.err.println("Done.");
@ -204,23 +205,8 @@ public class MultiPassIndexSplitter {
}
}
// override this as MultiReader precalculates the number of deletions
// (this method is never used by MultiPassIndexSplitter)
@Override
public int numDocs() {
int n = 0;
for (int i = 0; i < subReaders.length; i++)
n += subReaders[i].numDocs();
return n;
}
// override this as MultiReader precalculates the number of deletions
// (this method is never used by MultiPassIndexSplitter)
@Override
public boolean hasDeletions() {
return (maxDoc() != numDocs());
}
// no need to override numDocs/hasDeletions,
// as we pass the subreaders directly to IW.addIndexes().
}
private static final class FakeDeleteAtomicIndexReader extends FilterIndexReader {

15
lucene/src/java/org/apache/lucene/index/BaseMultiReader.java
View File

@ -19,13 +19,12 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ReaderUtil;
abstract class BaseMultiReader<R extends IndexReader> extends CompositeReader {
protected final R[] subReaders;
protected final int[] starts; // 1st docno for each segment
protected final int[] starts; // 1st docno for each reader
private final int maxDoc;
private final int numDocs;
private final boolean hasDeletions;
@ -50,14 +49,14 @@ abstract class BaseMultiReader<R extends IndexReader> extends CompositeReader {
}
@Override
public Fields getTermVectors(int docID) throws IOException {
public final Fields getTermVectors(int docID) throws IOException {
ensureOpen();
final int i = readerIndex(docID); // find segment num
return subReaders[i].getTermVectors(docID - starts[i]); // dispatch to segment
}
@Override
public int numDocs() {
public final int numDocs() {
// Don't call ensureOpen() here (it could affect performance)
return numDocs;
}
@ -69,20 +68,20 @@ abstract class BaseMultiReader<R extends IndexReader> extends CompositeReader {
}
@Override
public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
public final void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
ensureOpen();
final int i = readerIndex(docID); // find segment num
subReaders[i].document(docID - starts[i], visitor); // dispatch to segment reader
}
@Override
public boolean hasDeletions() {
public final boolean hasDeletions() {
// Don't call ensureOpen() here (it could affect performance)
return hasDeletions;
}
@Override
public int docFreq(String field, BytesRef t) throws IOException {
public final int docFreq(String field, BytesRef t) throws IOException {
ensureOpen();
int total = 0; // sum freqs in segments
for (int i = 0; i < subReaders.length; i++) {
@ -100,7 +99,7 @@ abstract class BaseMultiReader<R extends IndexReader> extends CompositeReader {
}
@Override
public IndexReader[] getSequentialSubReaders() {
public final R[] getSequentialSubReaders() {
return subReaders;
}
}

553
lucene/src/java/org/apache/lucene/index/DirectoryReader.java
View File

@ -20,16 +20,12 @@ package org.apache.lucene.index;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.search.SearcherManager; // javadocs
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.IOUtils;
/** DirectoryReader is an implementation of {@link CompositeReader}
that can read indexes in a {@link Directory}.
@ -52,15 +48,16 @@ import org.apache.lucene.util.IOUtils;
synchronization, you should <b>not</b> synchronize on the
<code>IndexReader</code> instance; use your own
(non-Lucene) objects instead.
<p><em>Please note:</em> This class extends from an internal (invisible)
superclass that is generic: The type parameter {@code R} is
{@link AtomicReader}, see {@link #subReaders} and
{@link #getSequentialSubReaders}.
*/
public final class DirectoryReader extends BaseMultiReader<SegmentReader> {
static int DEFAULT_TERMS_INDEX_DIVISOR = 1;
public abstract class DirectoryReader extends BaseMultiReader<AtomicReader> {
public static final int DEFAULT_TERMS_INDEX_DIVISOR = 1;
protected final Directory directory;
private final IndexWriter writer;
private final SegmentInfos segmentInfos;
private final int termInfosIndexDivisor;
private final boolean applyAllDeletes;
/** Returns a IndexReader reading the index in the given
* Directory
@ -69,7 +66,7 @@ public final class DirectoryReader extends BaseMultiReader<SegmentReader> {
* @throws IOException if there is a low-level IO error
*/
public static DirectoryReader open(final Directory directory) throws CorruptIndexException, IOException {
return open(directory, null, DEFAULT_TERMS_INDEX_DIVISOR);
return StandardDirectoryReader.open(directory, null, DEFAULT_TERMS_INDEX_DIVISOR);
}
/** Expert: Returns a IndexReader reading the index in the given
@ -89,7 +86,7 @@ public final class DirectoryReader extends BaseMultiReader<SegmentReader> {
* @throws IOException if there is a low-level IO error
*/
public static DirectoryReader open(final Directory directory, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
return open(directory, null, termInfosIndexDivisor);
return StandardDirectoryReader.open(directory, null, termInfosIndexDivisor);
}
/**
@ -122,10 +119,9 @@ public final class DirectoryReader extends BaseMultiReader<SegmentReader> {
* @throws IOException if there is a low-level IO error
*/
public static DirectoryReader open(final IndexCommit commit) throws CorruptIndexException, IOException {
return open(commit.getDirectory(), commit, DEFAULT_TERMS_INDEX_DIVISOR);
return StandardDirectoryReader.open(commit.getDirectory(), commit, DEFAULT_TERMS_INDEX_DIVISOR);
}
/** Expert: returns an IndexReader reading the index in the given
* {@link IndexCommit} and termInfosIndexDivisor.
* @param commit the commit point to open
@ -143,169 +139,7 @@ public final class DirectoryReader extends BaseMultiReader<SegmentReader> {
* @throws IOException if there is a low-level IO error
*/
public static DirectoryReader open(final IndexCommit commit, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
return open(commit.getDirectory(), commit, termInfosIndexDivisor);
}
DirectoryReader(SegmentReader[] readers, Directory directory, IndexWriter writer,
SegmentInfos sis, int termInfosIndexDivisor, boolean applyAllDeletes) throws IOException {
super(readers);
this.directory = directory;
this.writer = writer;
this.segmentInfos = sis;
this.termInfosIndexDivisor = termInfosIndexDivisor;
this.applyAllDeletes = applyAllDeletes;
}
private static DirectoryReader open(final Directory directory, final IndexCommit commit,
final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
return (DirectoryReader) new SegmentInfos.FindSegmentsFile(directory) {
@Override
protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
SegmentInfos sis = new SegmentInfos();
sis.read(directory, segmentFileName);
final SegmentReader[] readers = new SegmentReader[sis.size()];
for (int i = sis.size()-1; i >= 0; i--) {
IOException prior = null;
boolean success = false;
try {
readers[i] = new SegmentReader(sis.info(i), termInfosIndexDivisor, IOContext.READ);
success = true;
} catch(IOException ex) {
prior = ex;
} finally {
if (!success)
IOUtils.closeWhileHandlingException(prior, readers);
}
}
return new DirectoryReader(readers, directory, null, sis, termInfosIndexDivisor, false);
}
}.run(commit);
}
// Used by near real-time search
static DirectoryReader open(IndexWriter writer, SegmentInfos infos, boolean applyAllDeletes) throws IOException {
// IndexWriter synchronizes externally before calling
// us, which ensures infos will not change; so there's
// no need to process segments in reverse order
final int numSegments = infos.size();
List<SegmentReader> readers = new ArrayList<SegmentReader>();
final Directory dir = writer.getDirectory();
final SegmentInfos segmentInfos = (SegmentInfos) infos.clone();
int infosUpto = 0;
for (int i=0;i<numSegments;i++) {
IOException prior = null;
boolean success = false;
try {
final SegmentInfo info = infos.info(i);
assert info.dir == dir;
final IndexWriter.ReadersAndLiveDocs rld = writer.readerPool.get(info, true);
final SegmentReader reader = rld.getReadOnlyClone(IOContext.READ);
if (reader.numDocs() > 0 || writer.getKeepFullyDeletedSegments()) {
readers.add(reader);
infosUpto++;
} else {
reader.close();
segmentInfos.remove(infosUpto);
}
success = true;
} catch(IOException ex) {
prior = ex;
} finally {
if (!success)
IOUtils.closeWhileHandlingException(prior, readers);
}
}
return new DirectoryReader(readers.toArray(new SegmentReader[readers.size()]),
dir, writer, segmentInfos, writer.getConfig().getReaderTermsIndexDivisor(), applyAllDeletes);
}
/** This constructor is only used for {@link #doOpenIfChanged()} */
private static DirectoryReader open(Directory directory, IndexWriter writer, SegmentInfos infos, SegmentReader[] oldReaders,
int termInfosIndexDivisor) throws IOException {
// we put the old SegmentReaders in a map, that allows us
// to lookup a reader using its segment name
final Map<String,Integer> segmentReaders = new HashMap<String,Integer>();
if (oldReaders != null) {
// create a Map SegmentName->SegmentReader
for (int i = 0; i < oldReaders.length; i++) {
segmentReaders.put(oldReaders[i].getSegmentName(), Integer.valueOf(i));
}
}
SegmentReader[] newReaders = new SegmentReader[infos.size()];
// remember which readers are shared between the old and the re-opened
// DirectoryReader - we have to incRef those readers
boolean[] readerShared = new boolean[infos.size()];
for (int i = infos.size() - 1; i>=0; i--) {
// find SegmentReader for this segment
Integer oldReaderIndex = segmentReaders.get(infos.info(i).name);
if (oldReaderIndex == null) {
// this is a new segment, no old SegmentReader can be reused
newReaders[i] = null;
} else {
// there is an old reader for this segment - we'll try to reopen it
newReaders[i] = oldReaders[oldReaderIndex.intValue()];
}
boolean success = false;
IOException prior = null;
try {
SegmentReader newReader;
if (newReaders[i] == null || infos.info(i).getUseCompoundFile() != newReaders[i].getSegmentInfo().getUseCompoundFile()) {
// this is a new reader; in case we hit an exception we can close it safely
newReader = new SegmentReader(infos.info(i), termInfosIndexDivisor, IOContext.READ);
readerShared[i] = false;
newReaders[i] = newReader;
} else {
if (newReaders[i].getSegmentInfo().getDelGen() == infos.info(i).getDelGen()) {
// No change; this reader will be shared between
// the old and the new one, so we must incRef
// it:
readerShared[i] = true;
newReaders[i].incRef();
} else {
readerShared[i] = false;
// Steal the ref returned by SegmentReader ctor:
assert infos.info(i).dir == newReaders[i].getSegmentInfo().dir;
assert infos.info(i).hasDeletions();
newReaders[i] = new SegmentReader(infos.info(i), newReaders[i].core, IOContext.READ);
}
}
success = true;
} catch (IOException ex) {
prior = ex;
} finally {
if (!success) {
for (i++; i < infos.size(); i++) {
if (newReaders[i] != null) {
try {
if (!readerShared[i]) {
// this is a new subReader that is not used by the old one,
// we can close it
newReaders[i].close();
} else {
// this subReader is also used by the old reader, so instead
// closing we must decRef it
newReaders[i].decRef();
}
} catch (IOException ex) {
if (prior == null) prior = ex;
}
}
}
}
// throw the first exception
if (prior != null) throw prior;
}
}
return new DirectoryReader(newReaders, directory, writer,
infos, termInfosIndexDivisor, false);
return StandardDirectoryReader.open(commit.getDirectory(), commit, termInfosIndexDivisor);
}
/**
@ -418,209 +252,6 @@ public final class DirectoryReader extends BaseMultiReader<SegmentReader> {
return newReader;
}
/** {@inheritDoc} */
@Override
public String toString() {
final StringBuilder buffer = new StringBuilder();
buffer.append(getClass().getSimpleName());
buffer.append('(');
final String segmentsFile = segmentInfos.getCurrentSegmentFileName();
if (segmentsFile != null) {
buffer.append(segmentsFile).append(":").append(segmentInfos.getVersion());
}
if (writer != null) {
buffer.append(":nrt");
}
for(int i=0;i<subReaders.length;i++) {
buffer.append(' ');
buffer.append(subReaders[i]);
}
buffer.append(')');
return buffer.toString();
}
protected final DirectoryReader doOpenIfChanged() throws CorruptIndexException, IOException {
return doOpenIfChanged(null);
}
protected final DirectoryReader doOpenIfChanged(final IndexCommit commit) throws CorruptIndexException, IOException {
ensureOpen();
// If we were obtained by writer.getReader(), re-ask the
// writer to get a new reader.
if (writer != null) {
return doOpenFromWriter(commit);
} else {
return doOpenNoWriter(commit);
}
}
protected final DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws CorruptIndexException, IOException {
ensureOpen();
if (writer == this.writer && applyAllDeletes == this.applyAllDeletes) {
return doOpenFromWriter(null);
} else {
return writer.getReader(applyAllDeletes);
}
}
private final DirectoryReader doOpenFromWriter(IndexCommit commit) throws CorruptIndexException, IOException {
if (commit != null) {
throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit");
}
if (writer.nrtIsCurrent(segmentInfos)) {
return null;
}
DirectoryReader reader = writer.getReader(applyAllDeletes);
// If in fact no changes took place, return null:
if (reader.getVersion() == segmentInfos.getVersion()) {
reader.decRef();
return null;
}
return reader;
}
private synchronized DirectoryReader doOpenNoWriter(IndexCommit commit) throws CorruptIndexException, IOException {
if (commit == null) {
if (isCurrent()) {
return null;
}
} else {
if (directory != commit.getDirectory()) {
throw new IOException("the specified commit does not match the specified Directory");
}
if (segmentInfos != null && commit.getSegmentsFileName().equals(segmentInfos.getCurrentSegmentFileName())) {
return null;
}
}
return (DirectoryReader) new SegmentInfos.FindSegmentsFile(directory) {
@Override
protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
final SegmentInfos infos = new SegmentInfos();
infos.read(directory, segmentFileName);
return doOpenIfChanged(infos, null);
}
}.run(commit);
}
private synchronized DirectoryReader doOpenIfChanged(SegmentInfos infos, IndexWriter writer) throws CorruptIndexException, IOException {
return DirectoryReader.open(directory, writer, infos, subReaders, termInfosIndexDivisor);
}
/**
* Version number when this IndexReader was opened. Not
* implemented in the IndexReader base class.
*
* <p>This method
* returns the version recorded in the commit that the
* reader opened. This version is advanced every time
* a change is made with {@link IndexWriter}.</p>
*/
public long getVersion() {
ensureOpen();
return segmentInfos.getVersion();
}
/**
* Retrieve the String userData optionally passed to
* IndexWriter#commit. This will return null if {@link
* IndexWriter#commit(Map)} has never been called for
* this index.
*/
public Map<String,String> getCommitUserData() {
ensureOpen();
return segmentInfos.getUserData();
}
/**
* Check whether any new changes have occurred to the
* index since this reader was opened.
*
* <p>If this reader was created by calling {@link #open},
* then this method checks if any further commits
* (see {@link IndexWriter#commit}) have occurred in the
* directory.</p>
*
* <p>If instead this reader is a near real-time reader
* (ie, obtained by a call to {@link
* IndexWriter#getReader}, or by calling {@link #openIfChanged}
* on a near real-time reader), then this method checks if
* either a new commmit has occurred, or any new
* uncommitted changes have taken place via the writer.
* Note that even if the writer has only performed
* merging, this method will still return false.</p>
*
* <p>In any event, if this returns false, you should call
* {@link #openIfChanged} to get a new reader that sees the
* changes.</p>
*
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
public boolean isCurrent() throws CorruptIndexException, IOException {
ensureOpen();
if (writer == null || writer.isClosed()) {
// we loaded SegmentInfos from the directory
return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion();
} else {
return writer.nrtIsCurrent(segmentInfos);
}
}
@Override
protected synchronized void doClose() throws IOException {
IOException ioe = null;
for (int i = 0; i < subReaders.length; i++) {
// try to close each reader, even if an exception is thrown
try {
subReaders[i].decRef();
} catch (IOException e) {
if (ioe == null) ioe = e;
}
}
if (writer != null) {
// Since we just closed, writer may now be able to
// delete unused files:
writer.deletePendingFiles();
}
// throw the first exception
if (ioe != null) throw ioe;
}
/** Returns the directory this index resides in. */
public Directory directory() {
// Don't ensureOpen here -- in certain cases, when a
// cloned/reopened reader needs to commit, it may call
// this method on the closed original reader
return directory;
}
/** This returns the current indexDivisor as
* specified when the reader was opened.
*/
public int getTermInfosIndexDivisor() {
ensureOpen();
return termInfosIndexDivisor;
}
/**
* Expert: return the IndexCommit that this reader has opened.
* <p/>
* @lucene.experimental
*/
public IndexCommit getIndexCommit() throws IOException {
ensureOpen();
return new ReaderCommit(segmentInfos, directory);
}
/** Returns all commit points that exist in the Directory.
* Normally, because the default is {@link
* KeepOnlyLastCommitDeletionPolicy}, there would be only
@ -645,7 +276,7 @@ public final class DirectoryReader extends BaseMultiReader<SegmentReader> {
latest.read(dir);
final long currentGen = latest.getGeneration();
commits.add(new ReaderCommit(latest, dir));
commits.add(new StandardDirectoryReader.ReaderCommit(latest, dir));
for(int i=0;i<files.length;i++) {
@ -672,7 +303,7 @@ public final class DirectoryReader extends BaseMultiReader<SegmentReader> {
}
if (sis != null)
commits.add(new ReaderCommit(sis, dir));
commits.add(new StandardDirectoryReader.ReaderCommit(sis, dir));
}
}
@ -680,7 +311,7 @@ public final class DirectoryReader extends BaseMultiReader<SegmentReader> {
Collections.sort(commits);
return commits;
}
}
/**
* Reads version number from segments files. The version number is
@ -729,66 +360,100 @@ public final class DirectoryReader extends BaseMultiReader<SegmentReader> {
}
}
private static final class ReaderCommit extends IndexCommit {
private String segmentsFileName;
Collection<String> files;
Directory dir;
long generation;
final Map<String,String> userData;
private final int segmentCount;
ReaderCommit(SegmentInfos infos, Directory dir) throws IOException {
segmentsFileName = infos.getCurrentSegmentFileName();
this.dir = dir;
userData = infos.getUserData();
files = Collections.unmodifiableCollection(infos.files(dir, true));
generation = infos.getGeneration();
segmentCount = infos.size();
}
@Override
public String toString() {
return "DirectoryReader.ReaderCommit(" + segmentsFileName + ")";
}
@Override
public int getSegmentCount() {
return segmentCount;
}
@Override
public String getSegmentsFileName() {
return segmentsFileName;
}
@Override
public Collection<String> getFileNames() {
return files;
}
@Override
public Directory getDirectory() {
return dir;
}
@Override
public long getGeneration() {
return generation;
}
@Override
public boolean isDeleted() {
return false;
}
@Override
public Map<String,String> getUserData() {
return userData;
}
@Override
public void delete() {
throw new UnsupportedOperationException("This IndexCommit does not support deletions");
}
protected DirectoryReader(Directory directory, AtomicReader[] readers) throws CorruptIndexException, IOException {
super(readers);
this.directory = directory;
}
/** Returns the directory this index resides in. */
public final Directory directory() {
// Don't ensureOpen here -- in certain cases, when a
// cloned/reopened reader needs to commit, it may call
// this method on the closed original reader
return directory;
}
/** Implement this method to support {@link #openIfChanged(DirectoryReader)}.
* If this reader does not support reopen, return {@code null}, so
* client code is happy. This should be consistent with {@link #isCurrent}
* (should always return {@code true}) if reopen is not supported.
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
* @return null if there are no changes; else, a new
* DirectoryReader instance.
*/
protected abstract DirectoryReader doOpenIfChanged() throws CorruptIndexException, IOException;
/** Implement this method to support {@link #openIfChanged(DirectoryReader,IndexCommit)}.
* If this reader does not support reopen from a specific {@link IndexCommit},
* throw {@link UnsupportedOperationException}.
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
* @return null if there are no changes; else, a new
* DirectoryReader instance.
*/
protected abstract DirectoryReader doOpenIfChanged(final IndexCommit commit) throws CorruptIndexException, IOException;
/** Implement this method to support {@link #openIfChanged(DirectoryReader,IndexWriter,boolean)}.
* If this reader does not support reopen from {@link IndexWriter},
* throw {@link UnsupportedOperationException}.
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
* @return null if there are no changes; else, a new
* DirectoryReader instance.
*/
protected abstract DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws CorruptIndexException, IOException;
/**
* Version number when this IndexReader was opened.
*
* <p>This method
* returns the version recorded in the commit that the
* reader opened. This version is advanced every time
* a change is made with {@link IndexWriter}.</p>
*/
public abstract long getVersion();
/**
* Retrieve the String userData optionally passed to
* IndexWriter#commit. This will return null if {@link
* IndexWriter#commit(Map)} has never been called for
* this index.
*/
public abstract Map<String,String> getCommitUserData();
/**
* Check whether any new changes have occurred to the
* index since this reader was opened.
*
* <p>If this reader was created by calling {@link #open},
* then this method checks if any further commits
* (see {@link IndexWriter#commit}) have occurred in the
* directory.</p>
*
* <p>If instead this reader is a near real-time reader
* (ie, obtained by a call to {@link
* IndexWriter#getReader}, or by calling {@link #openIfChanged}
* on a near real-time reader), then this method checks if
* either a new commmit has occurred, or any new
* uncommitted changes have taken place via the writer.
* Note that even if the writer has only performed
* merging, this method will still return false.</p>
*
* <p>In any event, if this returns false, you should call
* {@link #openIfChanged} to get a new reader that sees the
* changes.</p>
*
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
public abstract boolean isCurrent() throws CorruptIndexException, IOException;
/**
* Expert: return the IndexCommit that this reader has opened.
* <p/>
* @lucene.experimental
*/
public abstract IndexCommit getIndexCommit() throws CorruptIndexException, IOException;
}

2
lucene/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -364,7 +364,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
// just like we do when loading segments_N
synchronized(this) {
maybeApplyDeletes(applyAllDeletes);
r = DirectoryReader.open(this, segmentInfos, applyAllDeletes);
r = StandardDirectoryReader.open(this, segmentInfos, applyAllDeletes);
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "return reader version=" + r.getVersion() + " reader=" + r);
}

8
lucene/src/java/org/apache/lucene/index/MultiReader.java
View File

@ -20,7 +20,13 @@ package org.apache.lucene.index;
import java.io.IOException;
/** An IndexReader which reads multiple indexes, appending
* their content. */
* their content.
<p><em>Please note:</em> This class extends from an internal (invisible)
superclass that is generic: The type parameter {@code R} is
{@link IndexReader}, see {@link #subReaders} and
{@link #getSequentialSubReaders}.
*/
public class MultiReader extends BaseMultiReader<IndexReader> {
private final boolean closeSubReaders;

411
lucene/src/java/org/apache/lucene/index/StandardDirectoryReader.java Normal file
View File

@ -0,0 +1,411 @@
package org.apache.lucene.index;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.IOUtils;
final class StandardDirectoryReader extends DirectoryReader {
private final IndexWriter writer;
private final SegmentInfos segmentInfos;
private final int termInfosIndexDivisor;
private final boolean applyAllDeletes;
/** called only from static open() methods */
StandardDirectoryReader(Directory directory, AtomicReader[] readers, IndexWriter writer,
SegmentInfos sis, int termInfosIndexDivisor, boolean applyAllDeletes) throws IOException {
super(directory, readers);
this.writer = writer;
this.segmentInfos = sis;
this.termInfosIndexDivisor = termInfosIndexDivisor;
this.applyAllDeletes = applyAllDeletes;
}
/** called from DirectoryReader.open(...) methods */
static DirectoryReader open(final Directory directory, final IndexCommit commit,
final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
return (DirectoryReader) new SegmentInfos.FindSegmentsFile(directory) {
@Override
protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
SegmentInfos sis = new SegmentInfos();
sis.read(directory, segmentFileName);
final SegmentReader[] readers = new SegmentReader[sis.size()];
for (int i = sis.size()-1; i >= 0; i--) {
IOException prior = null;
boolean success = false;
try {
readers[i] = new SegmentReader(sis.info(i), termInfosIndexDivisor, IOContext.READ);
success = true;
} catch(IOException ex) {
prior = ex;
} finally {
if (!success)
IOUtils.closeWhileHandlingException(prior, readers);
}
}
return new StandardDirectoryReader(directory, readers, null, sis, termInfosIndexDivisor, false);
}
}.run(commit);
}
/** Used by near real-time search */
static DirectoryReader open(IndexWriter writer, SegmentInfos infos, boolean applyAllDeletes) throws IOException {
// IndexWriter synchronizes externally before calling
// us, which ensures infos will not change; so there's
// no need to process segments in reverse order
final int numSegments = infos.size();
List<SegmentReader> readers = new ArrayList<SegmentReader>();
final Directory dir = writer.getDirectory();
final SegmentInfos segmentInfos = (SegmentInfos) infos.clone();
int infosUpto = 0;
for (int i=0;i<numSegments;i++) {
IOException prior = null;
boolean success = false;
try {
final SegmentInfo info = infos.info(i);
assert info.dir == dir;
final IndexWriter.ReadersAndLiveDocs rld = writer.readerPool.get(info, true);
final SegmentReader reader = rld.getReadOnlyClone(IOContext.READ);
if (reader.numDocs() > 0 || writer.getKeepFullyDeletedSegments()) {
readers.add(reader);
infosUpto++;
} else {
reader.close();
segmentInfos.remove(infosUpto);
}
success = true;
} catch(IOException ex) {
prior = ex;
} finally {
if (!success)
IOUtils.closeWhileHandlingException(prior, readers);
}
}
return new StandardDirectoryReader(dir, readers.toArray(new SegmentReader[readers.size()]),
writer, segmentInfos, writer.getConfig().getReaderTermsIndexDivisor(), applyAllDeletes);
}
/** This constructor is only used for {@link #doOpenIfChanged()} */
private static DirectoryReader open(Directory directory, IndexWriter writer, SegmentInfos infos, AtomicReader[] oldReaders,
int termInfosIndexDivisor) throws IOException {
// we put the old SegmentReaders in a map, that allows us
// to lookup a reader using its segment name
final Map<String,Integer> segmentReaders = new HashMap<String,Integer>();
if (oldReaders != null) {
// create a Map SegmentName->SegmentReader
for (int i = 0; i < oldReaders.length; i++) {
segmentReaders.put(((SegmentReader) oldReaders[i]).getSegmentName(), Integer.valueOf(i));
}
}
SegmentReader[] newReaders = new SegmentReader[infos.size()];
// remember which readers are shared between the old and the re-opened
// DirectoryReader - we have to incRef those readers
boolean[] readerShared = new boolean[infos.size()];
for (int i = infos.size() - 1; i>=0; i--) {
// find SegmentReader for this segment
Integer oldReaderIndex = segmentReaders.get(infos.info(i).name);
if (oldReaderIndex == null) {
// this is a new segment, no old SegmentReader can be reused
newReaders[i] = null;
} else {
// there is an old reader for this segment - we'll try to reopen it
newReaders[i] = (SegmentReader) oldReaders[oldReaderIndex.intValue()];
}
boolean success = false;
IOException prior = null;
try {
SegmentReader newReader;
if (newReaders[i] == null || infos.info(i).getUseCompoundFile() != newReaders[i].getSegmentInfo().getUseCompoundFile()) {
// this is a new reader; in case we hit an exception we can close it safely
newReader = new SegmentReader(infos.info(i), termInfosIndexDivisor, IOContext.READ);
readerShared[i] = false;
newReaders[i] = newReader;
} else {
if (newReaders[i].getSegmentInfo().getDelGen() == infos.info(i).getDelGen()) {
// No change; this reader will be shared between
// the old and the new one, so we must incRef
// it:
readerShared[i] = true;
newReaders[i].incRef();
} else {
readerShared[i] = false;
// Steal the ref returned by SegmentReader ctor:
assert infos.info(i).dir == newReaders[i].getSegmentInfo().dir;
assert infos.info(i).hasDeletions();
newReaders[i] = new SegmentReader(infos.info(i), newReaders[i].core, IOContext.READ);
}
}
success = true;
} catch (IOException ex) {
prior = ex;
} finally {
if (!success) {
for (i++; i < infos.size(); i++) {
if (newReaders[i] != null) {
try {
if (!readerShared[i]) {
// this is a new subReader that is not used by the old one,
// we can close it
newReaders[i].close();
} else {
// this subReader is also used by the old reader, so instead
// closing we must decRef it
newReaders[i].decRef();
}
} catch (IOException ex) {
if (prior == null) prior = ex;
}
}
}
}
// throw the first exception
if (prior != null) throw prior;
}
}
return new StandardDirectoryReader(directory, newReaders, writer, infos, termInfosIndexDivisor, false);
}
@Override
public String toString() {
final StringBuilder buffer = new StringBuilder();
buffer.append(getClass().getSimpleName());
buffer.append('(');
final String segmentsFile = segmentInfos.getCurrentSegmentFileName();
if (segmentsFile != null) {
buffer.append(segmentsFile).append(":").append(segmentInfos.getVersion());
}
if (writer != null) {
buffer.append(":nrt");
}
for(int i=0;i<subReaders.length;i++) {
buffer.append(' ');
buffer.append(subReaders[i]);
}
buffer.append(')');
return buffer.toString();
}
@Override
protected final DirectoryReader doOpenIfChanged() throws CorruptIndexException, IOException {
return doOpenIfChanged(null);
}
@Override
protected final DirectoryReader doOpenIfChanged(final IndexCommit commit) throws CorruptIndexException, IOException {
ensureOpen();
// If we were obtained by writer.getReader(), re-ask the
// writer to get a new reader.
if (writer != null) {
return doOpenFromWriter(commit);
} else {
return doOpenNoWriter(commit);
}
}
@Override
protected final DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws CorruptIndexException, IOException {
ensureOpen();
if (writer == this.writer && applyAllDeletes == this.applyAllDeletes) {
return doOpenFromWriter(null);
} else {
return writer.getReader(applyAllDeletes);
}
}
private final DirectoryReader doOpenFromWriter(IndexCommit commit) throws CorruptIndexException, IOException {
if (commit != null) {
throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit");
}
if (writer.nrtIsCurrent(segmentInfos)) {
return null;
}
DirectoryReader reader = writer.getReader(applyAllDeletes);
// If in fact no changes took place, return null:
if (reader.getVersion() == segmentInfos.getVersion()) {
reader.decRef();
return null;
}
return reader;
}
private synchronized DirectoryReader doOpenNoWriter(IndexCommit commit) throws CorruptIndexException, IOException {
if (commit == null) {
if (isCurrent()) {
return null;
}
} else {
if (directory != commit.getDirectory()) {
throw new IOException("the specified commit does not match the specified Directory");
}
if (segmentInfos != null && commit.getSegmentsFileName().equals(segmentInfos.getCurrentSegmentFileName())) {
return null;
}
}
return (DirectoryReader) new SegmentInfos.FindSegmentsFile(directory) {
@Override
protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
final SegmentInfos infos = new SegmentInfos();
infos.read(directory, segmentFileName);
return doOpenIfChanged(infos, null);
}
}.run(commit);
}
synchronized DirectoryReader doOpenIfChanged(SegmentInfos infos, IndexWriter writer) throws CorruptIndexException, IOException {
return StandardDirectoryReader.open(directory, writer, infos, subReaders, termInfosIndexDivisor);
}
@Override
public long getVersion() {
ensureOpen();
return segmentInfos.getVersion();
}
@Override
public Map<String,String> getCommitUserData() {
ensureOpen();
return segmentInfos.getUserData();
}
@Override
public boolean isCurrent() throws CorruptIndexException, IOException {
ensureOpen();
if (writer == null || writer.isClosed()) {
// we loaded SegmentInfos from the directory
return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion();
} else {
return writer.nrtIsCurrent(segmentInfos);
}
}
@Override
protected synchronized void doClose() throws IOException {
IOException ioe = null;
for (int i = 0; i < subReaders.length; i++) {
// try to close each reader, even if an exception is thrown
try {
subReaders[i].decRef();
} catch (IOException e) {
if (ioe == null) ioe = e;
}
}
if (writer != null) {
// Since we just closed, writer may now be able to
// delete unused files:
writer.deletePendingFiles();
}
// throw the first exception
if (ioe != null) throw ioe;
}
@Override
public IndexCommit getIndexCommit() throws IOException {
ensureOpen();
return new ReaderCommit(segmentInfos, directory);
}
static final class ReaderCommit extends IndexCommit {
private String segmentsFileName;
Collection<String> files;
Directory dir;
long generation;
final Map<String,String> userData;
private final int segmentCount;
ReaderCommit(SegmentInfos infos, Directory dir) throws IOException {
segmentsFileName = infos.getCurrentSegmentFileName();
this.dir = dir;
userData = infos.getUserData();
files = Collections.unmodifiableCollection(infos.files(dir, true));
generation = infos.getGeneration();
segmentCount = infos.size();
}
@Override
public String toString() {
return "DirectoryReader.ReaderCommit(" + segmentsFileName + ")";
}
@Override
public int getSegmentCount() {
return segmentCount;
}
@Override
public String getSegmentsFileName() {
return segmentsFileName;
}
@Override
public Collection<String> getFileNames() {
return files;
}
@Override
public Directory getDirectory() {
return dir;
}
@Override
public long getGeneration() {
return generation;
}
@Override
public boolean isDeleted() {
return false;
}
@Override
public Map<String,String> getUserData() {
return userData;
}
@Override
public void delete() {
throw new UnsupportedOperationException("This IndexCommit does not support deletions");
}
}
}

2
lucene/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
View File

@ -696,7 +696,7 @@ public class TestDocValuesIndexing extends LuceneTestCase {
w.forceMerge(1);
DirectoryReader r = w.getReader();
w.close();
assertEquals(17, ((AtomicReader) r.getSequentialSubReaders()[0]).docValues("field").load().getInt(0));
assertEquals(17, r.getSequentialSubReaders()[0].docValues("field").load().getInt(0));
r.close();
d.close();
}

2
lucene/src/test/org/apache/lucene/index/TestIndexReader.java
View File

@ -700,7 +700,7 @@ public class TestIndexReader extends LuceneTestCase {
DirectoryReader r2 = DirectoryReader.openIfChanged(r);
assertNotNull(r2);
r.close();
AtomicReader sub0 = (AtomicReader) r2.getSequentialSubReaders()[0];
AtomicReader sub0 = r2.getSequentialSubReaders()[0];
final int[] ints2 = FieldCache.DEFAULT.getInts(sub0, "number", false);
r2.close();
assertTrue(ints == ints2);

3
solr/core/src/test/org/apache/solr/core/TestConfig.java
View File

@ -126,9 +126,6 @@ public class TestConfig extends SolrTestCaseJ4 {
IndexReaderFactory irf = h.getCore().getIndexReaderFactory();
StandardIndexReaderFactory sirf = (StandardIndexReaderFactory) irf;
assertEquals(12, sirf.termInfosIndexDivisor);
SolrQueryRequest req = req();
assertEquals(12, req.getSearcher().getIndexReader().getTermInfosIndexDivisor());
req.close();
}