From 0b000470a71db1d19ee80813a62622daf2e7ccde Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Wed, 16 Dec 2009 18:51:44 +0000 Subject: [PATCH] LUCENE-2161: improve concurrency of IndexReader git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@891377 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 3 + .../apache/lucene/index/DirectoryReader.java | 55 +++++++++++-------- .../org/apache/lucene/index/MultiReader.java | 4 +- .../apache/lucene/index/SegmentReader.java | 26 ++++++--- .../org/apache/lucene/util/BitVector.java | 17 +++++- 5 files changed, 71 insertions(+), 34 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index cf45cc5152d..98eee52d9c2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -109,6 +109,9 @@ Optimizations the FieldCache rather than waiting for the WeakHashMap to release the reference (Mike McCandless) +* LUCENE-2161: Improve concurrency of IndexReader, especially in the + context of near real-time readers. (Mike McCandless) + Build * LUCENE-2124: Moved the JDK-based collation support from contrib/collation diff --git a/src/java/org/apache/lucene/index/DirectoryReader.java b/src/java/org/apache/lucene/index/DirectoryReader.java index 6234dcd4a83..c9d648765f2 100644 --- a/src/java/org/apache/lucene/index/DirectoryReader.java +++ b/src/java/org/apache/lucene/index/DirectoryReader.java @@ -125,7 +125,7 @@ class DirectoryReader extends IndexReader implements Cloneable { DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException { this.directory = writer.getDirectory(); this.readOnly = true; - this.segmentInfos = infos; + segmentInfos = infos; segmentInfosStart = (SegmentInfos) infos.clone(); this.termInfosIndexDivisor = termInfosIndexDivisor; if (!readOnly) { @@ -345,22 +345,39 @@ class DirectoryReader extends IndexReader implements Cloneable { } @Override - public final synchronized IndexReader reopen() throws CorruptIndexException, IOException { + public final IndexReader reopen() throws CorruptIndexException, IOException { // Preserve current readOnly return doReopen(readOnly, null); } @Override - public final synchronized IndexReader reopen(boolean openReadOnly) throws CorruptIndexException, IOException { + public final IndexReader reopen(boolean openReadOnly) throws CorruptIndexException, IOException { return doReopen(openReadOnly, null); } @Override - public final synchronized IndexReader reopen(final IndexCommit commit) throws CorruptIndexException, IOException { + public final IndexReader reopen(final IndexCommit commit) throws CorruptIndexException, IOException { return doReopen(true, commit); } - private synchronized IndexReader doReopen(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException { + private final IndexReader doReopenFromWriter(boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException { + assert readOnly; + + if (!openReadOnly) { + throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() can only be reopened with openReadOnly=true (got false)"); + } + + if (commit != null) { + throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit"); + } + + // TODO: right now we *always* make a new reader; in + // the future we could have write make some effort to + // detect that no changes have occurred + return writer.getReader(); + } + + private IndexReader doReopen(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException { ensureOpen(); assert commit == null || openReadOnly; @@ -368,22 +385,13 @@ class DirectoryReader extends IndexReader implements Cloneable { // If we were obtained by writer.getReader(), re-ask the // writer to get a new reader. if (writer != null) { - assert readOnly; - - if (!openReadOnly) { - throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() can only be reopened with openReadOnly=true (got false)"); - } - - if (commit != null) { - throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader() cannot currently accept a commit"); - } - - // TODO: right now we *always* make a new reader; in - // the future we could have write make some effort to - // detect that no changes have occurred - IndexReader reader = writer.getReader(); - return reader; + return doReopenFromWriter(openReadOnly, commit); + } else { + return doReopenNoWriter(openReadOnly, commit); } + } + + private synchronized IndexReader doReopenNoWriter(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException, IOException { if (commit == null) { if (hasChanges) { @@ -487,10 +495,13 @@ class DirectoryReader extends IndexReader implements Cloneable { ensureOpen(); return segmentInfos.size() == 1 && !hasDeletions(); } - + @Override - public synchronized int numDocs() { + public int numDocs() { // Don't call ensureOpen() here (it could affect performance) + + // NOTE: multiple threads may wind up init'ing + // numDocs... but that's harmless if (numDocs == -1) { // check cache int n = 0; // cache miss--recompute for (int i = 0; i < subReaders.length; i++) diff --git a/src/java/org/apache/lucene/index/MultiReader.java b/src/java/org/apache/lucene/index/MultiReader.java index f6487bf5d3f..3933815b532 100644 --- a/src/java/org/apache/lucene/index/MultiReader.java +++ b/src/java/org/apache/lucene/index/MultiReader.java @@ -224,8 +224,10 @@ public class MultiReader extends IndexReader implements Cloneable { } @Override - public synchronized int numDocs() { + public int numDocs() { // Don't call ensureOpen() here (it could affect performance) + // NOTE: multiple threads may wind up init'ing + // numDocs... but that's harmless if (numDocs == -1) { // check cache int n = 0; // cache miss--recompute for (int i = 0; i < subReaders.length; i++) diff --git a/src/java/org/apache/lucene/index/SegmentReader.java b/src/java/org/apache/lucene/index/SegmentReader.java index b7eed7f840a..8b2f7f2982f 100644 --- a/src/java/org/apache/lucene/index/SegmentReader.java +++ b/src/java/org/apache/lucene/index/SegmentReader.java @@ -588,20 +588,28 @@ public class SegmentReader extends IndexReader implements Cloneable { core.openDocStores(si); } + private boolean checkDeletedCounts() throws IOException { + final int recomputedCount = deletedDocs.getRecomputedCount(); + + assert deletedDocs.count() == recomputedCount : "deleted count=" + deletedDocs.count() + " vs recomputed count=" + recomputedCount; + + assert si.getDelCount() == recomputedCount : + "delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + recomputedCount; + + // Verify # deletes does not exceed maxDoc for this + // segment: + assert si.getDelCount() <= maxDoc() : + "delete count mismatch: " + recomputedCount + ") exceeds max doc (" + maxDoc() + ") for segment " + si.name; + + return true; + } + private void loadDeletedDocs() throws IOException { // NOTE: the bitvector is stored using the regular directory, not cfs if (hasDeletions(si)) { deletedDocs = new BitVector(directory(), si.getDelFileName()); deletedDocsRef = new AtomicInteger(1); - - assert si.getDelCount() == deletedDocs.count() : - "delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + deletedDocs.count(); - - // Verify # deletes does not exceed maxDoc for this - // segment: - assert si.getDelCount() <= maxDoc() : - "delete count mismatch: " + deletedDocs.count() + ") exceeds max doc (" + maxDoc() + ") for segment " + si.name; - + assert checkDeletedCounts(); } else assert si.getDelCount() == 0; } diff --git a/src/java/org/apache/lucene/util/BitVector.java b/src/java/org/apache/lucene/util/BitVector.java index 1c451c7fa92..c661ba4fb0c 100644 --- a/src/java/org/apache/lucene/util/BitVector.java +++ b/src/java/org/apache/lucene/util/BitVector.java @@ -36,24 +36,28 @@ public final class BitVector implements Cloneable { private byte[] bits; private int size; - private int count = -1; + private int count; /** Constructs a vector capable of holding n bits. */ public BitVector(int n) { size = n; bits = new byte[(size >> 3) + 1]; + count = 0; } BitVector(byte[] bits, int size) { this.bits = bits; this.size = size; + count = -1; } @Override public Object clone() { byte[] copyBits = new byte[bits.length]; System.arraycopy(bits, 0, copyBits, 0, bits.length); - return new BitVector(copyBits, size); + BitVector clone = new BitVector(copyBits, size); + clone.count = count; + return clone; } /** Sets the value of bit to one. */ @@ -121,6 +125,15 @@ public final class BitVector implements Cloneable { return count; } + /** For testing */ + public final int getRecomputedCount() { + int c = 0; + int end = bits.length; + for (int i = 0; i < end; i++) + c += BYTE_COUNTS[bits[i] & 0xFF]; // sum bits per byte + return c; + } + private static final byte[] BYTE_COUNTS = { // table of bits/byte 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,