From 7b6955d42a9a437025ad5f0b1e0d57102b084291 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Wed, 15 Aug 2012 11:58:34 +0000 Subject: [PATCH] LUCENE-3892: merge trunk git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/pforcodec_3892@1373358 13f79535-47bb-0310-9956-ffa450edef68 --- .../MappingMultiDocsAndPositionsEnum.java | 13 +- .../lucene/codecs/TermVectorsWriter.java | 28 +- .../apache/lucene/codecs/TermsConsumer.java | 18 +- .../codecs/block/BlockPostingsWriter.java | 2 +- .../lucene40/Lucene40TermVectorsWriter.java | 15 +- .../org/apache/lucene/index/AtomicReader.java | 2 +- .../lucene/index/AtomicReaderContext.java | 3 +- .../lucene/index/BufferedDeletesStream.java | 2 +- .../apache/lucene/index/CoalescedDeletes.java | 1 - .../apache/lucene/index/CompositeReader.java | 16 +- .../lucene/index/CompositeReaderContext.java | 1 - .../org/apache/lucene/index/IndexReader.java | 28 +- .../lucene/index/IndexReaderContext.java | 1 - .../org/apache/lucene/index/IndexWriter.java | 53 +--- .../org/apache/lucene/index/MergeState.java | 10 - .../{util => index}/MergedIterator.java | 8 +- .../apache/lucene/index/MultiDocValues.java | 4 +- .../org/apache/lucene/index/MultiFields.java | 11 +- .../index/PayloadProcessorProvider.java | 81 ------ .../apache/lucene/index/SegmentMerger.java | 12 +- .../index/SlowCompositeReaderWrapper.java | 2 +- .../apache/lucene/search/IndexSearcher.java | 8 +- .../lucene/search/QueryWrapperFilter.java | 2 +- .../lucene/search/TermCollectingRewrite.java | 2 +- .../search/payloads/PayloadSpanUtil.java | 2 +- .../lucene/util/FieldCacheSanityChecker.java | 24 +- .../codecs/lucene40/TestReuseDocsEnum.java | 25 +- .../apache/lucene/index/TestCustomNorms.java | 2 +- .../lucene/index/TestDeletionPolicy.java | 14 +- .../lucene/index/TestDirectoryReader.java | 19 +- .../index/TestDirectoryReaderReopen.java | 79 ++--- .../test/org/apache/lucene/index/TestDoc.java | 2 +- .../apache/lucene/index/TestDocTermOrds.java | 12 +- .../lucene/index/TestDocValuesIndexing.java | 2 +- .../lucene/index/TestDocsAndPositions.java | 8 +- .../apache/lucene/index/TestFieldsReader.java | 5 +- .../apache/lucene/index/TestIndexWriter.java | 6 +- .../lucene/index/TestIndexWriterCommit.java | 6 +- .../index/TestIndexWriterExceptions.java | 2 +- .../index/TestIndexWriterForceMerge.java | 4 +- .../lucene/index/TestIndexWriterUnicode.java | 2 +- .../index/TestParallelCompositeReader.java | 8 +- .../index/TestPayloadProcessorProvider.java | 271 ------------------ .../org/apache/lucene/index/TestPayloads.java | 2 +- .../lucene/index/TestPostingsOffsets.java | 4 +- .../lucene/index/TestPrefixCodedTerms.java | 17 +- .../lucene/index/TestSegmentMerger.java | 2 +- .../lucene/index/TestStressIndexing2.java | 6 +- .../lucene/index/TestTermVectorsReader.java | 6 +- .../lucene/index/TestThreadedForceMerge.java | 2 +- .../lucene/index/TestTypePromotion.java | 8 +- .../search/TestCachingWrapperFilter.java | 10 +- .../search/TestNumericRangeQuery32.java | 2 +- .../search/TestNumericRangeQuery64.java | 2 +- .../lucene/search/TestShardSearching.java | 8 +- .../lucene/search/spans/TestPayloadSpans.java | 12 +- .../apache/lucene/store/TestLockFactory.java | 4 +- .../TestFailIfDirectoryNotClosed.java | 35 +++ 58 files changed, 243 insertions(+), 693 deletions(-) rename lucene/core/src/java/org/apache/lucene/{util => index}/MergedIterator.java (95%) delete mode 100644 lucene/core/src/java/org/apache/lucene/index/PayloadProcessorProvider.java delete mode 100644 lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java diff --git a/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java index 4d46fad5274..72ac63bd6f9 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java @@ -123,18 +123,7 @@ public final class MappingMultiDocsAndPositionsEnum extends DocsAndPositionsEnum @Override public BytesRef getPayload() throws IOException { - BytesRef payload = current.getPayload(); - if (mergeState.currentPayloadProcessor[upto] != null && payload != null) { - // to not violate the D&P api, we must give the processor a private copy - // TODO: reuse a BytesRef if there is a PPP - payload = BytesRef.deepCopyOf(payload); - mergeState.currentPayloadProcessor[upto].processPayload(payload); - if (payload.length == 0) { - // don't let PayloadProcessors corrumpt the index - return null; - } - } - return payload; + return current.getPayload(); } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java index 9b4a8a056ad..5bc626af3a5 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java @@ -27,8 +27,6 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Fields; import org.apache.lucene.index.MergeState; -import org.apache.lucene.index.PayloadProcessorProvider.PayloadProcessor; -import org.apache.lucene.index.PayloadProcessorProvider.ReaderPayloadProcessor; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSetIterator; @@ -170,12 +168,7 @@ public abstract class TermVectorsWriter implements Closeable { final AtomicReader reader = mergeState.readers.get(i); final int maxDoc = reader.maxDoc(); final Bits liveDocs = reader.getLiveDocs(); - // set PayloadProcessor - if (mergeState.payloadProcessorProvider != null) { - mergeState.currentReaderPayloadProcessor = mergeState.readerPayloadProcessor[i]; - } else { - mergeState.currentReaderPayloadProcessor = null; - } + for (int docID = 0; docID < maxDoc; docID++) { if (liveDocs != null && !liveDocs.get(docID)) { // skip deleted docs @@ -215,9 +208,6 @@ public abstract class TermVectorsWriter implements Closeable { TermsEnum termsEnum = null; DocsAndPositionsEnum docsAndPositionsEnum = null; - final ReaderPayloadProcessor readerPayloadProcessor = mergeState.currentReaderPayloadProcessor; - PayloadProcessor payloadProcessor = null; - for(String fieldName : vectors) { final FieldInfo fieldInfo = mergeState.fieldInfos.fieldInfo(fieldName); @@ -250,10 +240,6 @@ public abstract class TermVectorsWriter implements Closeable { final int freq = (int) termsEnum.totalTermFreq(); startTerm(termsEnum.term(), freq); - - if (hasPayloads && readerPayloadProcessor != null) { - payloadProcessor = readerPayloadProcessor.getProcessor(fieldName, termsEnum.term()); - } if (hasPositions || hasOffsets) { docsAndPositionsEnum = termsEnum.docsAndPositions(null, docsAndPositionsEnum); @@ -268,17 +254,7 @@ public abstract class TermVectorsWriter implements Closeable { final int startOffset = docsAndPositionsEnum.startOffset(); final int endOffset = docsAndPositionsEnum.endOffset(); - BytesRef payload = docsAndPositionsEnum.getPayload(); - - if (payloadProcessor != null && payload != null) { - // to not violate the D&P api, we must give the processor a private copy - payload = BytesRef.deepCopyOf(payload); - payloadProcessor.processPayload(payload); - if (payload.length == 0) { - // don't let PayloadProcessors corrumpt the index - payload = null; - } - } + final BytesRef payload = docsAndPositionsEnum.getPayload(); assert !hasPositions || pos >= 0; addPosition(pos, startOffset, endOffset, payload); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java index d3ac4bbdf29..26e0268df41 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java @@ -154,14 +154,7 @@ public abstract class TermsConsumer { postingsEnumIn = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(null, postingsEnumIn, DocsAndPositionsEnum.FLAG_PAYLOADS); assert postingsEnumIn != null; postingsEnum.reset(postingsEnumIn); - // set PayloadProcessor - if (mergeState.payloadProcessorProvider != null) { - for (int i = 0; i < mergeState.readers.size(); i++) { - if (mergeState.readerPayloadProcessor[i] != null) { - mergeState.currentPayloadProcessor[i] = mergeState.readerPayloadProcessor[i].getProcessor(mergeState.fieldInfo.name, term); - } - } - } + final PostingsConsumer postingsConsumer = startTerm(term); final TermStats stats = postingsConsumer.merge(mergeState, postingsEnum, visitedDocs); if (stats.docFreq > 0) { @@ -188,14 +181,7 @@ public abstract class TermsConsumer { postingsEnumIn = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(null, postingsEnumIn); assert postingsEnumIn != null; postingsEnum.reset(postingsEnumIn); - // set PayloadProcessor - if (mergeState.payloadProcessorProvider != null) { - for (int i = 0; i < mergeState.readers.size(); i++) { - if (mergeState.readerPayloadProcessor[i] != null) { - mergeState.currentPayloadProcessor[i] = mergeState.readerPayloadProcessor[i].getProcessor(mergeState.fieldInfo.name, term); - } - } - } + final PostingsConsumer postingsConsumer = startTerm(term); final TermStats stats = postingsConsumer.merge(mergeState, postingsEnum, visitedDocs); if (stats.docFreq > 0) { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java index 94a3dfd4642..e885bb87010 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java @@ -318,7 +318,7 @@ final class BlockPostingsWriter extends PostingsWriterBase { // current block if (lastBlockDocID != -1 && docBufferUpto == 1) { - // nocomit move to startDoc? ie we can write skip + // TODO: can we move this to startDoc? ie we can write skip // data as soon as the next doc starts... if (DEBUG) { System.out.println(" bufferSkip at writeBlock: lastDocID=" + lastBlockDocID + " docCount=" + (docCount-1)); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java index c14d7c1fe03..92fae5d3611 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java @@ -315,12 +315,7 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { int numDocs = 0; for (int i = 0; i < mergeState.readers.size(); i++) { final AtomicReader reader = mergeState.readers.get(i); - // set PayloadProcessor - if (mergeState.payloadProcessorProvider != null) { - mergeState.currentReaderPayloadProcessor = mergeState.readerPayloadProcessor[i]; - } else { - mergeState.currentReaderPayloadProcessor = null; - } + final SegmentReader matchingSegmentReader = mergeState.matchingSegmentReaders[idx++]; Lucene40TermVectorsReader matchingVectorsReader = null; if (matchingSegmentReader != null) { @@ -353,8 +348,8 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { final int maxDoc = reader.maxDoc(); final Bits liveDocs = reader.getLiveDocs(); int totalNumDocs = 0; - if (matchingVectorsReader != null && mergeState.currentReaderPayloadProcessor == null) { - // We can bulk-copy because the fieldInfos are "congruent" and there is no payload processor + if (matchingVectorsReader != null) { + // We can bulk-copy because the fieldInfos are "congruent" for (int docNum = 0; docNum < maxDoc;) { if (!liveDocs.get(docNum)) { // skip deleted docs @@ -404,8 +399,8 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { int rawDocLengths2[]) throws IOException { final int maxDoc = reader.maxDoc(); - if (matchingVectorsReader != null && mergeState.currentReaderPayloadProcessor == null) { - // We can bulk-copy because the fieldInfos are "congruent" and there is no payload processor + if (matchingVectorsReader != null) { + // We can bulk-copy because the fieldInfos are "congruent" int docCount = 0; while (docCount < maxDoc) { int len = Math.min(MAX_RAW_MERGE_DOCS, maxDoc - docCount); diff --git a/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java b/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java index 94ef1760df8..a2ef5efbb73 100644 --- a/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java @@ -54,7 +54,7 @@ public abstract class AtomicReader extends IndexReader { } @Override - public final AtomicReaderContext getTopReaderContext() { + public final AtomicReaderContext getContext() { ensureOpen(); return readerContext; } diff --git a/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java b/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java index 2d6f1d73570..decfc3368b1 100644 --- a/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java +++ b/lucene/core/src/java/org/apache/lucene/index/AtomicReaderContext.java @@ -21,8 +21,7 @@ import java.util.Collections; import java.util.List; /** - * {@link IndexReaderContext} for {@link AtomicReader} instances - * @lucene.experimental + * {@link IndexReaderContext} for {@link AtomicReader} instances. */ public final class AtomicReaderContext extends IndexReaderContext { /** The readers ord in the top-level's leaves array */ diff --git a/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java b/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java index b4bb4deaac3..5c0a85630c5 100644 --- a/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java +++ b/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java @@ -439,7 +439,7 @@ class BufferedDeletesStream { // Delete by query private static long applyQueryDeletes(Iterable queriesIter, ReadersAndLiveDocs rld, final SegmentReader reader) throws IOException { long delCount = 0; - final AtomicReaderContext readerContext = reader.getTopReaderContext(); + final AtomicReaderContext readerContext = reader.getContext(); boolean any = false; for (QueryAndLimit ent : queriesIter) { Query query = ent.query; diff --git a/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java b/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java index 14349c97dfa..c24372e4e8b 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java +++ b/lucene/core/src/java/org/apache/lucene/index/CoalescedDeletes.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Map; import org.apache.lucene.search.Query; -import org.apache.lucene.util.MergedIterator; import org.apache.lucene.index.BufferedDeletesStream.QueryAndLimit; class CoalescedDeletes { diff --git a/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java b/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java index 277adf61f35..53f4ee9dda3 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/CompositeReader.java @@ -78,16 +78,18 @@ public abstract class CompositeReader extends IndexReader { } /** Expert: returns the sequential sub readers that this - * reader is logically composed of. It contrast to previous - * Lucene versions may not return null. - * If this method returns an empty array, that means this - * reader is a null reader (for example a MultiReader - * that has no sub readers). + * reader is logically composed of. This method may not + * return {@code null}. + * + *

NOTE: In contrast to previous Lucene versions this method + * is no longer public, code that wants to get all {@link AtomicReader}s + * this composite is composed of should use {@link IndexReader#leaves()}. + * @see IndexReader#leaves() */ - public abstract List getSequentialSubReaders(); + protected abstract List getSequentialSubReaders(); @Override - public final CompositeReaderContext getTopReaderContext() { + public final CompositeReaderContext getContext() { ensureOpen(); // lazy init without thread safety for perf reasons: Building the readerContext twice does not hurt! if (readerContext == null) { diff --git a/lucene/core/src/java/org/apache/lucene/index/CompositeReaderContext.java b/lucene/core/src/java/org/apache/lucene/index/CompositeReaderContext.java index 2ea78abbf54..0c51872085a 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CompositeReaderContext.java +++ b/lucene/core/src/java/org/apache/lucene/index/CompositeReaderContext.java @@ -24,7 +24,6 @@ import java.util.List; /** * {@link IndexReaderContext} for {@link CompositeReader} instance. - * @lucene.experimental */ public final class CompositeReaderContext extends IndexReaderContext { private final List children; diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexReader.java b/lucene/core/src/java/org/apache/lucene/index/IndexReader.java index 59b6e911929..9a41845202f 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexReader.java @@ -21,6 +21,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.Collections; import java.util.LinkedHashSet; +import java.util.List; import java.util.WeakHashMap; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -377,9 +378,11 @@ public abstract class IndexReader implements Closeable { protected abstract void doClose() throws IOException; /** - * Expert: Returns a the root {@link IndexReaderContext} for this - * {@link IndexReader}'s sub-reader tree. Iff this reader is composed of sub - * readers ,ie. this reader being a composite reader, this method returns a + * Expert: Returns the root {@link IndexReaderContext} for this + * {@link IndexReader}'s sub-reader tree. + *

+ * Iff this reader is composed of sub + * readers, i.e. this reader being a composite reader, this method returns a * {@link CompositeReaderContext} holding the reader's direct children as well as a * view of the reader tree's atomic leaf contexts. All sub- * {@link IndexReaderContext} instances referenced from this readers top-level @@ -388,14 +391,21 @@ public abstract class IndexReader implements Closeable { * atomic leaf reader at a time. If this reader is not composed of child * readers, this method returns an {@link AtomicReaderContext}. *

- * Note: Any of the sub-{@link CompositeReaderContext} instances reference from this - * top-level context holds a null {@link CompositeReaderContext#leaves()} - * reference. Only the top-level context maintains the convenience leaf-view + * Note: Any of the sub-{@link CompositeReaderContext} instances referenced + * from this top-level context do not support {@link CompositeReaderContext#leaves()}. + * Only the top-level context maintains the convenience leaf-view * for performance reasons. - * - * @lucene.experimental */ - public abstract IndexReaderContext getTopReaderContext(); + public abstract IndexReaderContext getContext(); + + /** + * Returns the reader's leaves, or itself if this reader is atomic. + * This is a convenience method calling {@code this.getContext().leaves()}. + * @see IndexReaderContext#leaves() + */ + public final List leaves() { + return getContext().leaves(); + } /** Expert: Returns a key for this IndexReader, so FieldCache/CachingWrapperFilter can find * it again. diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java b/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java index 76e57aa86de..49e4a8e19f2 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexReaderContext.java @@ -22,7 +22,6 @@ import java.util.List; /** * A struct like class that represents a hierarchical relationship between * {@link IndexReader} instances. - * @lucene.experimental */ public abstract class IndexReaderContext { /** The reader context for this reader's immediate parent, or null if none */ diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index fcdb9ae954f..bbfa46c1aef 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -260,9 +260,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // to allow users to query an IndexWriter settings. private final LiveIndexWriterConfig config; - // The PayloadProcessorProvider to use when segments are merged - private PayloadProcessorProvider payloadProcessorProvider; - DirectoryReader getReader() throws IOException { return getReader(true); } @@ -763,8 +760,15 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } /** - * Commits all changes to an index and closes all - * associated files. Note that this may be a costly + * Commits all changes to an index, waits for pending merges + * to complete, and closes all associated files. + *

+ * This is a "slow graceful shutdown" which may take a long time + * especially if a big merge is pending: If you only want to close + * resources use {@link #rollback()}. If you only want to commit + * pending changes and close resources see {@link #close(boolean)}. + *

+ * Note that this may be a costly * operation, so, try to re-use a single writer instead of * closing and opening a new one. See {@link #commit()} for * caveats about write caching done by some IO devices. @@ -1263,7 +1267,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { reader = (AtomicReader) readerIn; } else { // Composite reader: lookup sub-reader and re-base docID: - List leaves = readerIn.getTopReaderContext().leaves(); + List leaves = readerIn.leaves(); int subIndex = ReaderUtil.subIndex(docID, leaves); reader = leaves.get(subIndex).reader(); docID -= leaves.get(subIndex).docBase; @@ -2399,8 +2403,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { false, codec, null, null); SegmentMerger merger = new SegmentMerger(info, infoStream, trackingDir, config.getTermIndexInterval(), - MergeState.CheckAbort.NONE, payloadProcessorProvider, - globalFieldNumberMap, context); + MergeState.CheckAbort.NONE, globalFieldNumberMap, context); for (IndexReader reader : readers) { // add new indexes merger.add(reader); @@ -3503,7 +3506,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { final TrackingDirectoryWrapper dirWrapper = new TrackingDirectoryWrapper(directory); SegmentMerger merger = new SegmentMerger(merge.info.info, infoStream, dirWrapper, config.getTermIndexInterval(), checkAbort, - payloadProcessorProvider, globalFieldNumberMap, context); + globalFieldNumberMap, context); if (infoStream.isEnabled("IW")) { infoStream.message("IW", "merging " + segString(merge.segments)); @@ -4058,38 +4061,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { synchronized void deletePendingFiles() throws IOException { deleter.deletePendingFiles(); } - - /** - * Sets the {@link PayloadProcessorProvider} to use when merging payloads. - * Note that the given pcp will be invoked for every segment that - * is merged, not only external ones that are given through - * {@link #addIndexes}. If you want only the payloads of the external segments - * to be processed, you can return null whenever a - * {@link PayloadProcessorProvider.ReaderPayloadProcessor} is requested for the {@link Directory} of the - * {@link IndexWriter}. - *

- * The default is null which means payloads are processed - * normally (copied) during segment merges. You can also unset it by passing - * null. - *

- * NOTE: the set {@link PayloadProcessorProvider} will be in effect - * immediately, potentially for already running merges too. If you want to be - * sure it is used for further operations only, such as {@link #addIndexes} or - * {@link #forceMerge}, you can call {@link #waitForMerges()} before. - */ - public void setPayloadProcessorProvider(PayloadProcessorProvider pcp) { - ensureOpen(); - payloadProcessorProvider = pcp; - } - - /** - * Returns the {@link PayloadProcessorProvider} that is used during segment - * merges to process payloads. - */ - public PayloadProcessorProvider getPayloadProcessorProvider() { - ensureOpen(); - return payloadProcessorProvider; - } /** * NOTE: this method creates a compound file for all files returned by diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeState.java b/lucene/core/src/java/org/apache/lucene/index/MergeState.java index c21de53b909..fbfe063a5a7 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MergeState.java +++ b/lucene/core/src/java/org/apache/lucene/index/MergeState.java @@ -19,8 +19,6 @@ package org.apache.lucene.index; import java.util.List; -import org.apache.lucene.index.PayloadProcessorProvider.PayloadProcessor; -import org.apache.lucene.index.PayloadProcessorProvider.ReaderPayloadProcessor; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.InfoStream; @@ -194,14 +192,6 @@ public class MergeState { // Updated per field; public FieldInfo fieldInfo; - // Used to process payloads - // TODO: this is a FactoryFactory here basically - // and we could make a codec(wrapper) to do all of this privately so IW is uninvolved - public PayloadProcessorProvider payloadProcessorProvider; - public ReaderPayloadProcessor[] readerPayloadProcessor; - public ReaderPayloadProcessor currentReaderPayloadProcessor; - public PayloadProcessor[] currentPayloadProcessor; - // TODO: get rid of this? it tells you which segments are 'aligned' (e.g. for bulk merging) // but is this really so expensive to compute again in different components, versus once in SM? public SegmentReader[] matchingSegmentReaders; diff --git a/lucene/core/src/java/org/apache/lucene/util/MergedIterator.java b/lucene/core/src/java/org/apache/lucene/index/MergedIterator.java similarity index 95% rename from lucene/core/src/java/org/apache/lucene/util/MergedIterator.java rename to lucene/core/src/java/org/apache/lucene/index/MergedIterator.java index c77abb90af1..9150930780f 100644 --- a/lucene/core/src/java/org/apache/lucene/util/MergedIterator.java +++ b/lucene/core/src/java/org/apache/lucene/index/MergedIterator.java @@ -1,4 +1,4 @@ -package org.apache.lucene.util; +package org.apache.lucene.index; /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -20,6 +20,8 @@ package org.apache.lucene.util; import java.util.Iterator; import java.util.NoSuchElementException; +import org.apache.lucene.util.PriorityQueue; + /** * Provides a merged sorted view from several sorted iterators, each * iterating over a unique set of elements. @@ -39,13 +41,13 @@ import java.util.NoSuchElementException; * * @lucene.internal */ -public class MergedIterator> implements Iterator { +final class MergedIterator> implements Iterator { private T current; private final TermMergeQueue queue; private final SubIterator[] top; private int numTop; - @SuppressWarnings("unchecked") + @SuppressWarnings({"unchecked","rawtypes"}) public MergedIterator(Iterator... iterators) { queue = new TermMergeQueue(iterators.length); top = new SubIterator[iterators.length]; diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java index 746885b95db..abfc979c0dd 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java @@ -36,7 +36,7 @@ import org.apache.lucene.util.packed.PackedInts.Reader; * *

NOTE: for multi readers, you'll get better * performance by gathering the sub readers using - * {@link IndexReader#getTopReaderContext()} to get the + * {@link IndexReader#getContext()} to get the * atomic leaves and then operate per-AtomicReader, * instead of using this class. * @@ -128,7 +128,7 @@ public class MultiDocValues extends DocValues { return puller.pull((AtomicReader) reader, field); } assert reader instanceof CompositeReader; - final List leaves = reader.getTopReaderContext().leaves(); + final List leaves = reader.leaves(); switch (leaves.size()) { case 0: // no fields diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java index 991decaaea5..0e44b58623b 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java @@ -28,7 +28,6 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.MergedIterator; /** * Exposes flex API, merged from flex API of sub-segments. @@ -39,7 +38,7 @@ import org.apache.lucene.util.MergedIterator; * *

NOTE: for composite readers, you'll get better * performance by gathering the sub readers using - * {@link IndexReader#getTopReaderContext()} to get the + * {@link IndexReader#getContext()} to get the * atomic leaves and then operate per-AtomicReader, * instead of using this class. * @@ -60,7 +59,7 @@ public final class MultiFields extends Fields { * It's better to get the sub-readers and iterate through them * yourself. */ public static Fields getFields(IndexReader reader) throws IOException { - final List leaves = reader.getTopReaderContext().leaves(); + final List leaves = reader.leaves(); switch (leaves.size()) { case 0: // no fields @@ -92,7 +91,7 @@ public final class MultiFields extends Fields { public static Bits getLiveDocs(IndexReader reader) { if (reader.hasDeletions()) { - final List leaves = reader.getTopReaderContext().leaves(); + final List leaves = reader.leaves(); final int size = leaves.size(); assert size > 0 : "A reader with deletions must have at least one leave"; if (size == 1) { @@ -182,7 +181,7 @@ public final class MultiFields extends Fields { this.subSlices = subSlices; } - @SuppressWarnings("unchecked") + @SuppressWarnings({"unchecked","rawtypes"}) @Override public Iterator iterator() { Iterator subIterators[] = new Iterator[subs.length]; @@ -251,7 +250,7 @@ public final class MultiFields extends Fields { */ public static FieldInfos getMergedFieldInfos(IndexReader reader) { final FieldInfos.Builder builder = new FieldInfos.Builder(); - for(final AtomicReaderContext ctx : reader.getTopReaderContext().leaves()) { + for(final AtomicReaderContext ctx : reader.leaves()) { builder.add(ctx.reader().getFieldInfos()); } return builder.finish(); diff --git a/lucene/core/src/java/org/apache/lucene/index/PayloadProcessorProvider.java b/lucene/core/src/java/org/apache/lucene/index/PayloadProcessorProvider.java deleted file mode 100644 index e59af58467c..00000000000 --- a/lucene/core/src/java/org/apache/lucene/index/PayloadProcessorProvider.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.apache.lucene.index; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; - -/** - * Provides a {@link ReaderPayloadProcessor} to be used for a {@link Directory}. - * This allows using different {@link ReaderPayloadProcessor}s for different - * source {@link AtomicReader}, for e.g. to perform different processing of payloads of - * different directories. - *

- * NOTE: to avoid processing payloads of certain directories, you can - * return null in {@link #getReaderProcessor}. - *

- * NOTE: it is possible that the same {@link ReaderPayloadProcessor} will be - * requested for the same {@link Directory} concurrently. Therefore, to avoid - * concurrency issues you should return different instances for different - * threads. Usually, if your {@link ReaderPayloadProcessor} does not maintain state - * this is not a problem. The merge code ensures that the - * {@link ReaderPayloadProcessor} instance you return will be accessed by one - * thread to obtain the {@link PayloadProcessor}s for different terms. - * - * @lucene.experimental - */ -public abstract class PayloadProcessorProvider { - - /** - * Returns a {@link PayloadProcessor} for a given {@link Term} which allows - * processing the payloads of different terms differently. If you intent to - * process all your payloads the same way, then you can ignore the given term. - *

- * NOTE: if you protect your {@link ReaderPayloadProcessor} from - * concurrency issues, then you shouldn't worry about any such issues when - * {@link PayloadProcessor}s are requested for different terms. - */ - public static abstract class ReaderPayloadProcessor { - - /** Returns a {@link PayloadProcessor} for the given term. */ - public abstract PayloadProcessor getProcessor(String field, BytesRef text) throws IOException; - - } - - /** - * Processes the given payload. - * - * @lucene.experimental - */ - public static abstract class PayloadProcessor { - - /** Process the incoming payload and stores the result in the given {@link BytesRef}. */ - public abstract void processPayload(BytesRef payload) throws IOException; - - } - - /** - * Returns a {@link ReaderPayloadProcessor} for the given {@link Directory}, - * through which {@link PayloadProcessor}s can be obtained for each - * {@link Term}, or null if none should be used. - */ - public abstract ReaderPayloadProcessor getReaderProcessor(AtomicReader reader) throws IOException; - -} diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java index ecb6b93da96..716839ae514 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java @@ -56,13 +56,11 @@ final class SegmentMerger { // note, just like in codec apis Directory 'dir' is NOT the same as segmentInfo.dir!! SegmentMerger(SegmentInfo segmentInfo, InfoStream infoStream, Directory dir, int termIndexInterval, - MergeState.CheckAbort checkAbort, PayloadProcessorProvider payloadProcessorProvider, - FieldInfos.FieldNumbers fieldNumbers, IOContext context) { + MergeState.CheckAbort checkAbort, FieldInfos.FieldNumbers fieldNumbers, IOContext context) { mergeState.segmentInfo = segmentInfo; mergeState.infoStream = infoStream; mergeState.readers = new ArrayList(); mergeState.checkAbort = checkAbort; - mergeState.payloadProcessorProvider = payloadProcessorProvider; directory = dir; this.termIndexInterval = termIndexInterval; this.codec = segmentInfo.getCodec(); @@ -75,7 +73,7 @@ final class SegmentMerger { * @param reader */ final void add(IndexReader reader) { - for (final AtomicReaderContext ctx : reader.getTopReaderContext().leaves()) { + for (final AtomicReaderContext ctx : reader.leaves()) { final AtomicReader r = ctx.reader(); mergeState.readers.add(r); } @@ -274,8 +272,6 @@ final class SegmentMerger { // Remap docIDs mergeState.docMaps = new MergeState.DocMap[numReaders]; mergeState.docBase = new int[numReaders]; - mergeState.readerPayloadProcessor = new PayloadProcessorProvider.ReaderPayloadProcessor[numReaders]; - mergeState.currentPayloadProcessor = new PayloadProcessorProvider.PayloadProcessor[numReaders]; int docBase = 0; @@ -289,10 +285,6 @@ final class SegmentMerger { mergeState.docMaps[i] = docMap; docBase += docMap.numDocs(); - if (mergeState.payloadProcessorProvider != null) { - mergeState.readerPayloadProcessor[i] = mergeState.payloadProcessorProvider.getReaderProcessor(reader); - } - i++; } diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java index a616661f34d..b7883afc9c9 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java @@ -37,7 +37,7 @@ import org.apache.lucene.index.MultiReader; // javadoc *

NOTE: this class almost always results in a * performance hit. If this is important to your use case, * you'll get better performance by gathering the sub readers using - * {@link IndexReader#getTopReaderContext()} to get the + * {@link IndexReader#getContext()} to get the * atomic leaves and then operate per-AtomicReader, * instead of using this class. */ diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java index 9ca8548ee5b..389bf3f9abf 100644 --- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java +++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java @@ -122,7 +122,7 @@ public class IndexSearcher { * * @lucene.experimental */ public IndexSearcher(IndexReader r, ExecutorService executor) { - this(r.getTopReaderContext(), executor); + this(r.getContext(), executor); } /** @@ -138,7 +138,7 @@ public class IndexSearcher { * href="https://issues.apache.org/jira/browse/LUCENE-2239">LUCENE-2239). * * @see IndexReaderContext - * @see IndexReader#getTopReaderContext() + * @see IndexReader#getContext() * @lucene.experimental */ public IndexSearcher(IndexReaderContext context, ExecutorService executor) { @@ -154,7 +154,7 @@ public class IndexSearcher { * Creates a searcher searching the provided top-level {@link IndexReaderContext}. * * @see IndexReaderContext - * @see IndexReader#getTopReaderContext() + * @see IndexReader#getContext() * @lucene.experimental */ public IndexSearcher(IndexReaderContext context) { @@ -639,7 +639,7 @@ public class IndexSearcher { /** * Returns this searchers the top-level {@link IndexReaderContext}. - * @see IndexReader#getTopReaderContext() + * @see IndexReader#getContext() */ /* sugar for #getReader().getTopReaderContext() */ public IndexReaderContext getTopReaderContext() { diff --git a/lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java b/lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java index 65ff2545cfb..441733e33cc 100644 --- a/lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java +++ b/lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java @@ -50,7 +50,7 @@ public class QueryWrapperFilter extends Filter { @Override public DocIdSet getDocIdSet(final AtomicReaderContext context, final Bits acceptDocs) throws IOException { // get a private context that is used to rewrite, createWeight and score eventually - final AtomicReaderContext privateContext = context.reader().getTopReaderContext(); + final AtomicReaderContext privateContext = context.reader().getContext(); final Weight weight = new IndexSearcher(privateContext).createNormalizedWeight(query); return new DocIdSet() { @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java b/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java index a237d07147f..a0534111e9e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermCollectingRewrite.java @@ -46,7 +46,7 @@ abstract class TermCollectingRewrite extends MultiTermQuery.Rew final void collectTerms(IndexReader reader, MultiTermQuery query, TermCollector collector) throws IOException { - IndexReaderContext topReaderContext = reader.getTopReaderContext(); + IndexReaderContext topReaderContext = reader.getContext(); Comparator lastTermComp = null; for (AtomicReaderContext context : topReaderContext.leaves()) { final Fields fields = context.reader().fields(); diff --git a/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java b/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java index 6bae19b7347..b8ab5d31d07 100644 --- a/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java +++ b/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java @@ -60,7 +60,7 @@ public class PayloadSpanUtil { * @param context * that contains doc with payloads to extract * - * @see IndexReader#getTopReaderContext() + * @see IndexReader#getContext() */ public PayloadSpanUtil(IndexReaderContext context) { this.context = context; diff --git a/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java b/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java index 6946329b405..5e27861b285 100644 --- a/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java +++ b/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java @@ -25,8 +25,10 @@ import java.util.Set; import org.apache.lucene.index.CompositeReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.FieldCache.CacheEntry; +import org.apache.lucene.store.AlreadyClosedException; /** * Provides methods for sanity checking that entries in the FieldCache @@ -272,20 +274,28 @@ public final class FieldCacheSanityChecker { /** * Checks if the seed is an IndexReader, and if so will walk * the hierarchy of subReaders building up a list of the objects - * returned by obj.getFieldCacheKey() + * returned by {@code seed.getCoreCacheKey()} */ private List getAllDescendantReaderKeys(Object seed) { List all = new ArrayList(17); // will grow as we iter all.add(seed); for (int i = 0; i < all.size(); i++) { - Object obj = all.get(i); - if (obj instanceof CompositeReader) { - List subs = ((CompositeReader)obj).getSequentialSubReaders(); - for (int j = 0; (null != subs) && (j < subs.size()); j++) { - all.add(subs.get(j).getCoreCacheKey()); + final Object obj = all.get(i); + // TODO: We don't check closed readers here (as getTopReaderContext + // throws AlreadyClosedException), what should we do? Reflection? + if (obj instanceof IndexReader) { + try { + final List childs = + ((IndexReader) obj).getContext().children(); + if (childs != null) { // it is composite reader + for (final IndexReaderContext ctx : childs) { + all.add(ctx.reader().getCoreCacheKey()); + } + } + } catch (AlreadyClosedException ace) { + // ignore this reader } } - } // need to skip the first, because it was the seed return all.subList(1, all.size()); diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestReuseDocsEnum.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestReuseDocsEnum.java index 611006511dd..c799828c0ee 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestReuseDocsEnum.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestReuseDocsEnum.java @@ -23,9 +23,9 @@ import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -50,7 +50,8 @@ public class TestReuseDocsEnum extends LuceneTestCase { writer.commit(); DirectoryReader open = DirectoryReader.open(dir); - for (AtomicReader indexReader : open.getSequentialSubReaders()) { + for (AtomicReaderContext ctx : open.leaves()) { + AtomicReader indexReader = ctx.reader(); Terms terms = indexReader.terms("body"); TermsEnum iterator = terms.iterator(null); IdentityHashMap enums = new IdentityHashMap(); @@ -76,8 +77,8 @@ public class TestReuseDocsEnum extends LuceneTestCase { writer.commit(); DirectoryReader open = DirectoryReader.open(dir); - for (AtomicReader indexReader : open.getSequentialSubReaders()) { - Terms terms = indexReader.terms("body"); + for (AtomicReaderContext ctx : open.leaves()) { + Terms terms = ctx.reader().terms("body"); TermsEnum iterator = terms.iterator(null); IdentityHashMap enums = new IdentityHashMap(); MatchNoBits bits = new Bits.MatchNoBits(open.maxDoc()); @@ -121,11 +122,11 @@ public class TestReuseDocsEnum extends LuceneTestCase { DirectoryReader firstReader = DirectoryReader.open(dir); DirectoryReader secondReader = DirectoryReader.open(dir); - List sequentialSubReaders = firstReader.getSequentialSubReaders(); - List sequentialSubReaders2 = secondReader.getSequentialSubReaders(); + List leaves = firstReader.leaves(); + List leaves2 = secondReader.leaves(); - for (IndexReader indexReader : sequentialSubReaders) { - Terms terms = ((AtomicReader) indexReader).terms("body"); + for (AtomicReaderContext ctx : leaves) { + Terms terms = ctx.reader().terms("body"); TermsEnum iterator = terms.iterator(null); IdentityHashMap enums = new IdentityHashMap(); MatchNoBits bits = new Bits.MatchNoBits(firstReader.maxDoc()); @@ -133,7 +134,7 @@ public class TestReuseDocsEnum extends LuceneTestCase { DocsEnum docs = null; BytesRef term = null; while ((term = iterator.next()) != null) { - docs = iterator.docs(null, randomDocsEnum("body", term, sequentialSubReaders2, bits), random().nextBoolean() ? DocsEnum.FLAG_FREQS : 0); + docs = iterator.docs(null, randomDocsEnum("body", term, leaves2, bits), random().nextBoolean() ? DocsEnum.FLAG_FREQS : 0); enums.put(docs, true); } assertEquals(terms.size(), enums.size()); @@ -142,7 +143,7 @@ public class TestReuseDocsEnum extends LuceneTestCase { enums.clear(); docs = null; while ((term = iterator.next()) != null) { - docs = iterator.docs(bits, randomDocsEnum("body", term, sequentialSubReaders2, bits), random().nextBoolean() ? DocsEnum.FLAG_FREQS : 0); + docs = iterator.docs(bits, randomDocsEnum("body", term, leaves2, bits), random().nextBoolean() ? DocsEnum.FLAG_FREQS : 0); enums.put(docs, true); } assertEquals(terms.size(), enums.size()); @@ -150,11 +151,11 @@ public class TestReuseDocsEnum extends LuceneTestCase { IOUtils.close(writer, firstReader, secondReader, dir); } - public DocsEnum randomDocsEnum(String field, BytesRef term, List readers, Bits bits) throws IOException { + public DocsEnum randomDocsEnum(String field, BytesRef term, List readers, Bits bits) throws IOException { if (random().nextInt(10) == 0) { return null; } - AtomicReader indexReader = (AtomicReader) readers.get(random().nextInt(readers.size())); + AtomicReader indexReader = readers.get(random().nextInt(readers.size())).reader(); return indexReader.termDocsEnum(bits, field, term, random().nextBoolean() ? DocsEnum.FLAG_FREQS : 0); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java index 3eaaa1389aa..d5036157058 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java @@ -143,7 +143,7 @@ public class TestCustomNorms extends LuceneTestCase { IndexReader reader = writer.getReader(); writer.close(); assertEquals(numAdded, reader.numDocs()); - IndexReaderContext topReaderContext = reader.getTopReaderContext(); + IndexReaderContext topReaderContext = reader.getContext(); for (final AtomicReaderContext ctx : topReaderContext.leaves()) { AtomicReader atomicReader = ctx.reader(); Source source = random().nextBoolean() ? atomicReader.normValues("foo").getSource() : atomicReader.normValues("foo").getDirectSource(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java index 7d9bc3c8262..add2504dbf0 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java @@ -68,7 +68,7 @@ public class TestDeletionPolicy extends LuceneTestCase { public void onCommit(List commits) throws IOException { IndexCommit lastCommit = commits.get(commits.size()-1); DirectoryReader r = DirectoryReader.open(dir); - assertEquals("lastCommit.segmentCount()=" + lastCommit.getSegmentCount() + " vs IndexReader.segmentCount=" + r.getSequentialSubReaders().size(), r.getSequentialSubReaders().size(), lastCommit.getSegmentCount()); + assertEquals("lastCommit.segmentCount()=" + lastCommit.getSegmentCount() + " vs IndexReader.segmentCount=" + r.leaves().size(), r.leaves().size(), lastCommit.getSegmentCount()); r.close(); verifyCommitOrder(commits); numOnCommit++; @@ -318,7 +318,7 @@ public class TestDeletionPolicy extends LuceneTestCase { final boolean needsMerging; { DirectoryReader r = DirectoryReader.open(dir); - needsMerging = r.getSequentialSubReaders().size() != 1; + needsMerging = r.leaves().size() != 1; r.close(); } if (needsMerging) { @@ -435,7 +435,7 @@ public class TestDeletionPolicy extends LuceneTestCase { DirectoryReader r = DirectoryReader.open(dir); // Still merged, still 11 docs - assertEquals(1, r.getSequentialSubReaders().size()); + assertEquals(1, r.leaves().size()); assertEquals(11, r.numDocs()); r.close(); @@ -451,7 +451,7 @@ public class TestDeletionPolicy extends LuceneTestCase { r = DirectoryReader.open(dir); // Not fully merged because we rolled it back, and now only // 10 docs - assertTrue(r.getSequentialSubReaders().size() > 1); + assertTrue(r.leaves().size() > 1); assertEquals(10, r.numDocs()); r.close(); @@ -461,7 +461,7 @@ public class TestDeletionPolicy extends LuceneTestCase { writer.close(); r = DirectoryReader.open(dir); - assertEquals(1, r.getSequentialSubReaders().size()); + assertEquals(1, r.leaves().size()); assertEquals(10, r.numDocs()); r.close(); @@ -473,7 +473,7 @@ public class TestDeletionPolicy extends LuceneTestCase { // Reader still sees fully merged index, because writer // opened on the prior commit has not yet committed: r = DirectoryReader.open(dir); - assertEquals(1, r.getSequentialSubReaders().size()); + assertEquals(1, r.leaves().size()); assertEquals(10, r.numDocs()); r.close(); @@ -481,7 +481,7 @@ public class TestDeletionPolicy extends LuceneTestCase { // Now reader sees not-fully-merged index: r = DirectoryReader.open(dir); - assertTrue(r.getSequentialSubReaders().size() > 1); + assertTrue(r.leaves().size() > 1); assertEquals(10, r.numDocs()); r.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java index d7d0f24d76f..bb4247fbe30 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java @@ -549,7 +549,7 @@ public void testFilesOpenClose() throws IOException { assertEquals("IndexReaders have different values for numDocs.", index1.numDocs(), index2.numDocs()); assertEquals("IndexReaders have different values for maxDoc.", index1.maxDoc(), index2.maxDoc()); assertEquals("Only one IndexReader has deletions.", index1.hasDeletions(), index2.hasDeletions()); - assertEquals("Single segment test differs.", index1.getSequentialSubReaders().size() == 1, index2.getSequentialSubReaders().size() == 1); + assertEquals("Single segment test differs.", index1.leaves().size() == 1, index2.leaves().size() == 1); // check field names FieldInfos fieldInfos1 = MultiFields.getMergedFieldInfos(index1); @@ -785,7 +785,7 @@ public void testFilesOpenClose() throws IOException { DirectoryReader r2 = DirectoryReader.openIfChanged(r); assertNotNull(r2); r.close(); - AtomicReader sub0 = r2.getSequentialSubReaders().get(0); + AtomicReader sub0 = r2.leaves().get(0).reader(); final int[] ints2 = FieldCache.DEFAULT.getInts(sub0, "number", false); r2.close(); assertTrue(ints == ints2); @@ -814,9 +814,8 @@ public void testFilesOpenClose() throws IOException { assertNotNull(r2); r.close(); - List subs = r2.getSequentialSubReaders(); - for(AtomicReader s : subs) { - assertEquals(36, s.getUniqueTermCount()); + for(AtomicReaderContext s : r2.leaves()) { + assertEquals(36, s.reader().getUniqueTermCount()); } r2.close(); writer.close(); @@ -842,7 +841,7 @@ public void testFilesOpenClose() throws IOException { // expected } - assertEquals(-1, ((SegmentReader) r.getSequentialSubReaders().get(0)).getTermInfosIndexDivisor()); + assertEquals(-1, ((SegmentReader) r.leaves().get(0).reader()).getTermInfosIndexDivisor()); writer = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())). @@ -857,11 +856,11 @@ public void testFilesOpenClose() throws IOException { assertNotNull(r2); assertNull(DirectoryReader.openIfChanged(r2)); r.close(); - List subReaders = r2.getSequentialSubReaders(); - assertEquals(2, subReaders.size()); - for(AtomicReader s : subReaders) { + List leaves = r2.leaves(); + assertEquals(2, leaves.size()); + for(AtomicReaderContext ctx : leaves) { try { - s.docFreq(new Term("field", "f")); + ctx.reader().docFreq(new Term("field", "f")); fail("did not hit expected exception"); } catch (IllegalStateException ise) { // expected diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java index 8e6f3ea588e..c2308666a2e 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java @@ -171,8 +171,8 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { TestDirectoryReader.assertIndexEquals(index1, index2_refreshed); index2_refreshed.close(); - assertReaderClosed(index2, true, true); - assertReaderClosed(index2_refreshed, true, true); + assertReaderClosed(index2, true); + assertReaderClosed(index2_refreshed, true); index2 = test.openReader(); @@ -190,28 +190,8 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { index1.close(); index2.close(); - assertReaderClosed(index1, true, true); - assertReaderClosed(index2, true, true); - } - - private void performTestsWithExceptionInReopen(TestReopen test) throws Exception { - DirectoryReader index1 = test.openReader(); - DirectoryReader index2 = test.openReader(); - - TestDirectoryReader.assertIndexEquals(index1, index2); - - try { - refreshReader(index1, test, 0, true); - fail("Expected exception not thrown."); - } catch (Exception e) { - // expected exception - } - - // index2 should still be usable and unaffected by the failed reopen() call - TestDirectoryReader.assertIndexEquals(index1, index2); - - index1.close(); - index2.close(); + assertReaderClosed(index1, true); + assertReaderClosed(index2, true); } public void testThreadSafety() throws Exception { @@ -355,11 +335,11 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { reader.close(); for (final DirectoryReader readerToClose : readersToClose) { - assertReaderClosed(readerToClose, true, true); + assertReaderClosed(readerToClose, true); } - assertReaderClosed(reader, true, true); - assertReaderClosed(firstReader, true, true); + assertReaderClosed(reader, true); + assertReaderClosed(firstReader, true); dir.close(); } @@ -374,7 +354,7 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { DirectoryReader refreshedReader; } - private abstract static class ReaderThreadTask { + abstract static class ReaderThreadTask { protected volatile boolean stopped; public void stop() { this.stopped = true; @@ -384,8 +364,8 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { } private static class ReaderThread extends Thread { - private ReaderThreadTask task; - private Throwable error; + ReaderThreadTask task; + Throwable error; ReaderThread(ReaderThreadTask task) { @@ -469,9 +449,9 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { DirectoryReader r = DirectoryReader.open(dir); if (multiSegment) { - assertTrue(r.getSequentialSubReaders().size() > 1); + assertTrue(r.leaves().size() > 1); } else { - assertTrue(r.getSequentialSubReaders().size() == 1); + assertTrue(r.leaves().size() == 1); } r.close(); } @@ -533,46 +513,25 @@ public class TestDirectoryReaderReopen extends LuceneTestCase { } } - static void assertReaderClosed(IndexReader reader, boolean checkSubReaders, boolean checkNormsClosed) { + static void assertReaderClosed(IndexReader reader, boolean checkSubReaders) { assertEquals(0, reader.getRefCount()); - if (checkNormsClosed && reader instanceof AtomicReader) { - // TODO: should we really assert something here? we check for open files and this is obselete... - // assertTrue(((SegmentReader) reader).normsClosed()); - } - if (checkSubReaders && reader instanceof CompositeReader) { + // we cannot use reader context here, as reader is + // already closed and calling getTopReaderContext() throws AlreadyClosed! List subReaders = ((CompositeReader) reader).getSequentialSubReaders(); - for (IndexReader r : subReaders) { - assertReaderClosed(r, checkSubReaders, checkNormsClosed); + for (final IndexReader r : subReaders) { + assertReaderClosed(r, checkSubReaders); } } } - /* - private void assertReaderOpen(DirectoryReader reader) { - reader.ensureOpen(); - - if (reader instanceof DirectoryReader) { - DirectoryReader[] subReaders = reader.getSequentialSubReaders(); - for (int i = 0; i < subReaders.length; i++) { - assertReaderOpen(subReaders[i]); - } - } - } - */ - - private void assertRefCountEquals(int refCount, DirectoryReader reader) { - assertEquals("Reader has wrong refCount value.", refCount, reader.getRefCount()); - } - - - private abstract static class TestReopen { + abstract static class TestReopen { protected abstract DirectoryReader openReader() throws IOException; protected abstract void modifyIndex(int i) throws IOException; } - private static class KeepAllCommits implements IndexDeletionPolicy { + static class KeepAllCommits implements IndexDeletionPolicy { public void onInit(List commits) { } public void onCommit(List commits) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java index 1b826b36b22..772c0fbeead 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java @@ -204,7 +204,7 @@ public class TestDoc extends LuceneTestCase { final SegmentInfo si = new SegmentInfo(si1.info.dir, Constants.LUCENE_MAIN_VERSION, merged, -1, false, codec, null, null); SegmentMerger merger = new SegmentMerger(si, InfoStream.getDefault(), trackingDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, - MergeState.CheckAbort.NONE, null, new FieldInfos.FieldNumbers(), context); + MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), context); merger.add(r1); merger.add(r2); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java b/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java index 827fbe29f8c..9ca5250c730 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocTermOrds.java @@ -154,11 +154,11 @@ public class TestDocTermOrds extends LuceneTestCase { System.out.println("TEST: reader=" + r); } - for(IndexReader subR : r.getSequentialSubReaders()) { + for(AtomicReaderContext ctx : r.leaves()) { if (VERBOSE) { - System.out.println("\nTEST: sub=" + subR); + System.out.println("\nTEST: sub=" + ctx.reader()); } - verify((AtomicReader) subR, idToOrds, termsArray, null); + verify(ctx.reader(), idToOrds, termsArray, null); } // Also test top-level reader: its enum does not support @@ -273,11 +273,11 @@ public class TestDocTermOrds extends LuceneTestCase { idToOrdsPrefix[id] = newOrdsArray; } - for(IndexReader subR : r.getSequentialSubReaders()) { + for(AtomicReaderContext ctx : r.leaves()) { if (VERBOSE) { - System.out.println("\nTEST: sub=" + subR); + System.out.println("\nTEST: sub=" + ctx.reader()); } - verify((AtomicReader) subR, idToOrdsPrefix, termsArray, prefixRef); + verify(ctx.reader(), idToOrdsPrefix, termsArray, prefixRef); } // Also test top-level reader: its enum does not support diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java index 0ce1e2dd531..6596eee788b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java @@ -91,7 +91,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { writer.close(true); DirectoryReader reader = DirectoryReader.open(dir, 1); - assertEquals(1, reader.getSequentialSubReaders().size()); + assertEquals(1, reader.leaves().size()); IndexSearcher searcher = new IndexSearcher(reader); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java b/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java index ddc5e293848..0f92f0bf020 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java @@ -63,7 +63,7 @@ public class TestDocsAndPositions extends LuceneTestCase { int num = atLeast(13); for (int i = 0; i < num; i++) { BytesRef bytes = new BytesRef("1"); - IndexReaderContext topReaderContext = reader.getTopReaderContext(); + IndexReaderContext topReaderContext = reader.getContext(); for (AtomicReaderContext atomicReaderContext : topReaderContext.leaves()) { DocsAndPositionsEnum docsAndPosEnum = getDocsAndPositions( atomicReaderContext.reader(), bytes, null); @@ -138,7 +138,7 @@ public class TestDocsAndPositions extends LuceneTestCase { int num = atLeast(13); for (int i = 0; i < num; i++) { BytesRef bytes = new BytesRef("" + term); - IndexReaderContext topReaderContext = reader.getTopReaderContext(); + IndexReaderContext topReaderContext = reader.getContext(); for (AtomicReaderContext atomicReaderContext : topReaderContext.leaves()) { DocsAndPositionsEnum docsAndPosEnum = getDocsAndPositions( atomicReaderContext.reader(), bytes, null); @@ -214,7 +214,7 @@ public class TestDocsAndPositions extends LuceneTestCase { int num = atLeast(13); for (int i = 0; i < num; i++) { BytesRef bytes = new BytesRef("" + term); - IndexReaderContext topReaderContext = reader.getTopReaderContext(); + IndexReaderContext topReaderContext = reader.getContext(); for (AtomicReaderContext context : topReaderContext.leaves()) { int maxDoc = context.reader().maxDoc(); DocsEnum docsEnum = _TestUtil.docs(random(), context.reader(), fieldName, bytes, null, null, DocsEnum.FLAG_FREQS); @@ -292,7 +292,7 @@ public class TestDocsAndPositions extends LuceneTestCase { for (int i = 0; i < num; i++) { BytesRef bytes = new BytesRef("even"); - IndexReaderContext topReaderContext = reader.getTopReaderContext(); + IndexReaderContext topReaderContext = reader.getContext(); for (AtomicReaderContext atomicReaderContext : topReaderContext.leaves()) { DocsAndPositionsEnum docsAndPosEnum = getDocsAndPositions( atomicReaderContext.reader(), bytes, null); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java index 5bbc43ad547..4b803eb8763 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java @@ -289,8 +289,9 @@ public class TestFieldsReader extends LuceneTestCase { assertEquals(numDocs, r.numDocs()); - for(IndexReader sub : r.getSequentialSubReaders()) { - final int[] ids = FieldCache.DEFAULT.getInts((AtomicReader) sub, "id", false); + for(AtomicReaderContext ctx : r.leaves()) { + final AtomicReader sub = ctx.reader(); + final int[] ids = FieldCache.DEFAULT.getInts(sub, "id", false); for(int docID=0;docID 1); + assertTrue("Reader incorrectly sees one segment", reader.leaves().size() > 1); reader.close(); // Abort the writer: @@ -297,7 +297,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { reader = DirectoryReader.open(dir); // Reader should still see index as multi-segment - assertTrue("Reader incorrectly sees one segment", reader.getSequentialSubReaders().size() > 1); + assertTrue("Reader incorrectly sees one segment", reader.leaves().size() > 1); reader.close(); if (VERBOSE) { @@ -316,7 +316,7 @@ public class TestIndexWriterCommit extends LuceneTestCase { reader = DirectoryReader.open(dir); // Reader should see index as one segment - assertEquals("Reader incorrectly sees more than one segment", 1, reader.getSequentialSubReaders().size()); + assertEquals("Reader incorrectly sees more than one segment", 1, reader.leaves().size()); reader.close(); dir.close(); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java index 8e2836047ce..c57ff3d0a60 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java @@ -1293,7 +1293,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { assertTrue(reader.numDocs() > 0); SegmentInfos sis = new SegmentInfos(); sis.read(dir); - for(AtomicReaderContext context : reader.getTopReaderContext().leaves()) { + for(AtomicReaderContext context : reader.leaves()) { assertFalse(context.reader().getFieldInfos().hasVectors()); } reader.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java index 4f570bb2df4..e3042cb9cf8 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java @@ -187,7 +187,7 @@ public class TestIndexWriterForceMerge extends LuceneTestCase { if (0 == pass) { writer.close(); DirectoryReader reader = DirectoryReader.open(dir); - assertEquals(1, reader.getSequentialSubReaders().size()); + assertEquals(1, reader.leaves().size()); reader.close(); } else { // Get another segment to flush so we can verify it is @@ -197,7 +197,7 @@ public class TestIndexWriterForceMerge extends LuceneTestCase { writer.close(); DirectoryReader reader = DirectoryReader.open(dir); - assertTrue(reader.getSequentialSubReaders().size() > 1); + assertTrue(reader.leaves().size() > 1); reader.close(); SegmentInfos infos = new SegmentInfos(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java index 11a19e880c9..950b53ffc09 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java @@ -315,7 +315,7 @@ public class TestIndexWriterUnicode extends LuceneTestCase { IndexReader r = writer.getReader(); // Test each sub-segment - for (AtomicReaderContext ctx : r.getTopReaderContext().leaves()) { + for (AtomicReaderContext ctx : r.leaves()) { checkTermsOrder(ctx.reader(), allTerms, false); } checkTermsOrder(r, allTerms, true); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java index 68b417945c2..499b0cb12fe 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java @@ -339,13 +339,13 @@ public class TestParallelCompositeReader extends LuceneTestCase { if (compositeComposite) { rd1 = new MultiReader(DirectoryReader.open(dir1), DirectoryReader.open(dir1)); rd2 = new MultiReader(DirectoryReader.open(dir2), DirectoryReader.open(dir2)); - assertEquals(2, rd1.getSequentialSubReaders().size()); - assertEquals(2, rd2.getSequentialSubReaders().size()); + assertEquals(2, rd1.getContext().children().size()); + assertEquals(2, rd2.getContext().children().size()); } else { rd1 = DirectoryReader.open(dir1); rd2 = DirectoryReader.open(dir2); - assertEquals(3, rd1.getSequentialSubReaders().size()); - assertEquals(3, rd2.getSequentialSubReaders().size()); + assertEquals(3, rd1.getContext().children().size()); + assertEquals(3, rd2.getContext().children().size()); } ParallelCompositeReader pr = new ParallelCompositeReader(rd1, rd2); return newSearcher(pr); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java b/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java deleted file mode 100644 index c855154c2aa..00000000000 --- a/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java +++ /dev/null @@ -1,271 +0,0 @@ -package org.apache.lucene.index; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.PayloadProcessorProvider.ReaderPayloadProcessor; -import org.apache.lucene.index.PayloadProcessorProvider.PayloadProcessor; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LuceneTestCase; -import org.junit.Test; - -public class TestPayloadProcessorProvider extends LuceneTestCase { - - private static final class PerDirPayloadProcessor extends PayloadProcessorProvider { - - private final Map processors; - - public PerDirPayloadProcessor(Map processors) { - this.processors = processors; - } - - @Override - public ReaderPayloadProcessor getReaderProcessor(AtomicReader reader) { - if (reader instanceof SegmentReader) { - return processors.get(((SegmentReader) reader).directory()); - } else { - throw new UnsupportedOperationException("This shouldnot happen in this test: Reader is no SegmentReader"); - } - } - - } - - private static final class PerTermPayloadProcessor extends ReaderPayloadProcessor { - - @Override - public PayloadProcessor getProcessor(String field, BytesRef text) { - // don't process payloads of terms other than "p:p1" - if (!field.equals("p") || !text.bytesEquals(new BytesRef("p1"))) { - return null; - } - - // All other terms are processed the same way - return new DeletePayloadProcessor(); - } - - } - - /** deletes the incoming payload */ - private static final class DeletePayloadProcessor extends PayloadProcessor { - - @Override - public void processPayload(BytesRef payload) { - payload.length = 0; - } - - } - - private static final class PayloadTokenStream extends TokenStream { - - private final PayloadAttribute payload = addAttribute(PayloadAttribute.class); - private final CharTermAttribute term = addAttribute(CharTermAttribute.class); - - private boolean called = false; - private String t; - - public PayloadTokenStream(String t) { - this.t = t; - } - - @Override - public boolean incrementToken() { - if (called) { - return false; - } - - called = true; - byte[] p = new byte[] { 1 }; - payload.setPayload(new BytesRef(p)); - term.append(t); - return true; - } - - @Override - public void reset() throws IOException { - super.reset(); - called = false; - term.setEmpty(); - } - } - - private static final int NUM_DOCS = 10; - - private void populateDirs(Random random, Directory[] dirs, boolean multipleCommits) - throws IOException { - for (int i = 0; i < dirs.length; i++) { - dirs[i] = newDirectory(); - populateDocs(random, dirs[i], multipleCommits); - verifyPayloadExists(dirs[i], "p", new BytesRef("p1"), NUM_DOCS); - verifyPayloadExists(dirs[i], "p", new BytesRef("p2"), NUM_DOCS); - } - } - - private void populateDocs(Random random, Directory dir, boolean multipleCommits) - throws IOException { - IndexWriter writer = new IndexWriter( - dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)). - setMergePolicy(newLogMergePolicy(10)) - ); - TokenStream payloadTS1 = new PayloadTokenStream("p1"); - TokenStream payloadTS2 = new PayloadTokenStream("p2"); - FieldType customType = new FieldType(TextField.TYPE_NOT_STORED); - customType.setOmitNorms(true); - for (int i = 0; i < NUM_DOCS; i++) { - Document doc = new Document(); - doc.add(newField("id", "doc" + i, customType)); - doc.add(newTextField("content", "doc content " + i, Field.Store.NO)); - if (random.nextBoolean()) { - doc.add(new TextField("p", payloadTS1)); - doc.add(new TextField("p", payloadTS2)); - } else { - FieldType type = new FieldType(TextField.TYPE_NOT_STORED); - type.setStoreTermVectors(true); - type.setStoreTermVectorPositions(true); - type.setStoreTermVectorPayloads(true); - type.setStoreTermVectorOffsets(random.nextBoolean()); - doc.add(new Field("p", payloadTS1, type)); - doc.add(new Field("p", payloadTS2, type)); - } - writer.addDocument(doc); - if (multipleCommits && (i % 4 == 0)) { - writer.commit(); - } - } - writer.close(); - } - - private void verifyPayloadExists(Directory dir, String field, BytesRef text, int numExpected) - throws IOException { - IndexReader reader = DirectoryReader.open(dir); - try { - int numPayloads = 0; - DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text); - while (tpe.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { - tpe.nextPosition(); - BytesRef payload = tpe.getPayload(); - if (payload != null) { - assertEquals(1, payload.length); - assertEquals(1, payload.bytes[0]); - ++numPayloads; - } - } - assertEquals(numExpected, numPayloads); - } finally { - reader.close(); - } - } - - private void doTest(Random random, boolean addToEmptyIndex, - int numExpectedPayloads, boolean multipleCommits) throws IOException { - Directory[] dirs = new Directory[2]; - populateDirs(random, dirs, multipleCommits); - - Directory dir = newDirectory(); - if (!addToEmptyIndex) { - populateDocs(random, dir, multipleCommits); - verifyPayloadExists(dir, "p", new BytesRef("p1"), NUM_DOCS); - verifyPayloadExists(dir, "p", new BytesRef("p2"), NUM_DOCS); - } - - // Add two source dirs. By not adding the dest dir, we ensure its payloads - // won't get processed. - Map processors = new HashMap(); - for (Directory d : dirs) { - processors.put(d, new PerTermPayloadProcessor()); - } - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false))); - writer.setPayloadProcessorProvider(new PerDirPayloadProcessor(processors)); - - IndexReader[] readers = new IndexReader[dirs.length]; - for (int i = 0; i < readers.length; i++) { - readers[i] = DirectoryReader.open(dirs[i]); - } - try { - writer.addIndexes(readers); - } finally { - for (IndexReader r : readers) { - r.close(); - } - } - writer.close(); - verifyPayloadExists(dir, "p", new BytesRef("p1"), numExpectedPayloads); - // the second term should always have all payloads - numExpectedPayloads = NUM_DOCS * dirs.length - + (addToEmptyIndex ? 0 : NUM_DOCS); - verifyPayloadExists(dir, "p", new BytesRef("p2"), numExpectedPayloads); - for (Directory d : dirs) - d.close(); - dir.close(); - } - - @Test - public void testAddIndexes() throws Exception { - // addIndexes - single commit in each - doTest(random(), true, 0, false); - - // addIndexes - multiple commits in each - doTest(random(), true, 0, true); - } - - @Test - public void testAddIndexesIntoExisting() throws Exception { - // addIndexes - single commit in each - doTest(random(), false, NUM_DOCS, false); - - // addIndexes - multiple commits in each - doTest(random(), false, NUM_DOCS, true); - } - - @Test - public void testRegularMerges() throws Exception { - Directory dir = newDirectory(); - populateDocs(random(), dir, true); - verifyPayloadExists(dir, "p", new BytesRef("p1"), NUM_DOCS); - verifyPayloadExists(dir, "p", new BytesRef("p2"), NUM_DOCS); - - // Add two source dirs. By not adding the dest dir, we ensure its payloads - // won't get processed. - Map processors = new HashMap(); - processors.put(dir, new PerTermPayloadProcessor()); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))); - writer.setPayloadProcessorProvider(new PerDirPayloadProcessor(processors)); - writer.forceMerge(1); - writer.close(); - - verifyPayloadExists(dir, "p", new BytesRef("p1"), 0); - verifyPayloadExists(dir, "p", new BytesRef("p2"), NUM_DOCS); - dir.close(); - } - -} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java b/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java index 5a4cacd2df8..2fbff1b5fdc 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java @@ -604,7 +604,7 @@ public class TestPayloads extends LuceneTestCase { field.setTokenStream(ts); writer.addDocument(doc); DirectoryReader reader = writer.getReader(); - AtomicReader sr = reader.getSequentialSubReaders().get(0); + AtomicReader sr = SlowCompositeReaderWrapper.wrap(reader); DocsAndPositionsEnum de = sr.termPositionsEnum(null, "field", new BytesRef("withPayload")); de.nextDoc(); de.nextPosition(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java b/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java index 3e8652f6ae8..0c9d59b67e8 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java @@ -289,9 +289,9 @@ public class TestPostingsOffsets extends LuceneTestCase { w.close(); final String[] terms = new String[] {"a", "b", "c", "d"}; - for(IndexReader reader : r.getSequentialSubReaders()) { + for(AtomicReaderContext ctx : r.leaves()) { // TODO: improve this - AtomicReader sub = (AtomicReader) reader; + AtomicReader sub = ctx.reader(); //System.out.println("\nsub=" + sub); final TermsEnum termsEnum = sub.fields().terms("content").iterator(null); DocsEnum docs = null; diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java b/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java index d1222fd64ba..4dcbd8d4724 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPrefixCodedTerms.java @@ -25,7 +25,6 @@ import java.util.Set; import java.util.TreeSet; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.MergedIterator; import org.apache.lucene.util._TestUtil; public class TestPrefixCodedTerms extends LuceneTestCase { @@ -70,14 +69,10 @@ public class TestPrefixCodedTerms extends LuceneTestCase { @SuppressWarnings("unchecked") public void testMergeEmpty() { - List> subs = Collections.emptyList(); - Iterator merged = new MergedIterator(subs.toArray(new Iterator[0])); + Iterator merged = new MergedIterator(); assertFalse(merged.hasNext()); - subs = new ArrayList>(); - subs.add(new PrefixCodedTerms.Builder().finish().iterator()); - subs.add(new PrefixCodedTerms.Builder().finish().iterator()); - merged = new MergedIterator(subs.toArray(new Iterator[0])); + merged = new MergedIterator(new PrefixCodedTerms.Builder().finish().iterator(), new PrefixCodedTerms.Builder().finish().iterator()); assertFalse(merged.hasNext()); } @@ -93,18 +88,14 @@ public class TestPrefixCodedTerms extends LuceneTestCase { b2.add(t2); PrefixCodedTerms pb2 = b2.finish(); - List> subs = new ArrayList>(); - subs.add(pb1.iterator()); - subs.add(pb2.iterator()); - - Iterator merged = new MergedIterator(subs.toArray(new Iterator[0])); + Iterator merged = new MergedIterator(pb1.iterator(), pb2.iterator()); assertTrue(merged.hasNext()); assertEquals(t1, merged.next()); assertTrue(merged.hasNext()); assertEquals(t2, merged.next()); } - @SuppressWarnings("unchecked") + @SuppressWarnings({"unchecked","rawtypes"}) public void testMergeRandom() { PrefixCodedTerms pb[] = new PrefixCodedTerms[_TestUtil.nextInt(random(), 2, 10)]; Set superSet = new TreeSet(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java index 927dc7e2bb7..306c8aef0d4 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java @@ -82,7 +82,7 @@ public class TestSegmentMerger extends LuceneTestCase { final SegmentInfo si = new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, -1, false, codec, null, null); SegmentMerger merger = new SegmentMerger(si, InfoStream.getDefault(), mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, - MergeState.CheckAbort.NONE, null, new FieldInfos.FieldNumbers(), newIOContext(random())); + MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), newIOContext(random())); merger.add(reader1); merger.add(reader2); MergeState mergeState = merger.merge(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java index 376a5003569..ccecbede737 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java @@ -281,10 +281,10 @@ public class TestStressIndexing2 extends LuceneTestCase { } private static void printDocs(DirectoryReader r) throws Throwable { - List subs = r.getSequentialSubReaders(); - for(IndexReader sub : subs) { + for(AtomicReaderContext ctx : r.leaves()) { // TODO: improve this - Bits liveDocs = ((AtomicReader)sub).getLiveDocs(); + AtomicReader sub = ctx.reader(); + Bits liveDocs = sub.getLiveDocs(); System.out.println(" " + ((SegmentReader) sub).getSegmentInfo()); for(int docID=0;docID leaves = topReaderContext.leaves(); DocValues docValues = leaves.get(0).reader().docValues("promote"); assertNotNull(docValues); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java b/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java index 31a88b4c9d5..e73df8362d3 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java @@ -43,7 +43,7 @@ public class TestCachingWrapperFilter extends LuceneTestCase { writer.close(); IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir)); - AtomicReaderContext context = (AtomicReaderContext) reader.getTopReaderContext(); + AtomicReaderContext context = (AtomicReaderContext) reader.getContext(); MockFilter filter = new MockFilter(); CachingWrapperFilter cacher = new CachingWrapperFilter(filter); @@ -69,7 +69,7 @@ public class TestCachingWrapperFilter extends LuceneTestCase { writer.close(); IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir)); - AtomicReaderContext context = (AtomicReaderContext) reader.getTopReaderContext(); + AtomicReaderContext context = (AtomicReaderContext) reader.getContext(); final Filter filter = new Filter() { @Override @@ -92,7 +92,7 @@ public class TestCachingWrapperFilter extends LuceneTestCase { writer.close(); IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir)); - AtomicReaderContext context = (AtomicReaderContext) reader.getTopReaderContext(); + AtomicReaderContext context = (AtomicReaderContext) reader.getContext(); final Filter filter = new Filter() { @Override @@ -115,8 +115,8 @@ public class TestCachingWrapperFilter extends LuceneTestCase { } private static void assertDocIdSetCacheable(IndexReader reader, Filter filter, boolean shouldCacheable) throws IOException { - assertTrue(reader.getTopReaderContext() instanceof AtomicReaderContext); - AtomicReaderContext context = (AtomicReaderContext) reader.getTopReaderContext(); + assertTrue(reader.getContext() instanceof AtomicReaderContext); + AtomicReaderContext context = (AtomicReaderContext) reader.getContext(); final CachingWrapperFilter cacher = new CachingWrapperFilter(filter); final DocIdSet originalSet = filter.getDocIdSet(context, context.reader().getLiveDocs()); final DocIdSet cachedSet = cacher.getDocIdSet(context, context.reader().getLiveDocs()); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java b/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java index b5765fd9c52..cdb2503c7ea 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java @@ -197,7 +197,7 @@ public class TestNumericRangeQuery32 extends LuceneTestCase { @Test public void testInverseRange() throws Exception { - AtomicReaderContext context = SlowCompositeReaderWrapper.wrap(reader).getTopReaderContext(); + AtomicReaderContext context = SlowCompositeReaderWrapper.wrap(reader).getContext(); NumericRangeFilter f = NumericRangeFilter.newIntRange("field8", 8, 1000, -1000, true, true); assertSame("A inverse range should return the EMPTY_DOCIDSET instance", DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(context, context.reader().getLiveDocs())); f = NumericRangeFilter.newIntRange("field8", 8, Integer.MAX_VALUE, null, false, false); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java b/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java index fb307355269..48a825125d0 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java @@ -211,7 +211,7 @@ public class TestNumericRangeQuery64 extends LuceneTestCase { @Test public void testInverseRange() throws Exception { - AtomicReaderContext context = SlowCompositeReaderWrapper.wrap(searcher.getIndexReader()).getTopReaderContext(); + AtomicReaderContext context = SlowCompositeReaderWrapper.wrap(searcher.getIndexReader()).getContext(); NumericRangeFilter f = NumericRangeFilter.newLongRange("field8", 8, 1000L, -1000L, true, true); assertSame("A inverse range should return the EMPTY_DOCIDSET instance", DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(context, context.reader().getLiveDocs())); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java b/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java index d0153dbf0ab..ac2588f7b3f 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java @@ -22,8 +22,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.apache.lucene.index.CompositeReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.Term; @@ -310,13 +310,11 @@ public class TestShardSearching extends ShardSearchingTestBase { final int numNodes = shardSearcher.nodeVersions.length; int[] base = new int[numNodes]; - final List subs = ((CompositeReader) mockSearcher.getIndexReader()).getSequentialSubReaders(); + final List subs = mockSearcher.getTopReaderContext().children(); assertEquals(numNodes, subs.size()); - int docCount = 0; for(int nodeID=0;nodeID